|
|
@ -443,18 +443,22 @@ class YoutubePlaylist:
|
|
|
|
def __init__(self, playlist_id):
|
|
|
|
def __init__(self, playlist_id):
|
|
|
|
self.playlist_id = playlist_id
|
|
|
|
self.playlist_id = playlist_id
|
|
|
|
self.stamp = int(datetime.now().strftime("%s"))
|
|
|
|
self.stamp = int(datetime.now().strftime("%s"))
|
|
|
|
|
|
|
|
self.playlist_dict = False
|
|
|
|
|
|
|
|
|
|
|
|
def get_playlist_dict(self, scrape=False):
|
|
|
|
def get_playlist_dict(self, scrape=False):
|
|
|
|
"""get data from es or youtube"""
|
|
|
|
"""get data from es or youtube"""
|
|
|
|
print(f"get playlist with id {self.playlist_id}")
|
|
|
|
print(f"get playlist with id {self.playlist_id}")
|
|
|
|
|
|
|
|
|
|
|
|
if scrape:
|
|
|
|
if scrape:
|
|
|
|
playlist_dict = self.get_youtube_playlist()
|
|
|
|
playlist_dict = self.get_youtube_playlist()
|
|
|
|
|
|
|
|
playlist_dict["playlist_entries"] = self.get_entries()
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
playlist_dict = self.get_es_playlist()
|
|
|
|
playlist_dict = self.get_es_playlist()
|
|
|
|
if not playlist_dict:
|
|
|
|
if not playlist_dict:
|
|
|
|
playlist_dict = self.get_youtube_playlist()
|
|
|
|
playlist_dict = self.get_youtube_playlist()
|
|
|
|
|
|
|
|
playlist_dict["playlist_entries"] = self.get_entries()
|
|
|
|
|
|
|
|
|
|
|
|
return playlist_dict
|
|
|
|
self.playlist_dict = playlist_dict
|
|
|
|
|
|
|
|
|
|
|
|
def get_youtube_playlist(self):
|
|
|
|
def get_youtube_playlist(self):
|
|
|
|
"""get meta data dict from youtube"""
|
|
|
|
"""get meta data dict from youtube"""
|
|
|
@ -488,7 +492,7 @@ class YoutubePlaylist:
|
|
|
|
url = f"{self.ES_URL}/ta_playlist/_doc/{self.playlist_id}"
|
|
|
|
url = f"{self.ES_URL}/ta_playlist/_doc/{self.playlist_id}"
|
|
|
|
response = requests.get(url, auth=self.ES_AUTH)
|
|
|
|
response = requests.get(url, auth=self.ES_AUTH)
|
|
|
|
if response.ok:
|
|
|
|
if response.ok:
|
|
|
|
return json.loads(response.text)
|
|
|
|
return json.loads(response.text)["_source"]
|
|
|
|
|
|
|
|
|
|
|
|
return False
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
@ -524,14 +528,109 @@ class YoutubePlaylist:
|
|
|
|
|
|
|
|
|
|
|
|
def upload_to_es(self):
|
|
|
|
def upload_to_es(self):
|
|
|
|
"""add playlist to es with its entries"""
|
|
|
|
"""add playlist to es with its entries"""
|
|
|
|
playlist = self.get_playlist_dict()
|
|
|
|
playlist = self.playlist_dict
|
|
|
|
playlist["playlist_entries"] = self.get_entries()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
url = f"{self.ES_URL}/ta_playlist/_doc/{self.playlist_id}"
|
|
|
|
url = f"{self.ES_URL}/ta_playlist/_doc/{self.playlist_id}"
|
|
|
|
response = requests.put(url, json=playlist, auth=self.ES_AUTH)
|
|
|
|
response = requests.put(url, json=playlist, auth=self.ES_AUTH)
|
|
|
|
if not response.ok:
|
|
|
|
if not response.ok:
|
|
|
|
print(response.text)
|
|
|
|
print(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def add_vids_to_playlist(self):
|
|
|
|
|
|
|
|
"""sync the playlistdict to video dict"""
|
|
|
|
|
|
|
|
print("sync playlist meta data for " + self.playlist_id)
|
|
|
|
|
|
|
|
playlist_dict = self.playlist_dict
|
|
|
|
|
|
|
|
all_entries = playlist_dict["playlist_entries"]
|
|
|
|
|
|
|
|
vid_ids = [i["youtube_id"] for i in all_entries]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
to_add = {
|
|
|
|
|
|
|
|
key: val
|
|
|
|
|
|
|
|
for key, val in playlist_dict.items()
|
|
|
|
|
|
|
|
if key != "playlist_entries"
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
bulk_list = []
|
|
|
|
|
|
|
|
for youtube_id in vid_ids:
|
|
|
|
|
|
|
|
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
|
|
|
|
|
|
|
|
source = {"doc": {"playlist": to_add}}
|
|
|
|
|
|
|
|
bulk_list.append(json.dumps(action))
|
|
|
|
|
|
|
|
bulk_list.append(json.dumps(source))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# add last newline
|
|
|
|
|
|
|
|
bulk_list.append("\n")
|
|
|
|
|
|
|
|
query_str = "\n".join(bulk_list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
headers = {"Content-type": "application/x-ndjson"}
|
|
|
|
|
|
|
|
url = self.ES_URL + "/_bulk"
|
|
|
|
|
|
|
|
response = requests.post(
|
|
|
|
|
|
|
|
url, data=query_str, headers=headers, auth=self.ES_AUTH
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
if not response.ok:
|
|
|
|
|
|
|
|
print(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def playlist_position(self):
|
|
|
|
|
|
|
|
"""sync playlist_position to video dict"""
|
|
|
|
|
|
|
|
all_entries = self.playlist_dict["playlist_entries"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
bulk_list = []
|
|
|
|
|
|
|
|
for idx, entry in enumerate(all_entries):
|
|
|
|
|
|
|
|
youtube_id = entry["youtube_id"]
|
|
|
|
|
|
|
|
playlist_position = self.get_position_dict(all_entries, idx)
|
|
|
|
|
|
|
|
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
|
|
|
|
|
|
|
|
source = {"doc": {"playlist_position": playlist_position}}
|
|
|
|
|
|
|
|
bulk_list.append(json.dumps(action))
|
|
|
|
|
|
|
|
bulk_list.append(json.dumps(source))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# add last newline
|
|
|
|
|
|
|
|
bulk_list.append("\n")
|
|
|
|
|
|
|
|
query_str = "\n".join(bulk_list)
|
|
|
|
|
|
|
|
headers = {"Content-type": "application/x-ndjson"}
|
|
|
|
|
|
|
|
url = self.ES_URL + "/_bulk"
|
|
|
|
|
|
|
|
response = requests.post(
|
|
|
|
|
|
|
|
url, data=query_str, headers=headers, auth=self.ES_AUTH
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
if not response.ok:
|
|
|
|
|
|
|
|
print(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def get_position_dict(all_entries, idx):
|
|
|
|
|
|
|
|
"""get previous and next videos in playlist"""
|
|
|
|
|
|
|
|
playlist_position = {"playlist_index": idx}
|
|
|
|
|
|
|
|
if idx == 0:
|
|
|
|
|
|
|
|
playlist_position.update(
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
"playlist_prev_id": False,
|
|
|
|
|
|
|
|
"playlist_prev_title": False,
|
|
|
|
|
|
|
|
"playlist_prev_channel_name": False,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
prev_vid = all_entries[idx - 1]
|
|
|
|
|
|
|
|
playlist_position.update(
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
"playlist_prev_id": prev_vid["youtube_id"],
|
|
|
|
|
|
|
|
"playlist_prev_title": prev_vid["title"],
|
|
|
|
|
|
|
|
"playlist_prev_channel_name": prev_vid["uploader"],
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if idx == len(all_entries) - 1:
|
|
|
|
|
|
|
|
playlist_position.update(
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
"playlist_next_id": False,
|
|
|
|
|
|
|
|
"playlist_next_title": False,
|
|
|
|
|
|
|
|
"playlist_next_channel_name": False,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
next_vid = all_entries[idx + 1]
|
|
|
|
|
|
|
|
playlist_position.update(
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
"playlist_next_id": next_vid["youtube_id"],
|
|
|
|
|
|
|
|
"playlist_next_title": next_vid["title"],
|
|
|
|
|
|
|
|
"playlist_next_channel_name": next_vid["uploader"],
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return playlist_position
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WatchState:
|
|
|
|
class WatchState:
|
|
|
|
"""handle watched checkbox for videos and channels"""
|
|
|
|
"""handle watched checkbox for videos and channels"""
|
|
|
|