Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-19 15:25:51 +00:00
Code Issues Projects Releases Wiki Activity

continue indexing if comment indexing failes, #383

Browse Source
This commit is contained in:
simon 2022-12-19 14:59:07 +07:00
parent f45a3095cb
commit e804dd1aec
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -150,6 +150,7 @@ class DownloadPostProcess:
for idx, video_id in enumerate(self.download.videos): for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config) comment = Comments(video_id, config=self.download.config)
comment.build_json(notify=(idx, total_videos)) comment.build_json(notify=(idx, total_videos))
if comment.json_data:
comment.upload_comments() comment.upload_comments()
key = "message:download" key = "message:download"

13
tubearchivist/home/src/index/comments.py
View File

@ -33,10 +33,10 @@ class Comments:
self._send_notification(notify) self._send_notification(notify)
comments_raw, channel_id = self.get_yt_comments() comments_raw, channel_id = self.get_yt_comments()
if comments_raw: if not comments_raw and not channel_id:
return
self.format_comments(comments_raw) self.format_comments(comments_raw)
else:
self.comments_format = []
self.json_data = { self.json_data = {
"youtube_id": self.youtube_id, "youtube_id": self.youtube_id,
@ -96,6 +96,9 @@ class Comments:
"""get comments from youtube""" """get comments from youtube"""
yt_obs = self.build_yt_obs() yt_obs = self.build_yt_obs()
info_json = YtWrap(yt_obs).extract(self.youtube_id) info_json = YtWrap(yt_obs).extract(self.youtube_id)
if not info_json:
return False, False
comments_raw = info_json.get("comments") comments_raw = info_json.get("comments")
channel_id = info_json.get("channel_id") channel_id = info_json.get("channel_id")
return comments_raw, channel_id return comments_raw, channel_id
@ -104,6 +107,7 @@ class Comments:
"""process comments to match format""" """process comments to match format"""
comments = [] comments = []
if comments_raw:
for comment in comments_raw: for comment in comments_raw:
cleaned_comment = self.clean_comment(comment) cleaned_comment = self.clean_comment(comment)
comments.append(cleaned_comment) comments.append(cleaned_comment)
@ -169,6 +173,9 @@ class Comments:
return return
self.build_json() self.build_json()
if not self.json_data:
return
es_comments = self.get_es_comments() es_comments = self.get_es_comments()
if not self.comments_format: if not self.comments_format: