Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-02 09:41:07 +00:00
Code Issues Projects Releases Wiki Activity

add notification for comment indexing

Browse Source
This commit is contained in:
simon 2022-11-22 11:23:34 +07:00
parent b48b7c7e72
commit 90258e75e6
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 33 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -146,11 +146,22 @@ class DownloadPostProcess:
if not self.download.config["downloads"]["comment_max"]: if not self.download.config["downloads"]["comment_max"]:
return return
for video_id in self.download.videos: total_videos = len(self.download.videos)
for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config) comment = Comments(video_id, config=self.download.config)
comment.build_json() comment.build_json(notify=(idx, total_videos))
comment.upload_comments() comment.upload_comments()
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)
class VideoDownloader: class VideoDownloader:
""" """

22
tubearchivist/home/src/index/comments.py
View File

@ -10,6 +10,7 @@ from datetime import datetime
from home.src.download.yt_dlp_base import YtWrap from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap from home.src.es.connect import ElasticWrap
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.ta_redis import RedisArchivist
class Comments: class Comments:
@ -23,14 +24,14 @@ class Comments:
self.is_activated = False self.is_activated = False
self.comments_format = False self.comments_format = False
def build_json(self): def build_json(self, notify=False):
"""build json document for es""" """build json document for es"""
print(f"{self.youtube_id}: get comments") print(f"{self.youtube_id}: get comments")
self.check_config() self.check_config()
if not self.is_activated: if not self.is_activated:
return return
self._send_notification(notify)
comments_raw, channel_id = self.get_yt_comments() comments_raw, channel_id = self.get_yt_comments()
self.format_comments(comments_raw) self.format_comments(comments_raw)
@ -48,6 +49,23 @@ class Comments:
self.is_activated = bool(self.config["downloads"]["comment_max"]) self.is_activated = bool(self.config["downloads"]["comment_max"])
@staticmethod
def _send_notification(notify):
"""send notification for download post process message"""
if not notify:
return
key = "message:download"
idx, total_videos = notify
message = {
"status": key,
"level": "info",
"title": "Download and index comments",
"message": f"Progress: {idx + 1}/{total_videos}",
}
RedisArchivist().set_message(key, message)
def build_yt_obs(self): def build_yt_obs(self):
""" """
get extractor config get extractor config