refact CommentList to use RedisQueue

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -446,5 +446,9 @@ class DownloadPostProcess(DownloaderBase):
 
     def get_comments(self):
         """get comments from youtube"""
-        videos = RedisQueue(self.VIDEO_QUEUE).get_all()
-        CommentList(videos, task=self.task).index()
+        video_queue = RedisQueue(self.VIDEO_QUEUE)
+        comment_list = CommentList(task=self.task)
+        comment_list.add(video_ids=video_queue.get_all())
+
+        video_queue.clear()
+        comment_list.index()

tubearchivist/home/src/index/comments.py

@@ -10,6 +10,7 @@ from datetime import datetime
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
+from home.src.ta.ta_redis import RedisQueue
 
 
 class Comments:
@@ -189,20 +190,30 @@ class Comments:
 class CommentList:
     """interact with comments in group"""
 
-    def __init__(self, video_ids, task=False):
-        self.video_ids = video_ids
+    COMMENT_QUEUE = "index:comment"
+
+    def __init__(self, task=False):
         self.task = task
         self.config = AppConfig().config
 
-    def index(self):
-        """index comments for list, init with task object to notify"""
+    def add(self, video_ids: list[str]) -> None:
+        """add list of videos to get comments, if enabled in config"""
         if not self.config["downloads"].get("comment_max"):
             return
 
-        total_videos = len(self.video_ids)
-        for idx, youtube_id in enumerate(self.video_ids):
+        RedisQueue(self.COMMENT_QUEUE).add_list(video_ids)
+
+    def index(self):
+        """run comment index"""
+        queue = RedisQueue(self.COMMENT_QUEUE)
+        while True:
+            total = queue.max_score()
+            youtube_id, idx = queue.get_next()
+            if not youtube_id or not idx or not total:
+                break
+
             if self.task:
-                self.notify(idx, total_videos)
+                self.notify(idx, total)
 
             comment = Comments(youtube_id, config=self.config)
             comment.build_json()
@@ -211,6 +222,6 @@ class CommentList:
 
     def notify(self, idx, total_videos):
         """send notification on task"""
-        message = [f"Add comments for new videos {idx + 1}/{total_videos}"]
-        progress = (idx + 1) / total_videos
+        message = [f"Add comments for new videos {idx}/{total_videos}"]
+        progress = idx / total_videos
         self.task.send_progress(message, progress=progress)

tubearchivist/home/src/index/filesystem.py

@@ -89,7 +89,9 @@ class Scanner:
                 )
             index_new_video(youtube_id)
 
-        CommentList(self.to_index, task=self.task).index()
+        comment_list = CommentList(task=self.task)
+        comment_list.add(video_ids=list(self.to_index))
+        comment_list.index()
 
     def url_fix(self) -> None:
         """

tubearchivist/home/src/index/manual.py

@@ -147,7 +147,9 @@ class ImportFolderScanner:
             ManualImport(current_video, self.CONFIG).run()
 
         video_ids = [i["video_id"] for i in self.to_import]
-        CommentList(video_ids, task=self.task).index()
+        comment_list = CommentList(task=self.task)
+        comment_list.add(video_ids=video_ids)
+        comment_list.index()
 
     def _notify(self, idx, current_video):
         """send notification back to task"""