Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-02 09:41:07 +00:00
Code Issues Projects Releases Wiki Activity

index comments as a DownloadPostProcess

Browse Source
This commit is contained in:
simon 2022-11-18 09:22:58 +07:00
parent 172ced7129
commit 939cd7fb62
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -15,6 +15,7 @@ from home.src.download.subscriptions import PlaylistSubscription
from home.src.download.yt_dlp_base import CookieHandler, YtWrap from home.src.download.yt_dlp_base import CookieHandler, YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.channel import YoutubeChannel from home.src.index.channel import YoutubeChannel
from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
@ -39,6 +40,7 @@ class DownloadPostProcess:
self.auto_delete_all() self.auto_delete_all()
self.auto_delete_overwrites() self.auto_delete_overwrites()
self.validate_playlists() self.validate_playlists()
self.get_comments()
def auto_delete_all(self): def auto_delete_all(self):
"""handle auto delete""" """handle auto delete"""
@ -139,6 +141,16 @@ class DownloadPostProcess:
RedisArchivist().set_message(key, mess_dict, expire=expire) RedisArchivist().set_message(key, mess_dict, expire=expire)
def get_comments(self):
"""get comments from youtube"""
if not self.download.config["downloads"]["comment_max"]:
return
for video_id in self.download.videos:
comment = Comments(video_id)
comment.build_json()
comment.upload_comments()
class VideoDownloader: class VideoDownloader:
""" """
@ -155,6 +167,7 @@ class VideoDownloader:
self.config = AppConfig().config self.config = AppConfig().config
self._build_obs() self._build_obs()
self.channels = set() self.channels = set()
self.videos = set()
def run_queue(self): def run_queue(self):
"""setup download queue in redis loop until no more items""" """setup download queue in redis loop until no more items"""
@ -187,6 +200,7 @@ class VideoDownloader:
youtube_id, video_overwrites=self.video_overwrites youtube_id, video_overwrites=self.video_overwrites
) )
self.channels.add(vid_dict["channel"]["channel_id"]) self.channels.add(vid_dict["channel"]["channel_id"])
self.videos.add(vid_dict["youtube_id"])
mess_dict = { mess_dict = {
"status": self.MSG, "status": self.MSG,
"level": "info", "level": "info",

2
tubearchivist/home/src/index/comments.py
View File

@ -23,6 +23,7 @@ class Comments:
def build_json(self): def build_json(self):
"""build json document for es""" """build json document for es"""
print(f"{self.youtube_id}: get comments")
self._check_config() self._check_config()
comments_raw = self.get_yt_comments() comments_raw = self.get_yt_comments()
comments_format = self.format_comments(comments_raw) comments_format = self.format_comments(comments_raw)
@ -63,7 +64,6 @@ class Comments:
def get_yt_comments(self): def get_yt_comments(self):
"""get comments from youtube""" """get comments from youtube"""
print("comments: get comments")
yt_obs = self.build_yt_obs() yt_obs = self.build_yt_obs()
info_json = YtWrap(yt_obs).extract(self.youtube_id) info_json = YtWrap(yt_obs).extract(self.youtube_id)
comments_raw = info_json.get("comments") comments_raw = info_json.get("comments")