diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index e66e372e..53559920 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -20,21 +20,27 @@ from home.src.index import ( YoutubeVideo, index_new_video ) -from home.src.helper import get_total_hits, clean_string, set_message +from home.src.helper import ( + get_total_hits, + clean_string, + set_message, + get_message +) class Reindex: """ check for outdated documents and refresh data from youtube """ def __init__(self): - self.video_daily, self.channel_daily = self.get_daily() - self.all_youtube_ids = False - self.all_channel_ids = False # config config = AppConfig().config self.sleep_interval = config['downloads']['sleep_interval'] self.es_url = config['application']['es_url'] self.refresh_interval = 90 + # scan + self.video_daily, self.channel_daily = self.get_daily() + self.all_youtube_ids = False + self.all_channel_ids = False def get_daily(self): """ get daily refresh values """ @@ -178,11 +184,13 @@ class Reindex: def reindex(self): """ reindex what's needed """ # videos + print(f'reindexing {len(self.all_youtube_ids)} videos') for youtube_id in self.all_youtube_ids: self.reindex_single_video(youtube_id) if self.sleep_interval: sleep(self.sleep_interval) # channels + print(f'reindexing {len(self.all_channel_ids)} channels') for channel_id in self.all_channel_ids: self.reindex_single_channel(channel_id) if self.sleep_interval: @@ -342,6 +350,14 @@ def scan_filesystem(): def reindex_old_documents(): """ daily refresh of old documents """ + # check needed last run + now = int(datetime.now().strftime("%s")) + last_reindex = get_message('last_reindex') + if isinstance(last_reindex, int) and now - last_reindex < 60 * 60 * 12: + return + # continue if needed reindex_handler = Reindex() reindex_handler.check_outdated() reindex_handler.reindex() + # set timestamp + set_message('last_reindex', now, expire=False) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 3fd756f7..01f13531 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -14,6 +14,7 @@ from home.src.download import ( VideoDownloader ) from home.src.config import AppConfig +from home.src.reindex import reindex_old_documents CONFIG = AppConfig().config @@ -33,6 +34,8 @@ def update_subscribed(): if missing_videos: pending_handler = PendingList() pending_handler.add_to_pending(missing_videos) + # check if reindex is needed + check_reindex.delay() @shared_task @@ -61,3 +64,9 @@ def extrac_dl(youtube_ids): pending_handler = PendingList() missing_videos = pending_handler.parse_url_list(youtube_ids) pending_handler.add_to_pending(missing_videos) + + +@shared_task +def check_reindex(): + """ run the reindex main command """ + reindex_old_documents()