Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-19 15:25:51 +00:00
Code Issues Projects Releases Wiki Activity

appending reindex task to rescan subscriptions

Browse Source
This commit is contained in:
simon 2021-09-11 17:55:44 +07:00
parent 5dee409116
commit 3376412c8f
2 changed files with 29 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tubearchivist/home/src/reindex.py
View File

@ -20,21 +20,27 @@ from home.src.index import (
YoutubeVideo, YoutubeVideo,
index_new_video index_new_video
) )
from home.src.helper import get_total_hits, clean_string, set_message from home.src.helper import (
get_total_hits,
clean_string,
set_message,
get_message
)
class Reindex: class Reindex:
""" check for outdated documents and refresh data from youtube """ """ check for outdated documents and refresh data from youtube """
def __init__(self): def __init__(self):
self.video_daily, self.channel_daily = self.get_daily()
self.all_youtube_ids = False
self.all_channel_ids = False
# config # config
config = AppConfig().config config = AppConfig().config
self.sleep_interval = config['downloads']['sleep_interval'] self.sleep_interval = config['downloads']['sleep_interval']
self.es_url = config['application']['es_url'] self.es_url = config['application']['es_url']
self.refresh_interval = 90 self.refresh_interval = 90
# scan
self.video_daily, self.channel_daily = self.get_daily()
self.all_youtube_ids = False
self.all_channel_ids = False
def get_daily(self): def get_daily(self):
""" get daily refresh values """ """ get daily refresh values """
@ -178,11 +184,13 @@ class Reindex:
def reindex(self): def reindex(self):
""" reindex what's needed """ """ reindex what's needed """
# videos # videos
print(f'reindexing {len(self.all_youtube_ids)} videos')
for youtube_id in self.all_youtube_ids: for youtube_id in self.all_youtube_ids:
self.reindex_single_video(youtube_id) self.reindex_single_video(youtube_id)
if self.sleep_interval: if self.sleep_interval:
sleep(self.sleep_interval) sleep(self.sleep_interval)
# channels # channels
print(f'reindexing {len(self.all_channel_ids)} channels')
for channel_id in self.all_channel_ids: for channel_id in self.all_channel_ids:
self.reindex_single_channel(channel_id) self.reindex_single_channel(channel_id)
if self.sleep_interval: if self.sleep_interval:
@ -342,6 +350,14 @@ def scan_filesystem():
def reindex_old_documents(): def reindex_old_documents():
""" daily refresh of old documents """ """ daily refresh of old documents """
# check needed last run
now = int(datetime.now().strftime("%s"))
last_reindex = get_message('last_reindex')
if isinstance(last_reindex, int) and now - last_reindex < 60 * 60 * 12:
return
# continue if needed
reindex_handler = Reindex() reindex_handler = Reindex()
reindex_handler.check_outdated() reindex_handler.check_outdated()
reindex_handler.reindex() reindex_handler.reindex()
# set timestamp
set_message('last_reindex', now, expire=False)

9
tubearchivist/home/tasks.py
View File

@ -14,6 +14,7 @@ from home.src.download import (
VideoDownloader VideoDownloader
) )
from home.src.config import AppConfig from home.src.config import AppConfig
from home.src.reindex import reindex_old_documents
CONFIG = AppConfig().config CONFIG = AppConfig().config
@ -33,6 +34,8 @@ def update_subscribed():
if missing_videos: if missing_videos:
pending_handler = PendingList() pending_handler = PendingList()
pending_handler.add_to_pending(missing_videos) pending_handler.add_to_pending(missing_videos)
# check if reindex is needed
check_reindex.delay()
@shared_task @shared_task
@ -61,3 +64,9 @@ def extrac_dl(youtube_ids):
pending_handler = PendingList() pending_handler = PendingList()
missing_videos = pending_handler.parse_url_list(youtube_ids) missing_videos = pending_handler.parse_url_list(youtube_ids)
pending_handler.add_to_pending(missing_videos) pending_handler.add_to_pending(missing_videos)
@shared_task
def check_reindex():
""" run the reindex main command """
reindex_old_documents()