From 49b120ed2bc213c3a839737d5b4bcab5fbe4a3d5 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 10 Dec 2022 14:51:17 +0700 Subject: [PATCH 01/17] refactor RedisQueue to take a queue_name arg --- tubearchivist/api/views.py | 2 +- tubearchivist/home/src/download/yt_dlp_handler.py | 4 ++-- tubearchivist/home/src/frontend/api_calls.py | 4 ++-- tubearchivist/home/src/ta/ta_redis.py | 6 +++--- tubearchivist/home/tasks.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index f21292e8..3e5d79e8 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -382,7 +382,7 @@ class DownloadApiView(ApiBaseView): print(f"{video_id}: change status to {item_status}") PendingInteract(video_id=video_id, status=item_status).update_status() - RedisQueue().clear_item(video_id) + RedisQueue(queue_name="dl_queue").clear_item(video_id) return Response(request.data) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 3c6a4a56..c1d5ddde 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -184,7 +184,7 @@ class VideoDownloader: """setup download queue in redis loop until no more items""" self._setup_queue() - queue = RedisQueue() + queue = RedisQueue(queue_name="dl_queue") limit_queue = self.config["downloads"]["limit_count"] if limit_queue: @@ -275,7 +275,7 @@ class VideoDownloader: RedisArchivist().set_message(self.MSG, mess_dict, expire=True) return - RedisQueue().add_list(to_add) + RedisQueue(queue_name="dl_queue").add_list(to_add) def _progress_hook(self, response): """process the progress_hooks from yt_dlp""" diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index d8325fbe..6e0bb78d 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -123,7 +123,7 @@ class PostData: print(f"{video_id}: ignore video from download queue") PendingInteract(video_id=video_id, status="ignore").update_status() # also clear from redis queue - RedisQueue().clear_item(video_id) + RedisQueue(queue_name="dl_queue").clear_item(video_id) return {"success": True} @staticmethod @@ -141,7 +141,7 @@ class PostData: to_execute = self.exec_val if to_execute == "stop": print("stopping download queue") - RedisQueue().clear() + RedisQueue(queue_name="dl_queue").clear() elif to_execute == "kill": task_id = RedisArchivist().get_message("dl_queue_id") if not isinstance(task_id, str): diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 536209f5..e963dfdd 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -102,11 +102,11 @@ class RedisArchivist(RedisBase): class RedisQueue(RedisBase): - """dynamically interact with the download queue in redis""" + """dynamically interact with queues in redis""" - def __init__(self): + def __init__(self, queue_name): super().__init__() - self.key = self.NAME_SPACE + "dl_queue" + self.key = f"{self.NAME_SPACE}{queue_name}" def get_all(self): """return all elements in list""" diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index b868da50..d7200510 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -99,7 +99,7 @@ def download_pending(): @shared_task def download_single(youtube_id): """start download single video now""" - queue = RedisQueue() + queue = RedisQueue(queue_name="dl_queue") queue.add_priority(youtube_id) print("Added to queue with priority: " + youtube_id) # start queue if needed @@ -192,7 +192,7 @@ def kill_dl(task_id): app.control.revoke(task_id, terminate=True) _ = RedisArchivist().del_message("dl_queue_id") - RedisQueue().clear() + RedisQueue(queue_name="dl_queue").clear() clear_dl_cache(CONFIG) From abc3150f59db889d2e8b1fcf98c60908b5d3a0f1 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 12:02:38 +0700 Subject: [PATCH 02/17] bump libraries --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 50b135e6..a92e749e 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,11 +1,11 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.1.3 +Django==4.1.4 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.14.0 Pillow==9.3.0 -redis==4.3.5 +redis==4.4.0 requests==2.28.1 ryd-client==0.0.6 uWSGI==2.0.21 From 0d21bfe9296c1c906dbe7362709a48603be8121a Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 12:03:21 +0700 Subject: [PATCH 03/17] refactor Reindex to use RedisQueue --- tubearchivist/home/src/index/filesystem.py | 10 - tubearchivist/home/src/index/reindex.py | 280 +++++++++++---------- tubearchivist/home/tasks.py | 9 +- 3 files changed, 149 insertions(+), 150 deletions(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index bbe69fc4..38ca3e98 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -14,11 +14,9 @@ import subprocess from home.src.download.queue import PendingList from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap -from home.src.index.reindex import Reindex from home.src.index.video import YoutubeVideo, index_new_video from home.src.ta.config import AppConfig from home.src.ta.helper import clean_string, ignore_filelist -from home.src.ta.ta_redis import RedisArchivist from PIL import Image, ImageFile from yt_dlp.utils import ISO639Utils @@ -606,11 +604,3 @@ def scan_filesystem(): for missing_vid in filesystem_handler.to_index: youtube_id = missing_vid[2] index_new_video(youtube_id) - - -def reindex_old_documents(): - """daily refresh of old documents""" - handler = Reindex() - handler.check_outdated() - handler.reindex() - RedisArchivist().set_message("last_reindex", handler.now) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index a372b5bc..1bf9cb61 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -7,7 +7,6 @@ functionality: import os import shutil from datetime import datetime -from math import ceil from time import sleep from home.src.download.queue import PendingList @@ -20,135 +19,156 @@ from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig +from home.src.ta.ta_redis import RedisArchivist, RedisQueue + + +class ReindexBase: + """base config class for reindex task""" + + REINDEX_CONFIG = [ + { + "index_name": "ta_video", + "queue_name": "reindex:ta_video", + "active_key": "active", + "refresh_key": "vid_last_refresh", + }, + { + "index_name": "ta_channel", + "queue_name": "reindex:ta_channel", + "active_key": "channel_active", + "refresh_key": "channel_last_refresh", + }, + { + "index_name": "ta_playlist", + "queue_name": "reindex:ta_playlist", + "active_key": "playlist_active", + "refresh_key": "playlist_last_refresh", + }, + ] - -class Reindex: - """check for outdated documents and refresh data from youtube""" - - MATCH_FIELD = { - "ta_video": "active", - "ta_channel": "channel_active", - "ta_playlist": "playlist_active", - } MULTIPLY = 1.2 def __init__(self): - # config - self.now = int(datetime.now().strftime("%s")) self.config = AppConfig().config + self.now = int(datetime.now().strftime("%s")) + + def populate(self, all_ids, reindex_config): + """add all to reindex ids to redis queue""" + if not all_ids: + return + + RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids) + + +class ReindexOutdated(ReindexBase): + """add outdated documents to reindex queue""" + + def __init__(self): + super().__init__() self.interval = self.config["scheduler"]["check_reindex_days"] - # scan - self.all_youtube_ids = False - self.all_channel_ids = False - self.all_playlist_ids = False - - def check_cookie(self): - """validate cookie if enabled""" - if self.config["downloads"]["cookie_import"]: - valid = CookieHandler(self.config).validate() - if not valid: - return - - def _get_daily(self): - """get daily refresh values""" - total_videos = self._get_total_hits("ta_video") - video_daily = ceil(total_videos / self.interval * self.MULTIPLY) - if video_daily >= 10000: - video_daily = 9999 - - total_channels = self._get_total_hits("ta_channel") - channel_daily = ceil(total_channels / self.interval * self.MULTIPLY) - total_playlists = self._get_total_hits("ta_playlist") - playlist_daily = ceil(total_playlists / self.interval * self.MULTIPLY) - return (video_daily, channel_daily, playlist_daily) - - def _get_total_hits(self, index): + + def add_outdated(self): + """add outdated documents""" + for reindex_config in self.REINDEX_CONFIG: + total_hits = self._get_total_hits(reindex_config) + daily_should = self._get_daily_should(total_hits) + all_ids = self._get_outdated_ids(reindex_config, daily_should) + self.populate(all_ids, reindex_config) + + @staticmethod + def _get_total_hits(reindex_config): """get total hits from index""" - match_field = self.MATCH_FIELD[index] - path = f"{index}/_search?filter_path=hits.total" - data = {"query": {"match": {match_field: True}}} + index_name = reindex_config["index_name"] + active_key = reindex_config["active_key"] + path = f"{index_name}/_search?filter_path=hits.total" + data = {"query": {"match": {active_key: True}}} response, _ = ElasticWrap(path).post(data=data) total_hits = response["hits"]["total"]["value"] return total_hits - def _get_unrated_vids(self): - """get max 200 videos without rating if ryd integration is enabled""" - must_not_list = [ - {"exists": {"field": "stats.average_rating"}}, - {"term": {"active": {"value": False}}}, - ] - data = {"size": 200, "query": {"bool": {"must_not": must_not_list}}} - response, _ = ElasticWrap("ta_video/_search").get(data=data) + def _get_daily_should(self, total_hits): + """calc how many should reindex daily""" + daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY) + if daily_should >= 10000: + daily_should = 9999 - missing_rating = [i["_id"] for i in response["hits"]["hits"]] - self.all_youtube_ids = self.all_youtube_ids + missing_rating + return daily_should - def _get_outdated_vids(self, size): - """get daily videos to refresh""" + def _get_outdated_ids(self, reindex_config, daily_should): + """get outdated from index_name""" + index_name = reindex_config["index_name"] + refresh_key = reindex_config["refresh_key"] now_lte = self.now - self.interval * 24 * 60 * 60 must_list = [ {"match": {"active": True}}, - {"range": {"vid_last_refresh": {"lte": now_lte}}}, + {"range": {refresh_key: {"lte": now_lte}}}, ] data = { - "size": size, + "size": daily_should, "query": {"bool": {"must": must_list}}, - "sort": [{"vid_last_refresh": {"order": "asc"}}], + "sort": [{refresh_key: {"order": "asc"}}], "_source": False, } - response, _ = ElasticWrap("ta_video/_search").get(data=data) + response, _ = ElasticWrap(f"{index_name}/_search").get(data=data) - all_youtube_ids = [i["_id"] for i in response["hits"]["hits"]] - return all_youtube_ids + all_ids = [i["_id"] for i in response["hits"]["hits"]] + return all_ids - def _get_outdated_channels(self, size): - """get daily channels to refresh""" - now_lte = self.now - self.interval * 24 * 60 * 60 - must_list = [ - {"match": {"channel_active": True}}, - {"range": {"channel_last_refresh": {"lte": now_lte}}}, - ] - data = { - "size": size, - "query": {"bool": {"must": must_list}}, - "sort": [{"channel_last_refresh": {"order": "asc"}}], - "_source": False, - } - response, _ = ElasticWrap("ta_channel/_search").get(data=data) - all_channel_ids = [i["_id"] for i in response["hits"]["hits"]] - return all_channel_ids +class Reindex(ReindexBase): + """reindex all documents from redis queue""" - def _get_outdated_playlists(self, size): - """get daily outdated playlists to refresh""" - now_lte = self.now - self.interval * 24 * 60 * 60 - must_list = [ - {"match": {"playlist_active": True}}, - {"range": {"playlist_last_refresh": {"lte": now_lte}}}, - ] - data = { - "size": size, - "query": {"bool": {"must": must_list}}, - "sort": [{"playlist_last_refresh": {"order": "asc"}}], - "_source": False, - } - response, _ = ElasticWrap("ta_playlist/_search").get(data=data) + def __init__(self): + super().__init__() + self.all_indexed_ids = False - all_playlist_ids = [i["_id"] for i in response["hits"]["hits"]] - return all_playlist_ids + def reindex_all(self): + """reindex all in queue""" + if self.cookie_invalid(): + print("[reindex] cookie invalid, exiting...") + return - def check_outdated(self): - """add missing vids and channels""" - video_daily, channel_daily, playlist_daily = self._get_daily() - self.all_youtube_ids = self._get_outdated_vids(video_daily) - self.all_channel_ids = self._get_outdated_channels(channel_daily) - self.all_playlist_ids = self._get_outdated_playlists(playlist_daily) + for index_config in self.REINDEX_CONFIG: + if not RedisQueue(index_config["queue_name"]).has_item(): + continue + + while True: + has_next = self.reindex_index(index_config) + if not has_next: + break + + RedisArchivist().set_message("last_reindex", self.now) + + def reindex_index(self, index_config): + """reindex all of a single index""" + reindex = self.get_reindex_map(index_config["index_name"]) + youtube_id = RedisQueue(index_config["queue_name"]).get_next() + if youtube_id: + reindex(youtube_id) + sleep_interval = self.config["downloads"].get("sleep_interval", 0) + sleep(sleep_interval) + + return bool(youtube_id) + + def get_reindex_map(self, index_name): + """return def to run for index""" + def_map = { + "ta_video": self._reindex_single_video, + "ta_channel": self._reindex_single_channel, + "ta_playlist": self._reindex_single_playlist, + } - integrate_ryd = self.config["downloads"]["integrate_ryd"] - if integrate_ryd: - self._get_unrated_vids() + return def_map.get(index_name) def _reindex_single_video(self, youtube_id): + """wrapper to handle channel name changes""" + try: + self._reindex_single_video_call(youtube_id) + except FileNotFoundError: + ChannelUrlFixer(youtube_id, self.config) + self._reindex_single_video_call(youtube_id) + + def _reindex_single_video_call(self, youtube_id): """refresh data for single video""" video = YoutubeVideo(youtube_id) @@ -206,13 +226,13 @@ class Reindex: channel.upload_to_es() channel.sync_to_videos() - @staticmethod - def _reindex_single_playlist(playlist_id, all_indexed_ids): + def _reindex_single_playlist(self, playlist_id): """refresh playlist data""" + self._get_all_videos() playlist = YoutubePlaylist(playlist_id) playlist.get_from_es() subscribed = playlist.json_data["playlist_subscribed"] - playlist.all_youtube_ids = all_indexed_ids + playlist.all_youtube_ids = self.all_indexed_ids playlist.build_json(scrape=True) if not playlist.json_data: playlist.deactivate() @@ -222,37 +242,29 @@ class Reindex: playlist.upload_to_es() return - def reindex(self): - """reindex what's needed""" - sleep_interval = self.config["downloads"]["sleep_interval"] - # videos - print(f"reindexing {len(self.all_youtube_ids)} videos") - for youtube_id in self.all_youtube_ids: - try: - self._reindex_single_video(youtube_id) - except FileNotFoundError: - # handle channel name change here - ChannelUrlFixer(youtube_id, self.config).run() - self._reindex_single_video(youtube_id) - if sleep_interval: - sleep(sleep_interval) - # channels - print(f"reindexing {len(self.all_channel_ids)} channels") - for channel_id in self.all_channel_ids: - self._reindex_single_channel(channel_id) - if sleep_interval: - sleep(sleep_interval) - # playlist - print(f"reindexing {len(self.all_playlist_ids)} playlists") - if self.all_playlist_ids: - handler = PendingList() - handler.get_download() - handler.get_indexed() - all_indexed_ids = [i["youtube_id"] for i in handler.all_videos] - for playlist_id in self.all_playlist_ids: - self._reindex_single_playlist(playlist_id, all_indexed_ids) - if sleep_interval: - sleep(sleep_interval) + def _get_all_videos(self): + """add all videos for playlist index validation""" + if self.all_indexed_ids: + return + + handler = PendingList() + handler.get_download() + handler.get_indexed() + self.all_indexed_ids = [i["youtube_id"] for i in handler.all_videos] + + def cookie_invalid(self): + """return true if cookie is enabled and invalid""" + if not self.config["downloads"]["cookie_import"]: + return False + + valid = CookieHandler(self.config).validate() + return valid + + +def reindex_outdated(): + """reindex all outdated""" + ReindexOutdated().add_outdated() + Reindex().reindex_all() class ChannelUrlFixer: diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index d7200510..ecadf8e8 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -20,11 +20,8 @@ from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.backup import ElasticBackup from home.src.es.index_setup import ElasitIndexWrap from home.src.index.channel import YoutubeChannel -from home.src.index.filesystem import ( - ImportFolderScanner, - reindex_old_documents, - scan_filesystem, -) +from home.src.index.filesystem import ImportFolderScanner, scan_filesystem +from home.src.index.reindex import reindex_outdated from home.src.ta.config import AppConfig, ScheduleBuilder from home.src.ta.helper import UrlListParser, clear_dl_cache from home.src.ta.ta_redis import RedisArchivist, RedisQueue @@ -138,7 +135,7 @@ def extrac_dl(youtube_ids): @shared_task(name="check_reindex") def check_reindex(): """run the reindex main command""" - reindex_old_documents() + reindex_outdated() @shared_task From 762a0fe8a1ad8dd88969f598a04d0440b0e216a6 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 14:57:39 +0700 Subject: [PATCH 04/17] add default sort _doc in IndexPaginate --- tubearchivist/home/src/es/connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index 75ab6a9a..aac13dcc 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -125,8 +125,7 @@ class IndexPaginate: def validate_data(self): """add pit and size to data""" if "sort" not in self.data.keys(): - print(self.data) - raise ValueError("missing sort key in data") + self.data.update({"sort": [{"_doc": {"order": "desc"}}]}) self.data["size"] = self.size or self.DEFAULT_SIZE self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"} From 617790b68fadfda682b6f9b52210438b79fe491a Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 15:39:40 +0700 Subject: [PATCH 05/17] add ReindexManual to control reindex from API --- tubearchivist/home/src/index/reindex.py | 92 ++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 1bf9cb61..6c7b32d8 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -13,7 +13,7 @@ from home.src.download.queue import PendingList from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import CookieHandler from home.src.download.yt_dlp_handler import VideoDownloader -from home.src.es.connect import ElasticWrap +from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist @@ -28,18 +28,21 @@ class ReindexBase: REINDEX_CONFIG = [ { "index_name": "ta_video", + "index_type": "videos", "queue_name": "reindex:ta_video", "active_key": "active", "refresh_key": "vid_last_refresh", }, { "index_name": "ta_channel", + "index_type": "channels", "queue_name": "reindex:ta_channel", "active_key": "channel_active", "refresh_key": "channel_last_refresh", }, { "index_name": "ta_playlist", + "index_type": "playlists", "queue_name": "reindex:ta_playlist", "active_key": "playlist_active", "refresh_key": "playlist_last_refresh", @@ -115,6 +118,93 @@ class ReindexOutdated(ReindexBase): return all_ids +class ReindexManual(ReindexBase): + """ + manually add ids to reindex queue from API + data_example = { + "videos": ["video1", "video2", "video3"], + "channels": ["channel1", "channel2", "channel3"], + "playlists": ["playlist1", "playlist2"], + } + extract_videos to also reindex all videos of channel/playlist + """ + + def __init__(self, extract_videos=False): + super().__init__() + self.extract_videos = extract_videos + self.data = False + + def extract_data(self, data): + """process data""" + self.data = data + for key, values in self.data.items(): + reindex_config = self._get_reindex_config(key) + self.process_index(reindex_config, values) + + def _get_reindex_config(self, index_type): + """get reindex config for index""" + + for reindex_config in self.REINDEX_CONFIG: + if reindex_config["index_type"] == index_type: + return reindex_config + + print(f"reindex type {index_type} not valid") + raise ValueError + + def process_index(self, index_config, values): + """process values per index""" + index_name = index_config["index_name"] + if index_name == "ta_video": + self._add_videos(values) + elif index_name == "ta_channel": + self._add_channels(values) + elif index_name == "ta_playlist": + self._add_playlists(values) + + def _add_videos(self, values): + """add list of videos to reindex queue""" + if not values: + return + + RedisQueue("reindex:ta_video").add_list(values) + + def _add_channels(self, values): + """add list of channels to reindex queue""" + RedisQueue("reindex:ta_channel").add_list(values) + + if self.extract_videos: + for channel_id in values: + all_videos = self._get_channel_videos(channel_id) + self._add_videos(all_videos) + + def _add_playlists(self, values): + """add list of playlists to reindex queue""" + RedisQueue("reindex:ta_playlist").add_list(values) + + if self.extract_videos: + for playlist_id in values: + all_videos = self._get_playlist_videos(playlist_id) + self._add_videos(all_videos) + + def _get_channel_videos(self, channel_id): + """get all videos from channel""" + data = { + "query": {"term": {"channel.channel_id": {"value": channel_id}}}, + "_source": ["youtube_id"], + } + all_results = IndexPaginate("ta_video", data).get_results() + return [i["youtube_id"] for i in all_results] + + def _get_playlist_videos(self, playlist_id): + """get all videos from playlist""" + data = { + "query": {"term": {"playlist.keyword": {"value": playlist_id}}}, + "_source": ["youtube_id"], + } + all_results = IndexPaginate("ta_video", data).get_results() + return [i["youtube_id"] for i in all_results] + + class Reindex(ReindexBase): """reindex all documents from redis queue""" From 0f6bc3a420aa26f1827f49c1065fef9b59246651 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 15:56:44 +0700 Subject: [PATCH 06/17] refactor check_reindex task for ReindexManual --- tubearchivist/home/src/index/reindex.py | 6 ------ tubearchivist/home/tasks.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 6c7b32d8..f546f2d8 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -351,12 +351,6 @@ class Reindex(ReindexBase): return valid -def reindex_outdated(): - """reindex all outdated""" - ReindexOutdated().add_outdated() - Reindex().reindex_all() - - class ChannelUrlFixer: """fix not matching channel names in reindex""" diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index ecadf8e8..18ba2be1 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -21,7 +21,7 @@ from home.src.es.backup import ElasticBackup from home.src.es.index_setup import ElasitIndexWrap from home.src.index.channel import YoutubeChannel from home.src.index.filesystem import ImportFolderScanner, scan_filesystem -from home.src.index.reindex import reindex_outdated +from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated from home.src.ta.config import AppConfig, ScheduleBuilder from home.src.ta.helper import UrlListParser, clear_dl_cache from home.src.ta.ta_redis import RedisArchivist, RedisQueue @@ -133,9 +133,14 @@ def extrac_dl(youtube_ids): @shared_task(name="check_reindex") -def check_reindex(): +def check_reindex(data=False, extract_videos=False): """run the reindex main command""" - reindex_outdated() + if data: + ReindexManual(extract_videos=extract_videos).extract_data(data) + else: + ReindexOutdated().add_outdated() + + Reindex().reindex_all() @shared_task From 018b578982e4e7b9763d496e108685857b02055b Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 11 Dec 2022 17:13:07 +0700 Subject: [PATCH 07/17] [API] add reindex endpoing --- tubearchivist/api/README.md | 15 +++++++++++++++ tubearchivist/api/urls.py | 6 ++++++ tubearchivist/api/views.py | 22 +++++++++++++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 7b647488..3c133bbe 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -38,6 +38,7 @@ Note: **Additional** - [Login](#login-view) - [Task](#task-view) WIP +- [Refresh](#refresh-view) - [Cookie](#cookie-view) - [Search](#search-view) - [Ping](#ping-view) @@ -306,6 +307,20 @@ List of valid task names: - **download_pending**: Start the download queue - **rescan_pending**: Rescan your subscriptions +## Refresh View +GET /api/refresh/ +POST /api/refresh/ +Parameter: +- extract_videos: to refresh all videos for channels/playlists, default False + +Manually start a refresh task: post list of *videos*, *channels*, *playlists* +```json +{ + "videos": ["video1", "video2", "video3"], + "channels": ["channel1", "channel2", "channel3"], + "playlists": ["playlist1", "playlist2"] +} +``` ## Cookie View Check your youtube cookie settings, *status* turns to `true` if cookie has been validated. diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 8662ad0a..0476719f 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -12,6 +12,7 @@ from api.views import ( PlaylistApiListView, PlaylistApiVideoView, PlaylistApiView, + RefreshView, SearchView, SnapshotApiListView, SnapshotApiView, @@ -98,6 +99,11 @@ urlpatterns = [ DownloadApiView.as_view(), name="api-download", ), + path( + "refresh/", + RefreshView.as_view(), + name="api-refresh", + ), path( "task/", TaskApiView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 3e5d79e8..aebbf297 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -12,7 +12,7 @@ from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist, RedisQueue -from home.tasks import extrac_dl, subscribe_to +from home.tasks import check_reindex, extrac_dl, subscribe_to from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, @@ -589,6 +589,26 @@ class SnapshotApiView(ApiBaseView): return Response(response) +class RefreshView(ApiBaseView): + """resolves to /api/refresh/ + GET: get refresh progress + POST: start a manual refresh task + """ + + def get(self, request): + """handle get request""" + # pylint: disable=unused-argument + return Response({"status": False}) + + def post(self, request): + """handle post request""" + data = request.data + extract_videos = bool(request.GET.get("extract_videos", False)) + check_reindex.delay(data=data, extract_videos=extract_videos) + + return Response(data) + + class CookieView(ApiBaseView): """resolves to /api/cookie/ GET: check if cookie is enabled From c25af5cfaac9b6213731ca79f57f4160bf3d3919 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 12 Dec 2022 17:10:47 +0700 Subject: [PATCH 08/17] add basic architecture overview --- ARCHITECTURE.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ARCHITECTURE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..966d2380 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,48 @@ +# The Inner Workings of Tube Archivist +This is a high level overview of the architecture of Tube Archivist, intended for interested contributors to find your way around quickly. + +``` + Tube Archivist + ______________________|_____________________ + | | | +------------------- --------------- ------------------- +| | | | | | +| DjangoProject | | RedisJson | | ElasticSearch | +| | | | | | +------------------- --------------- ------------------- +``` + +## DjangoProject +This is the main Python application. Django serves its data container internally with **Uwsgi** on port 8080, the interface is served with **Nginx** on the public port 8000. + +Users created static files like media files and artwork as well as application artwork like logos and fonts are served directly from Nginx, while the rest of the application uses uwsgi_pass to proxy the requests to uwsgi. + +Config files are located in the `docker_assets` folder. The script `run.sh` is the container `CMD` command and entry point, validating env vars, connection to ElasticSearch (ES) and will start the application. + +Compared to other Django projects, this application doesn't make use of the database models, due to a lack of integration with ES. This project has its own abstractions and integrations, treating ES as a REST API. + +Long running application tasks are handed off to **Celery** - using **Redis** as a broker - to run asynchronously from the main threads. +- All tasks are defined in the `home.tasks.py` module. + +There are three Django apps: +- **config**: The root app, routing the main endpoints and the main `settings.py` file +- **api**: The API app with its views and functionality +- **home**: Most of the application logic, templates and views, will probably get split up further in the future. + +The *home* app is split up into packages in the `src` directory: +- **download**: All download related classes, interact with yt-dlp, download artwork, handle the download queue and post processing tasks. +- **es**: All index setup and validation classes, handles mapping validations and makes mapping changes, wrapper functions to simplify interactions with Elasticsearch, backup and restore. +- **frontend**: All direct interactions with the frontend, like Django forms, searching, watched state changes, and legacy api_calls in the process of moving to the api app. +- **index**: Contains all functionality for scraping and indexing videos, channels, playlists, comments, subtitles, etc... +- **ta**: Loose collection of functions and classes, handle application config and contains redis wrapper classes. + +## RedisJson +Holds the main application config json object that gets dynamically edited from the frontend, serves as a message broker for **Celery**. Redis serves as a temporary and thread safe link between Django and the frontend, storing progress messages and temporary queues for processing. Used to store locking keys for threads and execution details for tasks. + +- Wrapper classes to interact with Redis are located in the `home.src.ta.ta_redis.py` module. + +## ElasticSearch (ES) +Is used to store and index all metadata, functions as an application database and makes it all searchable. The mapping defines which fields are indexed as searchable text fields and which fields are used for match filtering. + +- The index setup and validation is handled in the `home.src.es.index_setup.py` module. +- Wrapper classes for making requests to ES are located in the `home.src.es.connect.py` module. From 3b3d151ec3dfa2ba738ab790525d1fadfad2066d Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 12 Dec 2022 17:33:57 +0700 Subject: [PATCH 09/17] add reindex task lock, implement add to running queue --- tubearchivist/home/apps.py | 1 + tubearchivist/home/tasks.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py index d52db7fc..657c0e2b 100644 --- a/tubearchivist/home/apps.py +++ b/tubearchivist/home/apps.py @@ -77,6 +77,7 @@ class StartupCheck: "downloading", "dl_queue", "dl_queue_id", + "reindex", "rescan", "run_backup", ] diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 18ba2be1..ee830e95 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -137,10 +137,23 @@ def check_reindex(data=False, extract_videos=False): """run the reindex main command""" if data: ReindexManual(extract_videos=extract_videos).extract_data(data) - else: - ReindexOutdated().add_outdated() - Reindex().reindex_all() + have_lock = False + reindex_lock = RedisArchivist().get_lock("reindex") + + try: + have_lock = reindex_lock.acquire(blocking=False) + if have_lock: + if not data: + ReindexOutdated().add_outdated() + + Reindex().reindex_all() + else: + print("Did not acquire reindex lock.") + + finally: + if have_lock: + reindex_lock.release() @shared_task From 2753ce93a24be571f071895f447a24e0093f4a66 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 12 Dec 2022 18:01:02 +0700 Subject: [PATCH 10/17] add contains to RedisQueue --- tubearchivist/home/src/ta/ta_redis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index e963dfdd..40529cd3 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -114,6 +114,11 @@ class RedisQueue(RedisBase): all_elements = [i.decode() for i in result] return all_elements + def contains(self, element): + """check if element is in list""" + result = self.conn.execute_command("LPOS", self.key, element) + return result is not None + def add_list(self, to_add): """add list to queue""" self.conn.execute_command("RPUSH", self.key, *to_add) From 21028d4e254df60ffd0f08d8aac34c2c80f3cdc1 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 12 Dec 2022 18:21:31 +0700 Subject: [PATCH 11/17] dict better REINDEX_CONFIG datatype --- tubearchivist/api/README.md | 8 ++--- tubearchivist/home/src/index/reindex.py | 39 ++++++++++--------------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 3c133bbe..1a0a7237 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -313,12 +313,12 @@ POST /api/refresh/ Parameter: - extract_videos: to refresh all videos for channels/playlists, default False -Manually start a refresh task: post list of *videos*, *channels*, *playlists* +Manually start a refresh task: post list of *video*, *channel*, *playlist* IDs. ```json { - "videos": ["video1", "video2", "video3"], - "channels": ["channel1", "channel2", "channel3"], - "playlists": ["playlist1", "playlist2"] + "video": ["video1", "video2", "video3"], + "channel": ["channel1", "channel2", "channel3"], + "playlist": ["playlist1", "playlist2"] } ``` diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index f546f2d8..e02d11a3 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -25,29 +25,26 @@ from home.src.ta.ta_redis import RedisArchivist, RedisQueue class ReindexBase: """base config class for reindex task""" - REINDEX_CONFIG = [ - { + REINDEX_CONFIG = { + "video": { "index_name": "ta_video", - "index_type": "videos", "queue_name": "reindex:ta_video", "active_key": "active", "refresh_key": "vid_last_refresh", }, - { + "channel": { "index_name": "ta_channel", - "index_type": "channels", "queue_name": "reindex:ta_channel", "active_key": "channel_active", "refresh_key": "channel_last_refresh", }, - { + "playlist": { "index_name": "ta_playlist", - "index_type": "playlists", "queue_name": "reindex:ta_playlist", "active_key": "playlist_active", "refresh_key": "playlist_last_refresh", }, - ] + } MULTIPLY = 1.2 @@ -72,7 +69,7 @@ class ReindexOutdated(ReindexBase): def add_outdated(self): """add outdated documents""" - for reindex_config in self.REINDEX_CONFIG: + for reindex_config in self.REINDEX_CONFIG.values(): total_hits = self._get_total_hits(reindex_config) daily_should = self._get_daily_should(total_hits) all_ids = self._get_outdated_ids(reindex_config, daily_should) @@ -122,9 +119,9 @@ class ReindexManual(ReindexBase): """ manually add ids to reindex queue from API data_example = { - "videos": ["video1", "video2", "video3"], - "channels": ["channel1", "channel2", "channel3"], - "playlists": ["playlist1", "playlist2"], + "video": ["video1", "video2", "video3"], + "channel": ["channel1", "channel2", "channel3"], + "playlist": ["playlist1", "playlist2"], } extract_videos to also reindex all videos of channel/playlist """ @@ -138,18 +135,12 @@ class ReindexManual(ReindexBase): """process data""" self.data = data for key, values in self.data.items(): - reindex_config = self._get_reindex_config(key) - self.process_index(reindex_config, values) - - def _get_reindex_config(self, index_type): - """get reindex config for index""" + reindex_config = self.REINDEX_CONFIG.get(key) + if not reindex_config: + print(f"reindex type {key} not valid") + raise ValueError - for reindex_config in self.REINDEX_CONFIG: - if reindex_config["index_type"] == index_type: - return reindex_config - - print(f"reindex type {index_type} not valid") - raise ValueError + self.process_index(reindex_config, values) def process_index(self, index_config, values): """process values per index""" @@ -218,7 +209,7 @@ class Reindex(ReindexBase): print("[reindex] cookie invalid, exiting...") return - for index_config in self.REINDEX_CONFIG: + for index_config in self.REINDEX_CONFIG.values(): if not RedisQueue(index_config["queue_name"]).has_item(): continue From 71dd63d3f06d8147748fc5ead38bc60ab667fad7 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 19 Dec 2022 11:24:37 +0700 Subject: [PATCH 12/17] [API] implement refresh GET and POST endpoints --- tubearchivist/api/README.md | 32 +++++++++++ tubearchivist/api/views.py | 17 +++++- tubearchivist/home/src/index/reindex.py | 74 +++++++++++++++++++++++++ tubearchivist/home/src/ta/ta_redis.py | 7 ++- 4 files changed, 126 insertions(+), 4 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 1a0a7237..3b69e763 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -309,6 +309,38 @@ List of valid task names: ## Refresh View GET /api/refresh/ +parameters: +- **type**: one of *video*, *channel*, *playlist*, optional +- **id**: item id, optional + +without specifying type: return total for all queued items: +```json +{ + "total_queued": 2, + "type": "all", + "state": "running" +} +``` + +specify type: return total items queue of this type: +```json +{ + "total_queued": 2, + "type": "video", + "state": "running" +} +``` + +specify type *and* id to get state of item in queue: +```json +{ + "total_queued": 2, + "type": "video", + "state": "in_queue", + "id": "video-id" +} +``` + POST /api/refresh/ Parameter: - extract_videos: to refresh all videos for channels/playlists, default False diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index aebbf297..df8554cc 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -8,6 +8,7 @@ from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.searching import SearchForm from home.src.index.generic import Pagination +from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser @@ -597,8 +598,20 @@ class RefreshView(ApiBaseView): def get(self, request): """handle get request""" - # pylint: disable=unused-argument - return Response({"status": False}) + request_type = request.GET.get("type") + request_id = request.GET.get("id") + + if request_id and not request_type: + return Response({"status": "Bad Request"}, status=400) + + try: + progress = ReindexProgress( + request_type=request_type, request_id=request_id + ).get_progress() + except ValueError: + return Response({"status": "Bad Request"}, status=400) + + return Response(progress) def post(self, request): """handle post request""" diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index e02d11a3..780d7dea 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -342,6 +342,80 @@ class Reindex(ReindexBase): return valid +class ReindexProgress(ReindexBase): + """ + get progress of reindex task + request_type: key of self.REINDEX_CONFIG + request_id: id of request_type + return = { + "state": "running" | "queued" | False + "total_queued": int + "in_queue_name": "queue_name" + } + """ + + def __init__(self, request_type=False, request_id=False): + super().__init__() + self.request_type = request_type + self.request_id = request_id + + def get_progress(self): + """get progress from task""" + queue_name, request_type = self._get_queue_name() + total = self._get_total_in_queue(queue_name) + + progress = { + "total_queued": total, + "type": request_type, + } + state = self._get_state(total, queue_name) + progress.update(state) + + return progress + + def _get_queue_name(self): + """return queue_name, queue_type, raise exception on error""" + if not self.request_type: + return "all", "all" + + reindex_config = self.REINDEX_CONFIG.get(self.request_type) + if not reindex_config: + print(f"reindex_config not found: {self.request_type}") + raise ValueError + + return reindex_config["queue_name"], self.request_type + + def _get_total_in_queue(self, queue_name): + """get all items in queue""" + total = 0 + if queue_name == "all": + queues = [i["queue_name"] for i in self.REINDEX_CONFIG.values()] + for queue in queues: + total += len(RedisQueue(queue).get_all()) + else: + total += len(RedisQueue(queue_name).get_all()) + + return total + + def _get_state(self, total, queue_name): + """get state based on request_id""" + state_dict = {} + if self.request_id: + state = RedisQueue(queue_name).in_queue(self.request_id) + state_dict.update({"id": self.request_id, "state": state}) + + return state_dict + + if total: + state = "running" + else: + state = "empty" + + state_dict.update({"state": state}) + + return state_dict + + class ChannelUrlFixer: """fix not matching channel names in reindex""" diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 40529cd3..14e90a11 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -114,10 +114,13 @@ class RedisQueue(RedisBase): all_elements = [i.decode() for i in result] return all_elements - def contains(self, element): + def in_queue(self, element): """check if element is in list""" result = self.conn.execute_command("LPOS", self.key, element) - return result is not None + if result is not None: + return "in_queue" + + return False def add_list(self, to_add): """add list to queue""" From 18f6455eb277082cadcafc32fa36c49e0933604e Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 19 Dec 2022 13:04:53 +0700 Subject: [PATCH 13/17] add reindex buttons to templates --- .../home/templates/home/channel_id_about.html | 16 +++++++-- .../home/templates/home/playlist_id.html | 8 +++++ tubearchivist/home/templates/home/video.html | 17 +++++++--- tubearchivist/home/views.py | 34 ++++++++++++------- tubearchivist/static/css/style.css | 4 +++ tubearchivist/static/script.js | 18 ++++++++++ 6 files changed, 77 insertions(+), 20 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index 5c6ac892..9f227b0a 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -47,10 +47,20 @@ {% elif channel_info.channel_views > 0 %}

Channel views: {{ channel_info.channel_views|intcomma }}

{% endif %} - -
- Delete {{ channel_info.channel_name }} including all videos? +
+ +
+ Delete {{ channel_info.channel_name }} including all videos? +
+ {% if reindex %} +

Reindex scheduled

+ {% else %} +
+ + +
+ {% endif %}
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index ca7a6c00..92a450dd 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -52,6 +52,14 @@

Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}

Watched:

{% endif %} + {% if reindex %} +

Reindex scheduled

+ {% else %} +
+ + +
+ {% endif %} diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index a5784231..904f8a34 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -56,10 +56,19 @@ {% else %}

Youtube: Deactivated

{% endif %} - - -
- Are you sure? + {% if reindex %} +

Reindex scheduled

+ {% else %} +
+ +
+ {% endif %} +
+ + +
+ Are you sure? +
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 18810ad9..a305bbe2 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -35,6 +35,7 @@ from home.src.frontend.searching import SearchHandler from home.src.index.channel import YoutubeChannel, channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist +from home.src.index.reindex import ReindexProgress from home.src.ta.config import AppConfig, ScheduleBuilder from home.src.ta.helper import UrlListParser, time_parser from home.src.ta.ta_redis import RedisArchivist @@ -569,17 +570,18 @@ class ChannelIdAboutView(ChannelIdBaseView): self.initiate_vars(request) self.channel_has_pending(channel_id) - path = f"ta_channel/_doc/{channel_id}" - response, _ = ElasticWrap(path).get() - + response, _ = ElasticWrap(f"ta_channel/_doc/{channel_id}").get() channel_info = SearchProcess(response).process() - channel_name = channel_info["channel_name"] + reindex = ReindexProgress( + request_type="channel", request_id=channel_id + ).get_progress() self.context.update( { - "title": "Channel: About " + channel_name, + "title": "Channel: About " + channel_info["channel_name"], "channel_info": channel_info, "channel_overwrite_form": ChannelOverwriteForm, + "reindex": reindex.get("state"), } ) @@ -706,12 +708,17 @@ class PlaylistIdView(ArchivistResultsView): self._update_view_data(playlist_id, playlist_info) self.find_results() self.match_progress() + reindex = ReindexProgress( + request_type="playlist", request_id=playlist_id + ).get_progress() + self.context.update( { "title": "Playlist: " + playlist_name, "playlist_info": playlist_info, "playlist_name": playlist_name, "channel_info": channel_info, + "reindex": reindex.get("state"), } ) return render(request, "home/playlist_id.html", self.context) @@ -844,11 +851,8 @@ class VideoView(View): def get(self, request, video_id): """get single video""" config_handler = AppConfig(request.user.id) - position = time_parser(request.GET.get("t")) - path = f"ta_video/_doc/{video_id}" - look_up = SearchHandler(path, config=False) - video_hit = look_up.get_data() - video_data = video_hit[0]["source"] + look_up = SearchHandler(f"ta_video/_doc/{video_id}", config=False) + video_data = look_up.get_data()[0]["source"] try: rating = video_data["stats"]["average_rating"] video_data["stats"]["average_rating"] = self.star_creator(rating) @@ -861,16 +865,20 @@ class VideoView(View): else: playlist_nav = False - video_title = video_data["title"] + reindex = ReindexProgress( + request_type="video", request_id=video_id + ).get_progress() + context = { "video": video_data, "playlist_nav": playlist_nav, - "title": video_title, + "title": video_data.get("title"), "colors": config_handler.colors, "cast": config_handler.config["application"]["enable_cast"], "version": settings.TA_VERSION, "config": config_handler.config, - "position": position, + "position": time_parser(request.GET.get("t")), + "reindex": reindex.get("state"), } return render(request, "home/video.html", context) diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 1e4763bf..4e747951 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -119,6 +119,10 @@ button:hover { color: var(--main-bg); } +.button-box { + padding: 5px 0; +} + .unsubscribe { background-color: var(--accent-font-light); } diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index aec85661..09a8580c 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -147,6 +147,24 @@ function toggleCheckbox(checkbox) { }, 500); } +// start reindex task +function reindex(button) { + let apiEndpoint = '/api/refresh/'; + if (button.getAttribute('data-extract-videos')) { + apiEndpoint += '?extract_videos=true'; + } + let type = button.getAttribute('data-type'); + let id = button.getAttribute('data-id'); + + let data = {}; + data[type] = [id]; + + apiRequest(apiEndpoint, 'POST', data); + let message = document.createElement('p'); + message.innerText = 'Reindex scheduled'; + document.getElementById('reindex-button').replaceWith(message); +} + // download page buttons function rescanPending() { let payload = JSON.stringify({ rescan_pending: true }); From 3b4969dcea0ca49692c28b82f76e715b5a0d6d0a Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 19 Dec 2022 14:12:48 +0700 Subject: [PATCH 14/17] distinct between none and false comments --- tubearchivist/home/templates/home/video.html | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index 904f8a34..a500a9c7 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -149,13 +149,17 @@ - {% if video.comment_count %} -
-

Comments: {{video.comment_count}}

-
-
+ {% if video.comment_count == 0 %} +
+ Video has no comments +
+ {% elif video.comment_count %} +
+

Comments: {{video.comment_count}}

+
+
{% endif %}