From c8dd391fd3d6ea4c13d09bdd155a01a39b307ae9 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 09:09:32 +0700 Subject: [PATCH 01/11] fix channel tv art validator callback --- tubearchivist/home/src/download/thumbnails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index d7a61e22..4d6dbcc0 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -270,6 +270,7 @@ class ValidatorCallback: urls = ( channel["_source"]["channel_thumb_url"], channel["_source"]["channel_banner_url"], + channel["_source"]["channel_tvart_url"], ) handler = ThumbManager(channel["_source"]["channel_id"]) handler.download_channel_art(urls, skip_existing=True) From 5b2d7db102c05e35c1bc84b81b5d6c8c00b6fca0 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 12:47:33 +0700 Subject: [PATCH 02/11] fix chown for channel folder creation --- tubearchivist/home/src/download/yt_dlp_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 5db44bfd..fe31c03f 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -395,8 +395,8 @@ class VideoDownloader: new_folder = os.path.join(videos, channel_name) if not os.path.exists(new_folder): os.makedirs(new_folder) - if host_uid and host_gid: - os.chown(new_folder, host_uid, host_gid) + if host_uid and host_gid: + os.chown(new_folder, host_uid, host_gid) # find real filename cache_dir = self.config["application"]["cache_dir"] all_cached = ignore_filelist(os.listdir(cache_dir + "/download/")) From 4038a837cfc040a83d6438ff190d23bf3e2f796e Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 13:11:40 +0700 Subject: [PATCH 03/11] fix chown for subtitles --- tubearchivist/home/src/index/subtitle.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index 7c56c4d0..e289fbe8 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -115,7 +115,7 @@ class YoutubeSubtitle: source = subtitle["source"] lang = subtitle.get("lang") response = requests.get( - subtitle["url"], headers=requests_headers() + subtitle["url"], headers=requests_headers(), timeout=30 ) if not response.ok: print(f"{self.video.youtube_id}: failed to download subtitle") @@ -137,14 +137,18 @@ class YoutubeSubtitle: return indexed - @staticmethod - def _write_subtitle_file(dest_path, subtitle_str): + def _write_subtitle_file(self, dest_path, subtitle_str): """write subtitle file to disk""" # create folder here for first video of channel os.makedirs(os.path.split(dest_path)[0], exist_ok=True) with open(dest_path, "w", encoding="utf-8") as subfile: subfile.write(subtitle_str) + host_uid = self.video.config["application"]["HOST_UID"] + host_gid = self.video.config["application"]["HOST_GID"] + if host_uid and host_gid: + os.chown(dest_path, host_uid, host_gid) + @staticmethod def _index_subtitle(query_str): """send subtitle to es for indexing""" From 841f5ab67f4eabaa0c8c5a7fd299ad436945182c Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 19:14:25 +0700 Subject: [PATCH 04/11] fix missing tvart fallback key --- tubearchivist/home/src/download/thumbnails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 4d6dbcc0..0431842a 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -81,7 +81,7 @@ class ThumbManagerBase: "banner": os.path.join( app_root, "static/img/default-channel-banner.jpg" ), - "art": os.path.join( + "tvart": os.path.join( app_root, "static/img/default-channel-art.jpg" ), } From 3fa5ed725ee36a5ba272c130d2b3fb4e3559fcac Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 19:29:21 +0700 Subject: [PATCH 05/11] fix missing progress script for channel notification --- tubearchivist/home/templates/home/channel_id_about.html | 1 + tubearchivist/home/templates/home/channel_id_playlist.html | 1 + 2 files changed, 2 insertions(+) diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index ac6a66c7..1a9b782f 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -131,4 +131,5 @@ + {% endblock content %} \ No newline at end of file diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 609883ed..cf07f003 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -66,4 +66,5 @@ {% endif %} + {% endblock content %} \ No newline at end of file From e43840da318919fffd8877b01d8a25760b1af641 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 19:41:53 +0700 Subject: [PATCH 06/11] fix manual index folder permission --- tubearchivist/home/src/index/manual.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index a4fa2e40..89969f81 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -427,12 +427,17 @@ class ManualImport: def _move_to_archive(self, json_data): """move identified media file to archive""" videos = self.config["application"]["videos"] + host_uid = self.config["application"]["HOST_UID"] + host_gid = self.config["application"]["HOST_GID"] channel, file = os.path.split(json_data["media_url"]) channel_folder = os.path.join(videos, channel) if not os.path.exists(channel_folder): os.makedirs(channel_folder) + if host_uid and host_gid: + os.chown(channel_folder, host_uid, host_gid) + old_path = self.current_video["media"] new_path = os.path.join(channel_folder, file) shutil.move(old_path, new_path, copy_function=shutil.copyfile) From fb77ab44eea554ac837a080a2f81097130f812cc Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 7 Apr 2023 19:51:40 +0700 Subject: [PATCH 07/11] fix manual import media file permission, add missing comment check trigger --- tubearchivist/home/src/index/manual.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index 89969f81..66b6bca2 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -12,6 +12,7 @@ import shutil import subprocess from home.src.download.thumbnails import ThumbManager +from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist @@ -141,6 +142,9 @@ class ImportFolderScanner: ManualImport(current_video, self.CONFIG).run() + video_ids = [i["video_id"] for i in self.to_import] + CommentList(video_ids, task=self.task).index() + def _notify(self, idx, current_video): """send notification back to task""" filename = os.path.split(current_video["media"])[-1] @@ -441,6 +445,8 @@ class ManualImport: old_path = self.current_video["media"] new_path = os.path.join(channel_folder, file) shutil.move(old_path, new_path, copy_function=shutil.copyfile) + if host_uid and host_gid: + os.chown(new_path, host_uid, host_gid) base_name, _ = os.path.splitext(new_path) for old_path in self.current_video["subtitle"]: From ff89432d86e28ca878163ee068529d851b434fb8 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 8 Apr 2023 14:28:03 +0700 Subject: [PATCH 08/11] fix off by one task notification error --- tubearchivist/home/src/es/connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index fd6e5ab0..5af1838d 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -166,7 +166,7 @@ class IndexPaginate: def _notify(self, processed): """send notification on task""" total = self.kwargs.get("total") - progress = (processed + 1) / total + progress = processed / total index_clean = self.index_name.lstrip("ta_").title() message = [f"Processing {index_clean}s {processed}/{total}"] self.kwargs.get("task").send_progress(message, progress=progress) From d2649c29c3bd321dcc7e3bfea2195f2a5c1daa5e Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 8 Apr 2023 16:00:16 +0700 Subject: [PATCH 09/11] add recent videos to reindex queue task --- tubearchivist/home/src/index/reindex.py | 26 +++++++++++++++++++++++-- tubearchivist/home/tasks.py | 7 +++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index baeadfa8..b5ae83cd 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -49,6 +49,7 @@ class ReindexBase: } MULTIPLY = 1.2 + DAYS3 = 60 * 60 * 24 * 3 def __init__(self): self.config = AppConfig().config @@ -62,13 +63,34 @@ class ReindexBase: RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids) -class ReindexOutdated(ReindexBase): - """add outdated documents to reindex queue""" +class ReindexPopulate(ReindexBase): + """add outdated and recent documents to reindex queue""" def __init__(self): super().__init__() self.interval = self.config["scheduler"]["check_reindex_days"] + def add_recent(self): + """add recent videos to refresh""" + gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat() + must_list = [ + {"term": {"active": {"value": True}}}, + {"range": {"published": {"gte": gte}}}, + ] + data = { + "size": 10000, + "query": {"bool": {"must": must_list}}, + "sort": [{"published": {"order": "desc"}}], + } + response, _ = ElasticWrap("ta_video/_search").get(data=data) + hits = response["hits"]["hits"] + if not hits: + return + + all_ids = [i["_source"]["youtube_id"] for i in hits] + reindex_config = self.REINDEX_CONFIG.get("video") + self.populate(all_ids, reindex_config) + def add_outdated(self): """add outdated documents""" for reindex_config in self.REINDEX_CONFIG.values(): diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index f2c5341e..a86c1dc1 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap from home.src.index.channel import YoutubeChannel from home.src.index.filesystem import Filesystem from home.src.index.manual import ImportFolderScanner -from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated +from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager @@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False): manager.init(self) if not data: # started from scheduler + populate = ReindexPopulate() print(f"[task][{self.name}] reindex outdated documents") + self.send_progress("Add recent documents to the reindex Queue.") + populate.add_recent() self.send_progress("Add outdated documents to the reindex Queue.") - ReindexOutdated().add_outdated() + populate.add_outdated() Reindex(task=self).reindex_all() From 1e96c3211f17ab20bb729a1e0c0952518956557a Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 8 Apr 2023 16:39:08 +0700 Subject: [PATCH 10/11] remove unused functions --- tubearchivist/static/script.js | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index b902be54..c79f918e 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -55,19 +55,6 @@ function createWatchStatusIndicator(videoId, videoWatchStatus) { return watchStatusIndicator; } -// function isWatched(youtube_id) { -// var payload = JSON.stringify({'watched': youtube_id}); -// sendPost(payload); -// var seenIcon = document.createElement('img'); -// seenIcon.setAttribute('src', "/static/img/icon-seen.svg"); -// seenIcon.setAttribute('alt', 'seen-icon'); -// seenIcon.setAttribute('id', youtube_id); -// seenIcon.setAttribute('title', "Mark as unwatched"); -// seenIcon.setAttribute('onclick', "isUnwatched(this.id)"); -// seenIcon.classList = 'seen-icon'; -// document.getElementById(youtube_id).replaceWith(seenIcon); -// } - // Removes the progress bar when passed a video id function removeProgressBar(videoId) { setProgressBar(videoId, 0, 1); @@ -84,20 +71,6 @@ function isWatchedButton(button) { }, 1000); } -// function isUnwatched(youtube_id) { -// postVideoProgress(youtube_id, 0); // Reset video progress on unwatched; -// var payload = JSON.stringify({'un_watched': youtube_id}); -// sendPost(payload); -// var unseenIcon = document.createElement('img'); -// unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg"); -// unseenIcon.setAttribute('alt', 'unseen-icon'); -// unseenIcon.setAttribute('id', youtube_id); -// unseenIcon.setAttribute('title', "Mark as watched"); -// unseenIcon.setAttribute('onclick', "isWatched(this.id)"); -// unseenIcon.classList = 'unseen-icon'; -// document.getElementById(youtube_id).replaceWith(unseenIcon); -// } - function unsubscribe(id_unsub) { let payload = JSON.stringify({ unsubscribe: id_unsub }); sendPost(payload); From 5d524e51106cd98ea78f8a2102a195f9a978c555 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 8 Apr 2023 17:18:42 +0700 Subject: [PATCH 11/11] fix reindex notification filter --- tubearchivist/home/templates/home/channel_id.html | 2 +- tubearchivist/home/templates/home/channel_id_about.html | 2 +- tubearchivist/home/templates/home/channel_id_playlist.html | 2 +- tubearchivist/home/templates/home/settings.html | 2 +- tubearchivist/static/progress.js | 2 +- tubearchivist/static/script.js | 3 +++ 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 9539f58d..fc85d0f0 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -22,7 +22,7 @@

Downloads

 {% endif %} -
+
diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index 1a9b782f..dc38b7ac 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -22,7 +22,7 @@

Downloads

 {% endif %}
-
+
diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index cf07f003..971e41ed 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -22,7 +22,7 @@

Downloads

 {% endif %}
-
+
Show subscribed only: diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index e239e1bd..15191364 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -2,7 +2,7 @@ {% load static %} {% block content %}
-
+

User Configurations

diff --git a/tubearchivist/static/progress.js b/tubearchivist/static/progress.js index 36e7ede8..a2b35daf 100644 --- a/tubearchivist/static/progress.js +++ b/tubearchivist/static/progress.js @@ -31,7 +31,7 @@ function getMessages(dataOrigin) { function buildMessage(responseData, dataOrigin) { // filter relevant messages let messages = responseData.filter(function (value) { - return value.group.startsWith(dataOrigin); + return dataOrigin.split(' ').includes(value.group.split(':')[0]); }, dataOrigin); let notifications = document.getElementById('notifications'); let currentNotifications = notifications.childElementCount; diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index c79f918e..5df3f08b 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -136,6 +136,9 @@ function reindex(button) { let message = document.createElement('p'); message.innerText = 'Reindex scheduled'; document.getElementById('reindex-button').replaceWith(message); + setTimeout(function () { + checkMessages(); + }, 500); } // download page buttons