diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 01b18c08..6ae35f4f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -2,6 +2,10 @@ from api.src.search_processor import SearchProcess from home.src.download.queue import PendingInteract +from home.src.download.subscriptions import ( + ChannelSubscription, + PlaylistSubscription, +) from home.src.download.yt_dlp_base import CookieHandler from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot @@ -9,6 +13,7 @@ from home.src.frontend.searching import SearchForm from home.src.frontend.watched import WatchState from home.src.index.channel import YoutubeChannel from home.src.index.generic import Pagination +from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig, ReleaseVersion @@ -318,9 +323,8 @@ class ChannelApiListView(ApiBaseView): return Response(self.response) - @staticmethod - def post(request): - """subscribe to list of channels""" + def post(self, request): + """subscribe/unsubscribe to list of channels""" data = request.data try: to_add = data["data"] @@ -329,12 +333,28 @@ class ChannelApiListView(ApiBaseView): print(message) return Response({"message": message}, status=400) - pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]] - url_str = " ".join(pending) - subscribe_to.delay(url_str) + pending = [] + for channel_item in to_add: + channel_id = channel_item["channel_id"] + if channel_item["channel_subscribed"]: + pending.append(channel_id) + else: + self._unsubscribe(channel_id) + + if pending: + url_str = " ".join(pending) + subscribe_to.delay(url_str, expected_type="channel") return Response(data) + @staticmethod + def _unsubscribe(channel_id: str): + """unsubscribe""" + print(f"[{channel_id}] unsubscribe from channel") + ChannelSubscription().change_subscribe( + channel_id, channel_subscribed=False + ) + class ChannelApiVideoView(ApiBaseView): """resolves to /api/channel//video @@ -373,6 +393,38 @@ class PlaylistApiListView(ApiBaseView): self.get_document_list(request) return Response(self.response) + def post(self, request): + """subscribe/unsubscribe to list of playlists""" + data = request.data + try: + to_add = data["data"] + except KeyError: + message = "missing expected data key" + print(message) + return Response({"message": message}, status=400) + + pending = [] + for playlist_item in to_add: + playlist_id = playlist_item["playlist_id"] + if playlist_item["playlist_subscribed"]: + pending.append(playlist_id) + else: + self._unsubscribe(playlist_id) + + if pending: + url_str = " ".join(pending) + subscribe_to.delay(url_str, expected_type="playlist") + + return Response(data) + + @staticmethod + def _unsubscribe(playlist_id: str): + """unsubscribe""" + print(f"[{playlist_id}] unsubscribe from playlist") + PlaylistSubscription().change_subscribe( + playlist_id, subscribe_status=False + ) + class PlaylistApiView(ApiBaseView): """resolves to /api/playlist// @@ -387,6 +439,17 @@ class PlaylistApiView(ApiBaseView): self.get_document(playlist_id) return Response(self.response, status=self.status_code) + def delete(self, request, playlist_id): + """delete playlist""" + print(f"{playlist_id}: delete playlist") + delete_videos = request.GET.get("delete-videos", False) + if delete_videos: + YoutubePlaylist(playlist_id).delete_videos_playlist() + else: + YoutubePlaylist(playlist_id).delete_metadata() + + return Response({"success": True}) + class PlaylistApiVideoView(ApiBaseView): """resolves to /api/playlist//video diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 6325cb4f..aef62d6b 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -332,7 +332,7 @@ class SubscriptionHandler: self.task = task self.to_subscribe = False - def subscribe(self): + def subscribe(self, expected_type=False): """subscribe to url_str items""" if self.task: self.task.send_progress(["Processing form content."]) @@ -343,11 +343,16 @@ class SubscriptionHandler: if self.task: self._notify(idx, item, total) - self.subscribe_type(item) + self.subscribe_type(item, expected_type=expected_type) - def subscribe_type(self, item): + def subscribe_type(self, item, expected_type): """process single item""" if item["type"] == "playlist": + if expected_type and expected_type != "playlist": + raise TypeError( + f"expected {expected_type} url but got {item.get('type')}" + ) + PlaylistSubscription().process_url_str([item]) return @@ -360,6 +365,11 @@ class SubscriptionHandler: else: raise ValueError("failed to subscribe to: " + item["url"]) + if expected_type and expected_type != "channel": + raise TypeError( + f"expected {expected_type} url but got {item.get('type')}" + ) + self._subscribe(channel_id) def _subscribe(self, channel_id): diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 95461e8c..7041ec2f 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -61,7 +61,7 @@ class ThumbManagerBase: print(f"{self.item_id}: retry thumbnail download {url}") sleep((i + 1) ** i) - return False + return self.get_fallback() def get_fallback(self): """get fallback thumbnail if not available""" diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 526dbd40..11478ddc 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -112,9 +112,9 @@ class CookieHandler: def set_cookie(self, cookie): """set cookie str and activate in cofig""" - RedisArchivist().set_message("cookie", cookie) + RedisArchivist().set_message("cookie", cookie, save=True) path = ".downloads.cookie_import" - RedisArchivist().set_message("config", True, path=path) + RedisArchivist().set_message("config", True, path=path, save=True) self.config["downloads"]["cookie_import"] = True print("cookie: activated and stored in Redis") diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index f2d6231c..60764ea2 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -4,14 +4,8 @@ Functionality: - called via user input """ -from home.src.download.subscriptions import ( - ChannelSubscription, - PlaylistSubscription, -) -from home.src.index.playlist import YoutubePlaylist from home.src.ta.ta_redis import RedisArchivist -from home.src.ta.urlparser import Parser -from home.tasks import run_restore_backup, subscribe_to +from home.tasks import run_restore_backup class PostData: @@ -36,14 +30,11 @@ class PostData: exec_map = { "change_view": self._change_view, "change_grid": self._change_grid, - "unsubscribe": self._unsubscribe, - "subscribe": self._subscribe, "sort_order": self._sort_order, "hide_watched": self._hide_watched, "show_subed_only": self._show_subed_only, "show_ignored_only": self._show_ignored_only, "db-restore": self._db_restore, - "delete-playlist": self._delete_playlist, } return exec_map[self.to_exec] @@ -67,34 +58,6 @@ class PostData: RedisArchivist().set_message(key, {"status": grid_items}) return {"success": True} - def _unsubscribe(self): - """unsubscribe from channels or playlists""" - id_unsub = self.exec_val - print(f"{id_unsub}: unsubscribe") - to_unsub_list = Parser(id_unsub).parse() - for to_unsub in to_unsub_list: - unsub_type = to_unsub["type"] - unsub_id = to_unsub["url"] - if unsub_type == "playlist": - PlaylistSubscription().change_subscribe( - unsub_id, subscribe_status=False - ) - elif unsub_type == "channel": - ChannelSubscription().change_subscribe( - unsub_id, channel_subscribed=False - ) - else: - raise ValueError("failed to process " + id_unsub) - - return {"success": True} - - def _subscribe(self): - """subscribe to channel or playlist, called from js buttons""" - id_sub = self.exec_val - print(f"{id_sub}: subscribe") - subscribe_to.delay(id_sub) - return {"success": True} - def _sort_order(self): """change the sort between published to downloaded""" sort_order = {"status": self.exec_val} @@ -139,16 +102,3 @@ class PostData: filename = self.exec_val run_restore_backup.delay(filename) return {"success": True} - - def _delete_playlist(self): - """delete playlist, only metadata or incl all videos""" - playlist_dict = self.exec_val - playlist_id = playlist_dict["playlist-id"] - playlist_action = playlist_dict["playlist-action"] - print(f"{playlist_id}: delete playlist {playlist_action}") - if playlist_action == "metadata": - YoutubePlaylist(playlist_id).delete_metadata() - elif playlist_action == "all": - YoutubePlaylist(playlist_id).delete_videos_playlist() - - return {"success": True} diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index d1cccced..4946bee0 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -100,7 +100,7 @@ class AppConfig: self.config[config_dict][config_value] = to_write updated.append((config_value, to_write)) - RedisArchivist().set_message("config", self.config) + RedisArchivist().set_message("config", self.config, save=True) return updated @staticmethod @@ -112,7 +112,7 @@ class AppConfig: message = {"status": value} redis_key = f"{user_id}:{key}" - RedisArchivist().set_message(redis_key, message) + RedisArchivist().set_message(redis_key, message, save=True) def get_colors(self): """overwrite config if user has set custom values""" @@ -225,7 +225,7 @@ class ScheduleBuilder: to_write = value redis_config["scheduler"][key] = to_write - RedisArchivist().set_message("config", redis_config) + RedisArchivist().set_message("config", redis_config, save=True) mess_dict = { "status": self.MSG, "level": "info", diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index eb924b3a..0028c11b 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -15,7 +15,12 @@ import requests def ignore_filelist(filelist: list[str]) -> list[str]: """ignore temp files for os.listdir sanitizer""" - to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"] + to_ignore = [ + "@eaDir", + "Icon\r\r", + "Network Trash Folder", + "Temporary Items", + ] cleaned: list[str] = [] for file_name in filelist: if file_name.startswith(".") or file_name in to_ignore: @@ -110,7 +115,7 @@ def clear_dl_cache(config: dict) -> int: """clear leftover files from dl cache""" print("clear download cache") cache_dir = os.path.join(config["application"]["cache_dir"], "download") - leftover_files = os.listdir(cache_dir) + leftover_files = ignore_filelist(os.listdir(cache_dir)) for cached in leftover_files: to_delete = os.path.join(cache_dir, cached) os.remove(to_delete) diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 46b1b5c0..77de5287 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -41,6 +41,7 @@ class RedisArchivist(RedisBase): message: dict, path: str = ".", expire: bool | int = False, + save: bool = False, ) -> None: """write new message to redis""" self.conn.execute_command( @@ -54,6 +55,16 @@ class RedisArchivist(RedisBase): secs = expire self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs) + if save: + self.bg_save() + + def bg_save(self) -> None: + """save to aof""" + try: + self.conn.bgsave() + except redis.exceptions.ResponseError: + pass + def get_message(self, key: str) -> dict: """get message dict from redis""" reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 5ce1d627..a2c4cdbf 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -343,9 +343,12 @@ def re_sync_thumbs(self): @shared_task(bind=True, name="subscribe_to", base=BaseTask) -def subscribe_to(self, url_str): - """take a list of urls to subscribe to""" - SubscriptionHandler(url_str, task=self).subscribe() +def subscribe_to(self, url_str: str, expected_type: str | bool = False): + """ + take a list of urls to subscribe to + optionally validate expected_type channel / playlist + """ + SubscriptionHandler(url_str, task=self).subscribe(expected_type) @shared_task(bind=True, name="index_playlists", base=BaseTask) diff --git a/tubearchivist/home/templates/home/channel.html b/tubearchivist/home/templates/home/channel.html index c2f1e0bc..7a7debf9 100644 --- a/tubearchivist/home/templates/home/channel.html +++ b/tubearchivist/home/templates/home/channel.html @@ -66,9 +66,9 @@

Last refreshed: {{ channel.source.channel_last_refresh }}

{% if channel.source.channel_subscribed %} - + {% else %} - + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 89487723..7c09888d 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -38,9 +38,9 @@

Subscribers: {{ channel_info.channel_subs|intcomma }}

{% endif %} {% if channel_info.channel_subscribed %} - + {% else %} - + {% endif %} diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 971e41ed..c40a1214 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -54,9 +54,9 @@

{{ playlist.source.playlist_name }}

Last refreshed: {{ playlist.source.playlist_last_refresh }}

{% if playlist.source.playlist_subscribed %} - + {% else %} - + {% endif %} diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index 2d786bf7..c82af747 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -49,9 +49,9 @@

{{ playlist.source.playlist_name }}

Last refreshed: {{ playlist.source.playlist_last_refresh }}

{% if playlist.source.playlist_subscribed %} - + {% else %} - + {% endif %} diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 92a450dd..d5d20dda 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -27,9 +27,9 @@

Last refreshed: {{ playlist_info.playlist_last_refresh }}

Playlist: {% if playlist_info.playlist_subscribed %} - + {% else %} - + {% endif %}

{% if playlist_info.playlist_active %} @@ -40,8 +40,8 @@
Delete {{ playlist_info.playlist_name }}? - -
+ +
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index ef880a74..507c9336 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -736,7 +736,7 @@ class ChannelView(ArchivistResultsView): if subscribe_form.is_valid(): url_str = request.POST.get("subscribe") print(url_str) - subscribe_to.delay(url_str) + subscribe_to.delay(url_str, expected_type="channel") sleep(1) return redirect("channel", permanent=True) @@ -879,7 +879,7 @@ class PlaylistView(ArchivistResultsView): if subscribe_form.is_valid(): url_str = request.POST.get("subscribe") print(url_str) - subscribe_to.delay(url_str) + subscribe_to.delay(url_str, expected_type="playlist") sleep(1) return redirect("playlist") diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 9933c48f..50f5f15b 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,13 +1,13 @@ apprise==1.4.5 celery==5.3.1 -Django==4.2.3 -django-auth-ldap==4.4.0 +Django==4.2.4 +django-auth-ldap==4.5.0 django-cors-headers==4.2.0 djangorestframework==3.14.0 Pillow==10.0.0 -redis==4.6.0 +redis==5.0.0 requests==2.31.0 ryd-client==0.0.6 -uWSGI==2.0.21 +uWSGI==2.0.22 whitenoise==6.5.0 yt_dlp==2023.7.6 diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 528b1a72..7d2d3cf8 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -71,20 +71,27 @@ function isWatchedButton(button) { }, 1000); } -function unsubscribe(id_unsub) { - let payload = JSON.stringify({ unsubscribe: id_unsub }); - sendPost(payload); +function subscribeStatus(subscribeButton) { + let id = subscribeButton.getAttribute('data-id'); + let type = subscribeButton.getAttribute('data-type'); + let subscribe = Boolean(subscribeButton.getAttribute('data-subscribe')); + let apiEndpoint; + let data; + if (type === 'channel') { + apiEndpoint = '/api/channel/'; + data = { data: [{ channel_id: id, channel_subscribed: subscribe }] }; + } else if (type === 'playlist') { + apiEndpoint = '/api/playlist/'; + data = { data: [{ playlist_id: id, playlist_subscribed: subscribe }] }; + } + apiRequest(apiEndpoint, 'POST', data); let message = document.createElement('span'); - message.innerText = 'You are unsubscribed.'; - document.getElementById(id_unsub).replaceWith(message); -} - -function subscribe(id_sub) { - let payload = JSON.stringify({ subscribe: id_sub }); - sendPost(payload); - let message = document.createElement('span'); - message.innerText = 'You are subscribed.'; - document.getElementById(id_sub).replaceWith(message); + if (subscribe) { + message.innerText = 'You are subscribed.'; + } else { + message.innerText = 'You are unsubscribed.'; + } + subscribeButton.replaceWith(message); } function changeView(image) { @@ -374,13 +381,11 @@ function deleteChannel(button) { function deletePlaylist(button) { let playlist_id = button.getAttribute('data-id'); let playlist_action = button.getAttribute('data-action'); - let payload = JSON.stringify({ - 'delete-playlist': { - 'playlist-id': playlist_id, - 'playlist-action': playlist_action, - }, - }); - sendPost(payload); + let apiEndpoint = `/api/playlist/${playlist_id}/`; + if (playlist_action === 'delete-videos') { + apiEndpoint += '?delete-videos=true'; + } + apiRequest(apiEndpoint, 'DELETE'); setTimeout(function () { window.location.replace('/playlist/'); }, 1000); @@ -1057,9 +1062,9 @@ function createChannel(channel, viewStyle) { const channelLastRefresh = channel.channel_last_refresh; let button; if (channel.channel_subscribed) { - button = ``; + button = ``; } else { - button = ``; + button = ``; } // build markup const markup = ` @@ -1103,9 +1108,9 @@ function createPlaylist(playlist, viewStyle) { const playlistLastRefresh = playlist.playlist_last_refresh; let button; if (playlist.playlist_subscribed) { - button = ``; + button = ``; } else { - button = ``; + button = ``; } const markup = `