diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index d938b2f3..4ad216ef 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -86,7 +86,8 @@ class VideoApiView(ApiBaseView): # pylint: disable=unused-argument """get request""" self.get_document(video_id) - self.process_keys() + if self.response.get("data"): + self.process_keys() return Response(self.response, status=self.status_code) diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py index 46a940cc..1053bc6e 100644 --- a/tubearchivist/home/apps.py +++ b/tubearchivist/home/apps.py @@ -97,9 +97,8 @@ class StartupCheck: if invalid: print( - "minial required elasticsearch version: " - + f"{self.MIN_MAJOR}.{self.MIN_MINOR}, " - + "please update to recommended version." + "required elasticsearch version: " + + f"{self.MIN_MAJOR}.{self.MIN_MINOR}" ) sys.exit(1) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 0ce492f3..d25f4d19 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -4,8 +4,10 @@ functionality: - check for missing thumbnails """ +import base64 import os from collections import Counter +from io import BytesIO from time import sleep import requests @@ -15,7 +17,7 @@ from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist from home.src.ta.ta_redis import RedisArchivist from mutagen.mp4 import MP4, MP4Cover -from PIL import Image +from PIL import Image, ImageFilter class ThumbManager: @@ -241,6 +243,21 @@ class ThumbManager: } RedisArchivist().set_message("message:download", mess_dict) + def get_base64_blur(self, youtube_id): + """return base64 encoded placeholder""" + img_path = self.vid_thumb_path(youtube_id) + file_path = os.path.join(self.CACHE_DIR, img_path) + img_raw = Image.open(file_path) + img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20)) + img_blur = img_raw.filter(ImageFilter.BLUR) + buffer = BytesIO() + img_blur.save(buffer, format="JPEG") + img_data = buffer.getvalue() + img_base64 = base64.b64encode(img_data).decode() + data_url = f"data:image/jpg;base64,{img_base64}" + + return data_url + @staticmethod def vid_thumb_path(youtube_id): """build expected path for video thumbnail from youtube_id""" diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index d764011d..693d0e1e 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -177,7 +177,9 @@ class VideoDownloader: except yt_dlp.utils.DownloadError: print("failed to download " + youtube_id) continue - vid_dict = index_new_video(youtube_id) + vid_dict = index_new_video( + youtube_id, video_overwrites=self.video_overwrites + ) self.channels.add(vid_dict["channel"]["channel_id"]) self.move_to_archive(vid_dict) self._delete_from_pending(youtube_id) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 29f6b7e0..9a0ee47e 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -50,6 +50,9 @@ }, "index_playlists": { "type": "boolean" + }, + "integrate_sponsorblock": { + "type" : "boolean" } } } @@ -73,6 +76,10 @@ "type": "text", "index": false }, + "vid_thumb_base64": { + "type": "text", + "index": false + }, "date_downloaded": { "type": "date" }, @@ -126,6 +133,9 @@ }, "index_playlists": { "type": "boolean" + }, + "integrate_sponsorblock": { + "type" : "boolean" } } } diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 53ef284a..337a8054 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -198,8 +198,17 @@ class ChannelOverwriteForm(forms.Form): ("1", "Enable playlist index"), ] + SP_CHOICES = [ + ("", "-- change sponsorblock integrations"), + ("0", "disable sponsorblock integration"), + ("1", "enable sponsorblock integration"), + ] + download_format = forms.CharField(label=False, required=False) autodelete_days = forms.IntegerField(label=False, required=False) index_playlists = forms.ChoiceField( widget=forms.Select, choices=PLAYLIST_INDEX, required=False ) + integrate_sponsorblock = forms.ChoiceField( + widget=forms.Select, choices=SP_CHOICES, required=False + ) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 953078df..75824d81 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -340,7 +340,12 @@ class YoutubeChannel(YouTubeItem): def set_overwrites(self, overwrites): """set per channel overwrites""" - valid_keys = ["download_format", "autodelete_days", "index_playlists"] + valid_keys = [ + "download_format", + "autodelete_days", + "index_playlists", + "integrate_sponsorblock", + ] to_write = self.json_data.get("channel_overwrites", {}) for key, value in overwrites.items(): diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 411b4af2..28ad5365 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -10,6 +10,7 @@ from datetime import datetime import requests from django.conf import settings +from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem @@ -357,9 +358,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): index_name = "ta_video" yt_base = "https://www.youtube.com/watch?v=" - def __init__(self, youtube_id): + def __init__(self, youtube_id, video_overwrites=False): super().__init__(youtube_id) self.channel_id = False + self.video_overwrites = video_overwrites self.es_path = f"{self.index_name}/_doc/{youtube_id}" def build_json(self): @@ -376,11 +378,24 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if self.config["downloads"]["integrate_ryd"]: self._get_ryd_stats() - if self.config["downloads"]["integrate_sponsorblock"]: + if self._check_get_sb(): self._get_sponsorblock() return + def _check_get_sb(self): + """check if need to run sponsor block""" + if self.config["downloads"]["integrate_sponsorblock"]: + return True + try: + single_overwrite = self.video_overwrites[self.youtube_id] + _ = single_overwrite["integrate_sponsorblock"] + return True + except KeyError: + return False + + return False + def _process_youtube_meta(self): """extract relevant fields from youtube""" # extract @@ -389,12 +404,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): upload_date_time = datetime.strptime(upload_date, "%Y%m%d") published = upload_date_time.strftime("%Y-%m-%d") last_refresh = int(datetime.now().strftime("%s")) + base64_blur = ThumbManager().get_base64_blur(self.youtube_id) # build json_data basics self.json_data = { "title": self.youtube_meta["title"], "description": self.youtube_meta["description"], "category": self.youtube_meta["categories"], "vid_thumb_url": self.youtube_meta["thumbnail"], + "vid_thumb_base64": base64_blur, "tags": self.youtube_meta["tags"], "published": published, "vid_last_refresh": last_refresh, @@ -495,7 +512,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): for media_url in to_del: file_path = os.path.join(video_base, media_url) - os.remove(file_path) + try: + os.remove(file_path) + except FileNotFoundError: + print(f"{self.youtube_id}: failed {media_url}, continue.") self.del_in_es() self.delete_subtitles() @@ -541,9 +561,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): _, _ = ElasticWrap(path).post(data=data) -def index_new_video(youtube_id): +def index_new_video(youtube_id, video_overwrites=False): """combined classes to create new video in index""" - video = YoutubeVideo(youtube_id) + video = YoutubeVideo(youtube_id, video_overwrites=video_overwrites) video.build_json() if not video.json_data: raise ValueError("failed to get metadata for " + youtube_id) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 3258ed0b..4b98c4ab 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -83,33 +83,32 @@ class AppConfig: def update_config(self, form_post): """update config values from settings form""" - config = self.config for key, value in form_post.items(): - to_write = value[0] - if len(to_write): - if to_write == "0": - to_write = False - elif to_write == "1": - to_write = True - elif to_write.isdigit(): - to_write = int(to_write) + if not value and not isinstance(value, int): + continue + + if value in ["0", 0]: + to_write = False + elif value == "1": + to_write = True + else: + to_write = value - config_dict, config_value = key.split("_", maxsplit=1) - config[config_dict][config_value] = to_write + config_dict, config_value = key.split("_", maxsplit=1) + self.config[config_dict][config_value] = to_write - RedisArchivist().set_message("config", config, expire=False) + RedisArchivist().set_message("config", self.config, expire=False) @staticmethod def set_user_config(form_post, user_id): """set values in redis for user settings""" for key, value in form_post.items(): - to_write = value[0] - if len(to_write): - if to_write.isdigit(): - to_write = int(to_write) - message = {"status": to_write} - redis_key = f"{user_id}:{key}" - RedisArchivist().set_message(redis_key, message, expire=False) + if not value: + continue + + message = {"status": value} + redis_key = f"{user_id}:{key}" + RedisArchivist().set_message(redis_key, message, expire=False) def get_colors(self): """overwrite config if user has set custom values""" @@ -172,12 +171,11 @@ class ScheduleBuilder: print("processing form, restart container for changes to take effect") redis_config = self.config for key, value in form_post.items(): - to_check = value[0] - if key in self.SCHEDULES and to_check: + if key in self.SCHEDULES and value: try: - to_write = self.value_builder(key, to_check) + to_write = self.value_builder(key, value) except ValueError: - print(f"failed: {key} {to_check}") + print(f"failed: {key} {value}") mess_dict = { "status": "message:setting", "level": "error", @@ -188,8 +186,8 @@ class ScheduleBuilder: return redis_config["scheduler"][key] = to_write - if key in self.CONFIG and to_check: - redis_config["scheduler"][key] = int(to_check) + if key in self.CONFIG and value: + redis_config["scheduler"][key] = int(value) RedisArchivist().set_message("config", redis_config, expire=False) mess_dict = { "status": "message:setting", @@ -199,37 +197,56 @@ class ScheduleBuilder: } RedisArchivist().set_message("message:setting", mess_dict) - def value_builder(self, key, to_check): + def value_builder(self, key, value): """validate single cron form entry and return cron dict""" - print(f"change schedule for {key} to {to_check}") - if to_check == "0": + print(f"change schedule for {key} to {value}") + if value == "0": # deactivate this schedule return False - if re.search(r"[\d]{1,2}\/[\d]{1,2}", to_check): + if re.search(r"[\d]{1,2}\/[\d]{1,2}", value): # number/number cron format will fail in celery print("number/number schedule formatting not supported") raise ValueError keys = ["minute", "hour", "day_of_week"] - if to_check == "auto": + if value == "auto": # set to sensible default values = self.SCHEDULES[key].split() else: - values = to_check.split() + values = value.split() if len(keys) != len(values): - print(f"failed to parse {to_check} for {key}") + print(f"failed to parse {value} for {key}") raise ValueError("invalid input") to_write = dict(zip(keys, values)) - try: - int(to_write["minute"]) - except ValueError as error: - print("too frequent: only number in minutes are supported") - raise ValueError("invalid input") from error + self._validate_cron(to_write) return to_write + @staticmethod + def _validate_cron(to_write): + """validate all fields, raise value error for impossible schedule""" + all_hours = list(re.split(r"\D+", to_write["hour"])) + for hour in all_hours: + if hour.isdigit() and int(hour) > 23: + print("hour can not be greater than 23") + raise ValueError("invalid input") + + all_days = list(re.split(r"\D+", to_write["day_of_week"])) + for day in all_days: + if day.isdigit() and int(day) > 6: + print("day can not be greater than 6") + raise ValueError("invalid input") + + if not to_write["minute"].isdigit(): + print("too frequent: only number in minutes are supported") + raise ValueError("invalid input") + + if int(to_write["minute"]) > 59: + print("minutes can not be greater than 59") + raise ValueError("invalid input") + def build_schedule(self): """build schedule dict as expected by app.conf.beat_schedule""" schedule_dict = {} diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 80b090b4..a30b4f53 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -89,7 +89,15 @@ {% endif %}
{{ channel_overwrite_form.index_playlists }}Enable SponsorBlock: + {% if channel_info.channel_overwrites.integrate_sponsorblock %} + {{ channel_info.channel_overwrites.integrate_sponsorblock }} + {% else %} + False + {% endif %}
+ {{ channel_overwrite_form.integrate_sponsorblock }}Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}
+Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}
Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}
+Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}
Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.Timestamps sent! (Not really)
+// `; +// setTimeout(function(){ +// sponsorBlockElement.innerHTML = ` +// +// `; +// }, 3000); +// } else { +// sponsorBlockElement.innerHTML = ` +// Invalid Timestamps! +// `; +// setTimeout(function(){ +// sponsorBlockElement.innerHTML = ` +// +// `; +// }, 3000); +// } +// sponsorBlockTimestamps = []; +// } else if (sponsorBlockTimestamps[0]) { +// sponsorBlockTimestamps.push(currentTime); +// sponsorBlockElement.innerHTML = ` +//${sponsorBlockTimestamps[0].toFixed(1)} s |
+//${sponsorBlockTimestamps[1].toFixed(1)} s |
+// +// `; +// } else { +// sponsorBlockTimestamps.push(currentTime); +// sponsorBlockElement.innerHTML = ` +// +// `; +// } +// } + // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)` function insertVideoTag(videoData, videoProgress) { var videoTag = createVideoTag(videoData, videoProgress); @@ -488,6 +560,32 @@ function onVideoProgress() { var videoId = getVideoPlayerVideoId(); var currentTime = getVideoPlayerCurrentTime(); var duration = getVideoPlayerDuration(); + var videoElement = getVideoPlayer(); + // var sponsorBlockElement = document.getElementById("sponsorblock"); + var notificationsElement = document.getElementById("notifications"); + if (sponsorBlock) { + for(let i in sponsorBlock) { + if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) { + videoElement.currentTime = sponsorBlock[i].segment[1]; + notificationsElement.innerHTML += `