From f79d30ab5490a5437e2d15f42e7246e94af621f5 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 10 Apr 2022 22:43:00 +0700 Subject: [PATCH 1/8] multi stage build Dockerfile for python wheel --- Dockerfile | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b4f44ae..56c6da48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,17 @@ -# build the tube archivist image from default python slim image +# multi stage to build tube archivist +# first stage to build python wheel, copy into final image -FROM python:3.10.4-slim-bullseye + +# First stage to build python wheel +FROM python:3.10.4-slim-bullseye AS builder ARG TARGETPLATFORM -ARG INSTALL_DEBUG -ENV PYTHONUNBUFFERED 1 +RUN apt-get update +RUN apt-get install -y --no-install-recommends build-essential gcc curl -# install distro packages needed -RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \ - build-essential \ - nginx \ - atomicparsley \ - curl && rm -rf /var/lib/apt/lists/* +# install requirements +COPY ./tubearchivist/requirements.txt /requirements.txt +RUN pip install --user -r requirements.txt # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ @@ -27,11 +27,31 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \ ; fi +# build final image +FROM python:3.10.4-slim-bullseye as tubearchivist + +ARG TARGETPLATFORM +ARG INSTALL_DEBUG + +ENV PYTHONUNBUFFERED 1 + +# copy build requirements +COPY --from=builder /root/.local /root/.local +COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg +COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe +ENV PATH=/root/.local/bin:$PATH + +# install distro packages needed +RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \ + nginx \ + atomicparsley \ + curl && rm -rf /var/lib/apt/lists/* + # install debug tools for testing environment RUN if [ "$INSTALL_DEBUG" ] ; then \ apt-get -y update && apt-get -y install --no-install-recommends \ vim htop bmon net-tools iputils-ping procps \ - && pip install --no-cache-dir ipython --src /usr/local/src \ + && pip install --user ipython \ ; fi # make folders @@ -39,12 +59,6 @@ RUN mkdir /cache RUN mkdir /youtube RUN mkdir /app -# install python dependencies -COPY ./tubearchivist/requirements.txt /requirements.txt -RUN pip install --upgrade pip && \ - pip install --upgrade setuptools && \ - pip install --no-cache-dir -r requirements.txt --src /usr/local/src - # copy config files COPY docker_assets/nginx.conf /etc/nginx/sites-available/default From 385f53372c09147e04751996610d379b5a19e2ed Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 11 Apr 2022 17:57:29 +0700 Subject: [PATCH 2/8] bump django --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 7d03d084..108590a2 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.11.1 celery==5.2.6 -Django==4.0.3 +Django==4.0.4 django-cors-headers==3.11.0 djangorestframework==3.13.1 Pillow==9.1.0 From 1c233dd4f4c4d90f7862d8a6f177e551dca2b507 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 11 Apr 2022 17:58:19 +0700 Subject: [PATCH 3/8] move install requirements after ffmpeg curl for better cache use --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 56c6da48..8f545a18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,6 @@ ARG TARGETPLATFORM RUN apt-get update RUN apt-get install -y --no-install-recommends build-essential gcc curl -# install requirements -COPY ./tubearchivist/requirements.txt /requirements.txt -RUN pip install --user -r requirements.txt - # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \ @@ -27,6 +23,10 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \ ; fi +# install requirements +COPY ./tubearchivist/requirements.txt /requirements.txt +RUN pip install --user -r requirements.txt + # build final image FROM python:3.10.4-slim-bullseye as tubearchivist From 3df1df8b5a842fad9f83d336b0469d1a55e70886 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 12 Apr 2022 17:45:06 +0700 Subject: [PATCH 4/8] set explicit mappings for sponsorblock key --- tubearchivist/home/src/es/index_mapping.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 9a0ee47e..3abf2c1f 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -218,6 +218,19 @@ "index": false } } + }, + "sponsorblock": { + "properties": { + "last_refresh": { + "type": "date" + }, + "has_unlocked": { + "type": "boolean" + }, + "is_enabled": { + "type": "boolean" + } + } } }, "expected_set": { From 8edde732b6247133d92901fb738bad7322592ab2 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 12 Apr 2022 17:47:13 +0700 Subject: [PATCH 5/8] improved sponsorblock key with additional metadata --- tubearchivist/home/src/index/video.py | 33 ++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 28ad5365..a5d8a42f 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -292,6 +292,7 @@ class SponsorBlock: def __init__(self, user_id=False): self.user_id = user_id self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" + self.last_refresh = int(datetime.now().strftime("%s")) def get_sb_id(self): """get sponsorblock userid or generate if needed""" @@ -315,9 +316,35 @@ class SponsorBlock: response = requests.get(url, headers=headers) if not response.ok: print(f"{youtube_id}: sponsorblock failed: {response.text}") - return False - - return response.json() + sponsor_dict = { + "last_refresh": self.last_refresh, + "is_enabled": True, + "segments": [], + } + else: + all_segments = response.json() + sponsor_dict = self._get_sponsor_dict(all_segments) + + return sponsor_dict + + def _get_sponsor_dict(self, all_segments): + """format and process response""" + has_unlocked = False + cleaned_segments = [] + for segment in all_segments: + if not segment["locked"]: + has_unlocked = True + del segment["userID"] + del segment["description"] + cleaned_segments.append(segment) + + sponsor_dict = { + "last_refresh": self.last_refresh, + "has_unlocked": has_unlocked, + "is_enabled": True, + "segments": cleaned_segments, + } + return sponsor_dict def post_timestamps(self, youtube_id, start_time, end_time): """post timestamps to api""" From cd3e9dd024e632dec16bfdb3743714381600f882 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 13 Apr 2022 09:51:15 +0700 Subject: [PATCH 6/8] add playlist API list view --- tubearchivist/api/README.md | 3 +++ tubearchivist/api/src/search_processor.py | 18 ++++++++++++++++++ tubearchivist/api/urls.py | 6 ++++++ tubearchivist/api/views.py | 16 ++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 3c019cdd..65cab1f0 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -123,6 +123,9 @@ POST /api/channel/ ## Channel Item View /api/channel/\/ +## Playlist List View +/api/playlist/ + ## Playlists Item View /api/playlist/\/ diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index dc0a5dbd..0fc672bc 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -40,6 +40,8 @@ class SearchProcess: processed = self._process_video(result["_source"]) if index == "ta_channel": processed = self._process_channel(result["_source"]) + if index == "ta_playlist": + processed = self._process_playlist(result["_source"]) return processed @@ -80,3 +82,19 @@ class SearchProcess: ) return dict(sorted(video_dict.items())) + + @staticmethod + def _process_playlist(playlist_dict): + """run on single playlist dict""" + playlist_id = playlist_dict["playlist_id"] + playlist_last_refresh = date_praser( + playlist_dict["playlist_last_refresh"] + ) + playlist_dict.update( + { + "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg", + "playlist_last_refresh": playlist_last_refresh, + } + ) + + return dict(sorted(playlist_dict.items())) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 0cdf11ab..cb70e4c1 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -7,6 +7,7 @@ from api.views import ( DownloadApiView, LoginApiView, PingView, + PlaylistApiListView, PlaylistApiView, VideoApiListView, VideoApiView, @@ -53,6 +54,11 @@ urlpatterns = [ PlaylistApiView.as_view(), name="api-playlist", ), + path( + "playlist/", + PlaylistApiListView.as_view(), + name="api-playlist-list", + ), path( "download/", DownloadApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 4ad216ef..afe1c0bf 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -258,6 +258,22 @@ class PlaylistApiView(ApiBaseView): return Response(self.response, status=self.status_code) +class PlaylistApiListView(ApiBaseView): + """resolves to /api/playlist/ + GET: returns list of indexed playlists + """ + + search_base = "ta_playlist/_search/" + + def get(self, request): + # pylint: disable=unused-argument + """handle get request""" + data = {"query": {"match_all": {}}} + self.get_document_list(data) + self.get_paginate() + return Response(self.response) + + class DownloadApiView(ApiBaseView): """resolves to /api/download// GET: returns metadata dict of an item in the download queue From 39c4bd8883c5232221e0cd703050fbcc3956221c Mon Sep 17 00:00:00 2001 From: Nathan DeTar Date: Tue, 12 Apr 2022 20:30:32 -0700 Subject: [PATCH 7/8] Adjust to API changes for SB integration. (#222) --- tubearchivist/home/templates/home/video.html | 28 +----- tubearchivist/static/script.js | 95 +++++--------------- 2 files changed, 25 insertions(+), 98 deletions(-) diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index d1ff3a7f..3848f163 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -5,31 +5,11 @@
- {% if video.channel.channel_overwrites.integrate_sponsorblock %} - {% if video.channel.channel_overwrites.integrate_sponsorblock == True %} - {% if not video.sponsorblock %} -

This video doesn't have any sponsor segments added. To add a segment go to this video on YouTube and add a segment using the SponsorBlock extension.

- {% endif %} - {% if video.sponsorblock %} - {% for segment in video.sponsorblock %} - {% if segment.locked != 1 %} -

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

- {{ break }} - {% endif %} - {% endfor %} - {% endif %} - {% endif %} - {% elif config.downloads.integrate_sponsorblock %} - {% if not video.sponsorblock %} + {% if video.sponsorblock.is_enabled %} + {% if video.sponsorblock.segments|length == 0 %}

This video doesn't have any sponsor segments added. To add a segment go to this video on YouTube and add a segment using the SponsorBlock extension.

- {% endif %} - {% if video.sponsorblock %} - {% for segment in video.sponsorblock %} - {% if segment.locked != 1 %} -

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

- {{ break }} - {% endif %} - {% endfor %} + {% elif video.sponsorblock.has_unlocked %} +

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

{% endif %} {% endif %}
diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index ee95b6f8..4031ec06 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -333,24 +333,21 @@ function createPlayer(button) { var videoData = getVideoData(videoId); var sponsorBlockElements = ''; - if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) { + if (videoData.data.sponsorblock.is_enabled) { sponsorBlock = videoData.data.sponsorblock; - if (!sponsorBlock) { + if (sponsorBlock.segments.length == 0) { sponsorBlockElements = `

This video doesn't have any sponsor segments added. To add a segment go to this video on Youtube and add a segment using the SponsorBlock extension.

`; } else { - for(let i in sponsorBlock) { - if(sponsorBlock[i].locked != 1) { - sponsorBlockElements = ` -
-

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

-
- `; - break; - } + if (sponsorBlock.has_unlocked) { + sponsorBlockElements = ` +
+

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

+
+ `; } } } @@ -425,53 +422,6 @@ function createPlayer(button) { divPlayer.innerHTML = markup; } -// function sendSponsorBlockVote(uuid, vote) { -// var videoId = getVideoPlayerVideoId(); -// postSponsorSegmentVote(videoId, uuid, vote); -// } - -// var sponsorBlockTimestamps = []; -// function sendSponsorBlockSegment() { -// var videoId = getVideoPlayerVideoId(); -// var currentTime = getVideoPlayerCurrentTime(); -// var sponsorBlockElement = document.getElementById("sponsorblock"); -// if (sponsorBlockTimestamps[1]) { -// if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) { -// postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]); -// sponsorBlockElement.innerHTML = ` -//

Timestamps sent! (Not really)

-// `; -// setTimeout(function(){ -// sponsorBlockElement.innerHTML = ` -// -// `; -// }, 3000); -// } else { -// sponsorBlockElement.innerHTML = ` -// Invalid Timestamps! -// `; -// setTimeout(function(){ -// sponsorBlockElement.innerHTML = ` -// -// `; -// }, 3000); -// } -// sponsorBlockTimestamps = []; -// } else if (sponsorBlockTimestamps[0]) { -// sponsorBlockTimestamps.push(currentTime); -// sponsorBlockElement.innerHTML = ` -//

${sponsorBlockTimestamps[0].toFixed(1)} s |

-//

${sponsorBlockTimestamps[1].toFixed(1)} s |

-// -// `; -// } else { -// sponsorBlockTimestamps.push(currentTime); -// sponsorBlockElement.innerHTML = ` -// -// `; -// } -// } - // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)` function insertVideoTag(videoData, videoProgress) { var videoTag = createVideoTag(videoData, videoProgress); @@ -563,23 +513,14 @@ function onVideoProgress() { var videoElement = getVideoPlayer(); // var sponsorBlockElement = document.getElementById("sponsorblock"); var notificationsElement = document.getElementById("notifications"); - if (sponsorBlock) { - for(let i in sponsorBlock) { - if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) { - videoElement.currentTime = sponsorBlock[i].segment[1]; - notificationsElement.innerHTML += `

Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.

`; + if (sponsorBlock.segments.length > 0) { + for(let i in sponsorBlock.segments) { + if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) { + videoElement.currentTime = sponsorBlock.segments[i].segment[1]; + notificationsElement.innerHTML += `

Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.

`; } - // if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) { - // if(sponsorBlock[i].locked != 1) { - // sponsorBlockElement.innerHTML += ` - //
- // - // - //
`; - // } - // } - if(currentTime > sponsorBlock[i].segment[1] + 10) { - var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID); + if(currentTime > sponsorBlock.segments[i].segment[1] + 10) { + var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID); if(notificationsElementUUID) { notificationsElementUUID.outerHTML = ''; } @@ -602,6 +543,12 @@ function onVideoEnded() { if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched updateVideoWatchStatus(videoId, "unwatched"); } + for(let i in sponsorBlock.segments) { + var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID); + if(notificationsElementUUID) { + notificationsElementUUID.outerHTML = ''; + } + } } function watchedThreshold(currentTime, duration) { From 3007e02fe5d5b5a2813ea110584b25fd5a70f48c Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 13 Apr 2022 15:53:00 +0700 Subject: [PATCH 8/8] fix per channel sb integration --- tubearchivist/home/src/frontend/forms.py | 2 +- tubearchivist/home/src/index/channel.py | 3 +++ tubearchivist/home/src/index/video.py | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 337a8054..8709ff11 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -200,7 +200,7 @@ class ChannelOverwriteForm(forms.Form): SP_CHOICES = [ ("", "-- change sponsorblock integrations"), - ("0", "disable sponsorblock integration"), + ("disable", "disable sponsorblock integration"), ("1", "enable sponsorblock integration"), ] diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 75824d81..06d0086c 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -351,6 +351,9 @@ class YoutubeChannel(YouTubeItem): for key, value in overwrites.items(): if key not in valid_keys: raise ValueError(f"invalid overwrite key: {key}") + if value == "disable": + to_write[key] = False + continue if value in [0, "0"]: del to_write[key] continue diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index a5d8a42f..88661185 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -412,16 +412,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _check_get_sb(self): """check if need to run sponsor block""" + integrate = False if self.config["downloads"]["integrate_sponsorblock"]: - return True - try: - single_overwrite = self.video_overwrites[self.youtube_id] - _ = single_overwrite["integrate_sponsorblock"] - return True - except KeyError: - return False + integrate = True + + if self.video_overwrites: + single_overwrite = self.video_overwrites.get(self.youtube_id) + if not single_overwrite: + return integrate + + integrate = single_overwrite.get("integrate_sponsorblock", False) - return False + return integrate def _process_youtube_meta(self): """extract relevant fields from youtube"""