improved sponsorblock indexing, #build

Changed:
- improved more flexible sb timestamps indexing
- fixed per channel sb activate/deactivate
- improved Dockerfile with multi stage build
- added playlist API list view

Dockerfile

@@ -1,17 +1,13 @@
-# build the tube archivist image from default python slim image
+# multi stage to build tube archivist
+# first stage to build python wheel, copy into final image
 
-FROM python:3.10.4-slim-bullseye
+
+# First stage to build python wheel
+FROM python:3.10.4-slim-bullseye AS builder
 ARG TARGETPLATFORM
-ARG INSTALL_DEBUG
 
-ENV PYTHONUNBUFFERED 1
+RUN apt-get update
-
+RUN apt-get install -y --no-install-recommends build-essential gcc curl
-# install distro packages needed
-RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
-    build-essential \
-    nginx \
-    atomicparsley \
-    curl && rm -rf /var/lib/apt/lists/*
 
 # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
@@ -27,11 +23,35 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
     ; fi
 
+# install requirements
+COPY ./tubearchivist/requirements.txt /requirements.txt
+RUN pip install --user -r requirements.txt
+
+# build final image
+FROM python:3.10.4-slim-bullseye as tubearchivist
+
+ARG TARGETPLATFORM
+ARG INSTALL_DEBUG
+
+ENV PYTHONUNBUFFERED 1
+
+# copy build requirements
+COPY --from=builder /root/.local /root/.local
+COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg
+COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe
+ENV PATH=/root/.local/bin:$PATH
+
+# install distro packages needed
+RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
+    nginx \
+    atomicparsley \
+    curl && rm -rf /var/lib/apt/lists/*
+
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends \
         vim htop bmon net-tools iputils-ping procps \
-        && pip install --no-cache-dir ipython --src /usr/local/src \
+        && pip install --user ipython \
     ; fi
 
 # make folders
@@ -39,12 +59,6 @@ RUN mkdir /cache
 RUN mkdir /youtube
 RUN mkdir /app
 
-# install python dependencies
-COPY ./tubearchivist/requirements.txt /requirements.txt
-RUN pip install --upgrade pip && \
-    pip install --upgrade setuptools && \
-    pip install --no-cache-dir -r requirements.txt --src /usr/local/src
-
 # copy config files
 COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
 

tubearchivist/api/README.md

@@ -123,6 +123,9 @@ POST /api/channel/
 ## Channel Item View
 /api/channel/\<channel_id>/
 
+## Playlist List View
+/api/playlist/
+
 ## Playlists Item View
 /api/playlist/\<playlist_id>/
 

tubearchivist/api/src/search_processor.py

@@ -40,6 +40,8 @@ class SearchProcess:
             processed = self._process_video(result["_source"])
         if index == "ta_channel":
             processed = self._process_channel(result["_source"])
+        if index == "ta_playlist":
+            processed = self._process_playlist(result["_source"])
 
         return processed
 
@@ -80,3 +82,19 @@ class SearchProcess:
         )
 
         return dict(sorted(video_dict.items()))
+
+    @staticmethod
+    def _process_playlist(playlist_dict):
+        """run on single playlist dict"""
+        playlist_id = playlist_dict["playlist_id"]
+        playlist_last_refresh = date_praser(
+            playlist_dict["playlist_last_refresh"]
+        )
+        playlist_dict.update(
+            {
+                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_last_refresh": playlist_last_refresh,
+            }
+        )
+
+        return dict(sorted(playlist_dict.items()))

tubearchivist/api/urls.py

@@ -7,6 +7,7 @@ from api.views import (
     DownloadApiView,
     LoginApiView,
     PingView,
+    PlaylistApiListView,
     PlaylistApiView,
     VideoApiListView,
     VideoApiView,
@@ -53,6 +54,11 @@ urlpatterns = [
         PlaylistApiView.as_view(),
         name="api-playlist",
     ),
+    path(
+        "playlist/",
+        PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
     path(
         "download/",
         DownloadApiListView.as_view(),

tubearchivist/api/views.py

@@ -258,6 +258,22 @@ class PlaylistApiView(ApiBaseView):
         return Response(self.response, status=self.status_code)
 
 
+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    """
+
+    search_base = "ta_playlist/_search/"
+
+    def get(self, request):
+        # pylint: disable=unused-argument
+        """handle get request"""
+        data = {"query": {"match_all": {}}}
+        self.get_document_list(data)
+        self.get_paginate()
+        return Response(self.response)
+
+
 class DownloadApiView(ApiBaseView):
     """resolves to /api/download/<video_id>/
     GET: returns metadata dict of an item in the download queue

tubearchivist/home/src/es/index_mapping.json

@@ -218,6 +218,19 @@
                             "index": false
                         }
                     }
+                },
+                "sponsorblock": {
+                    "properties": {
+                        "last_refresh": {
+                            "type": "date"
+                        },
+                        "has_unlocked": {
+                            "type": "boolean"
+                        },
+                        "is_enabled": {
+                            "type": "boolean"
+                        }
+                    }
                 }
             },
             "expected_set": {

tubearchivist/home/src/frontend/forms.py

@@ -200,7 +200,7 @@ class ChannelOverwriteForm(forms.Form):
 
     SP_CHOICES = [
         ("", "-- change sponsorblock integrations"),
-        ("0", "disable sponsorblock integration"),
+        ("disable", "disable sponsorblock integration"),
         ("1", "enable sponsorblock integration"),
     ]
 

tubearchivist/home/src/index/channel.py

@@ -351,6 +351,9 @@ class YoutubeChannel(YouTubeItem):
         for key, value in overwrites.items():
             if key not in valid_keys:
                 raise ValueError(f"invalid overwrite key: {key}")
+            if value == "disable":
+                to_write[key] = False
+                continue
             if value in [0, "0"]:
                 del to_write[key]
                 continue

tubearchivist/home/src/index/video.py

@@ -292,6 +292,7 @@ class SponsorBlock:
     def __init__(self, user_id=False):
         self.user_id = user_id
         self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+        self.last_refresh = int(datetime.now().strftime("%s"))
 
     def get_sb_id(self):
         """get sponsorblock userid or generate if needed"""
@@ -315,9 +316,35 @@ class SponsorBlock:
         response = requests.get(url, headers=headers)
         if not response.ok:
             print(f"{youtube_id}: sponsorblock failed: {response.text}")
-            return False
+            sponsor_dict = {
+                "last_refresh": self.last_refresh,
+                "is_enabled": True,
+                "segments": [],
+            }
+        else:
+            all_segments = response.json()
+            sponsor_dict = self._get_sponsor_dict(all_segments)
 
-        return response.json()
+        return sponsor_dict
+
+    def _get_sponsor_dict(self, all_segments):
+        """format and process response"""
+        has_unlocked = False
+        cleaned_segments = []
+        for segment in all_segments:
+            if not segment["locked"]:
+                has_unlocked = True
+            del segment["userID"]
+            del segment["description"]
+            cleaned_segments.append(segment)
+
+        sponsor_dict = {
+            "last_refresh": self.last_refresh,
+            "has_unlocked": has_unlocked,
+            "is_enabled": True,
+            "segments": cleaned_segments,
+        }
+        return sponsor_dict
 
     def post_timestamps(self, youtube_id, start_time, end_time):
         """post timestamps to api"""
@@ -385,16 +412,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def _check_get_sb(self):
         """check if need to run sponsor block"""
+        integrate = False
         if self.config["downloads"]["integrate_sponsorblock"]:
-            return True
+            integrate = True
-        try:
-            single_overwrite = self.video_overwrites[self.youtube_id]
-            _ = single_overwrite["integrate_sponsorblock"]
-            return True
-        except KeyError:
-            return False
 
-        return False
+        if self.video_overwrites:
+            single_overwrite = self.video_overwrites.get(self.youtube_id)
+            if not single_overwrite:
+                return integrate
+
+            integrate = single_overwrite.get("integrate_sponsorblock", False)
+
+        return integrate
 
     def _process_youtube_meta(self):
         """extract relevant fields from youtube"""

tubearchivist/home/templates/home/video.html

@@ -5,31 +5,11 @@
 <div class="video-main"></div>
 <div class="notifications" id="notifications"></div>
 <div class="sponsorblock" id="sponsorblock">
-    {% if video.channel.channel_overwrites.integrate_sponsorblock %}
+    {% if video.sponsorblock.is_enabled %}
-        {% if video.channel.channel_overwrites.integrate_sponsorblock == True %}
+        {% if video.sponsorblock.segments|length == 0 %}
-            {% if not video.sponsorblock %}
-                <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-            {% endif %}
-            {% if video.sponsorblock %}
-                {% for segment in video.sponsorblock %}
-                    {% if segment.locked != 1 %}
-                        <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                        {{ break }}
-                    {% endif %}
-                {% endfor %}
-            {% endif %}
-        {% endif %}
-    {% elif config.downloads.integrate_sponsorblock %}
-        {% if not video.sponsorblock %}
             <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-        {% endif %}
+        {% elif video.sponsorblock.has_unlocked %}
-        {% if video.sponsorblock %}
+            <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-            {% for segment in video.sponsorblock %}
-                {% if segment.locked != 1 %}
-                    <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                    {{ break }}
-                {% endif %}
-            {% endfor %}
         {% endif %}
     {% endif %}
 </div>

tubearchivist/requirements.txt

@@ -1,6 +1,6 @@
 beautifulsoup4==4.11.1
 celery==5.2.6
-Django==4.0.3
+Django==4.0.4
 django-cors-headers==3.11.0
 djangorestframework==3.13.1
 Pillow==9.1.0

tubearchivist/static/script.js

@@ -333,24 +333,21 @@ function createPlayer(button) {
     var videoData = getVideoData(videoId);
 
     var sponsorBlockElements = '';
-    if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) {
+    if (videoData.data.sponsorblock.is_enabled) {
         sponsorBlock = videoData.data.sponsorblock;
-        if (!sponsorBlock) {
+        if (sponsorBlock.segments.length == 0) {
             sponsorBlockElements = `
             <div class="sponsorblock" id="sponsorblock">
                 <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on Youtube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
             </div>
             `;
         } else {
-            for(let i in sponsorBlock) {
+            if (sponsorBlock.has_unlocked) {
-                if(sponsorBlock[i].locked != 1) {
+                sponsorBlockElements = `
-                    sponsorBlockElements = `
+                <div class="sponsorblock" id="sponsorblock">
-                    <div class="sponsorblock" id="sponsorblock">
+                    <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                        <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
+                </div>
-                    </div>
+                `;
-                    `;
-                    break;
-                }
             }
         }
     }
@@ -425,53 +422,6 @@ function createPlayer(button) {
     divPlayer.innerHTML = markup;
 }
 
-// function sendSponsorBlockVote(uuid, vote) {
-//     var videoId = getVideoPlayerVideoId();
-//     postSponsorSegmentVote(videoId, uuid, vote);
-// }
-
-// var sponsorBlockTimestamps = [];
-// function sendSponsorBlockSegment() {
-//     var videoId = getVideoPlayerVideoId();
-//     var currentTime = getVideoPlayerCurrentTime();
-//     var sponsorBlockElement = document.getElementById("sponsorblock");
-//     if (sponsorBlockTimestamps[1]) {
-//         if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) {
-//             postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]);
-//             sponsorBlockElement.innerHTML = `
-//             <p>Timestamps sent! (Not really)</p>
-//             `;
-//             setTimeout(function(){
-//                 sponsorBlockElement.innerHTML = `
-//                 <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
-//                 `;
-//             }, 3000);
-//         } else {
-//             sponsorBlockElement.innerHTML = `
-//             <span class="danger-zone">Invalid Timestamps!</span>
-//             `;
-//             setTimeout(function(){
-//                 sponsorBlockElement.innerHTML = `
-//                 <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
-//                 `;
-//             }, 3000);
-//         }
-//         sponsorBlockTimestamps = [];
-//     } else if (sponsorBlockTimestamps[0]) {
-//         sponsorBlockTimestamps.push(currentTime);
-//         sponsorBlockElement.innerHTML = `
-//         <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[0]}'"><p>${sponsorBlockTimestamps[0].toFixed(1)} s | </p>
-//         <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[1]}'"><p>${sponsorBlockTimestamps[1].toFixed(1)} s | </p>
-//         <img src="/static/img/PlayerUploadIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Confirm</button>
-//         `;
-//     } else {
-//         sponsorBlockTimestamps.push(currentTime);
-//         sponsorBlockElement.innerHTML = `
-//         <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">End</button>
-//         `;
-//     }
-// }
-
 // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)`
 function insertVideoTag(videoData, videoProgress) {
     var videoTag = createVideoTag(videoData, videoProgress);
@@ -563,23 +513,14 @@ function onVideoProgress() {
     var videoElement = getVideoPlayer();
     // var sponsorBlockElement = document.getElementById("sponsorblock");
     var notificationsElement = document.getElementById("notifications");
-    if (sponsorBlock) {
+    if (sponsorBlock.segments.length > 0) {
-        for(let i in sponsorBlock) {
+        for(let i in sponsorBlock.segments) {
-            if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) {
+            if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) {
-                videoElement.currentTime = sponsorBlock[i].segment[1];
+                videoElement.currentTime = sponsorBlock.segments[i].segment[1];
-                notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.</h3>`;
+                notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock.segments[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.</h3>`;
             }
-            // if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) {
+            if(currentTime > sponsorBlock.segments[i].segment[1] + 10) {
-            //     if(sponsorBlock[i].locked != 1) {
+                var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
-            //         sponsorBlockElement.innerHTML += `
-            //         <div id="${sponsorBlock[i].UUID}">
-            //             <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', 1)">Up Vote</button>
-            //             <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', -1)">Down Vote</button>
-            //         </div>`;
-            //     }
-            // }
-            if(currentTime > sponsorBlock[i].segment[1] + 10) {
-                var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID);
                 if(notificationsElementUUID) {
                     notificationsElementUUID.outerHTML = '';
                 }
@@ -602,6 +543,12 @@ function onVideoEnded() {
     if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
         updateVideoWatchStatus(videoId, "unwatched");
     }
+    for(let i in sponsorBlock.segments) {
+        var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
+        if(notificationsElementUUID) {
+            notificationsElementUUID.outerHTML = '';
+        }
+    }
 }
 
 function watchedThreshold(currentTime, duration) {