implement cookie import, #build

Changed:
- fix arm64 ffmpeg issue
- added cookie import
- use cookie for all yt-dlp calls
- API: add cookie validation view

Dockerfile

@@ -7,21 +7,7 @@ FROM python:3.10.4-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends build-essential gcc curl
+RUN apt-get install -y --no-install-recommends build-essential gcc
-
-# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
-    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
-        | grep browser_download_url \
-        | grep ".*master.*linux64.*tar.xz" \
-        | cut -d '"' -f 4 \
-        | xargs curl -L --output ffmpeg.tar.xz && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
-        rm ffmpeg.tar.xz \
-    ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
-        apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
-    ; fi
 
 # install requirements
 COPY ./tubearchivist/requirements.txt /requirements.txt
@@ -37,15 +23,28 @@ ENV PYTHONUNBUFFERED 1
 
 # copy build requirements
 COPY --from=builder /root/.local /root/.local
-COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg
-COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe
 ENV PATH=/root/.local/bin:$PATH
 
 # install distro packages needed
 RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
     nginx \
     atomicparsley \
-    curl && rm -rf /var/lib/apt/lists/*
+    curl \
+    xz-utils && rm -rf /var/lib/apt/lists/*
+
+# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
+    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
+        | grep browser_download_url \
+        | grep ".*master.*linux64.*tar.xz" \
+        | cut -d '"' -f 4 \
+        | xargs curl -L --output ffmpeg.tar.xz && \
+        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
+        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
+        rm ffmpeg.tar.xz \
+    ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
+        apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
+    ; fi
 
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \

tubearchivist/api/README.md

@@ -213,3 +213,21 @@ POST /api/task/
 List of valid task names:
 - **download_pending**: Start the download queue
 - **rescan_pending**: Rescan your subscriptions
+
+
+## Cookie View
+Check your youtube cookie settings  
+GET /api/cookie/
+```json
+{
+    "cookie_enabled": true
+}
+```
+
+POST /api/cookie/  
+Send empty post request to validate cookie.
+```json
+{
+    "cookie_validated": true
+}
+```

tubearchivist/api/urls.py

@@ -4,6 +4,7 @@ from api.views import (
     ChannelApiListView,
     ChannelApiVideoView,
     ChannelApiView,
+    CookieView,
     DownloadApiListView,
     DownloadApiView,
     LoginApiView,
@@ -87,4 +88,9 @@ urlpatterns = [
         TaskApiView.as_view(),
         name="api-task",
     ),
+    path(
+        "cookie/",
+        CookieView.as_view(),
+        name="api-cookie",
+    ),
 ]

tubearchivist/api/views.py

@@ -3,6 +3,7 @@
 from api.src.search_processor import SearchProcess
 from api.src.task_processor import TaskHandler
 from home.src.download.queue import PendingInteract
+from home.src.download.yt_cookie import CookieHandler
 from home.src.es.connect import ElasticWrap
 from home.src.index.generic import Pagination
 from home.src.index.video import SponsorBlock
@@ -462,3 +463,27 @@ class TaskApiView(ApiBaseView):
         response = TaskHandler(data).run_task()
 
         return Response(response)
+
+
+class CookieView(ApiBaseView):
+    """resolves to /api/cookie/
+    GET: check if cookie is enabled
+    POST: verify validity of cookie
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        cookie_enabled = config["downloads"]["cookie_import"]
+
+        return Response({"cookie_enabled": cookie_enabled})
+
+    @staticmethod
+    def post(request):
+        """handle post request"""
+        # pylint: disable=unused-argument
+        validated = CookieHandler().validate()
+
+        return Response({"cookie_validated": validated})

tubearchivist/home/config.json

@@ -26,6 +26,7 @@
         "subtitle": false,
         "subtitle_source": false,
         "subtitle_index": false,
+        "cookie_import": false,
         "throttledratelimit": false,
         "integrate_ryd": false,
         "integrate_sponsorblock": false

tubearchivist/home/src/download/queue.py

@@ -13,8 +13,10 @@ from home.src.download.subscriptions import (
     PlaylistSubscription,
 )
 from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_cookie import CookieHandler
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
+from home.src.ta.config import AppConfig
 from home.src.ta.helper import DurationConverter
 from home.src.ta.ta_redis import RedisArchivist
 
@@ -119,12 +121,29 @@ class PendingInteract:
 class PendingList(PendingIndex):
     """manage the pending videos list"""
 
+    yt_obs = {
+        "default_search": "ytsearch",
+        "quiet": True,
+        "check_formats": "selected",
+        "noplaylist": True,
+        "writethumbnail": True,
+        "simulate": True,
+    }
+
     def __init__(self, youtube_ids=False):
         super().__init__()
+        self.process_config()
         self.youtube_ids = youtube_ids
         self.to_skip = False
         self.missing_videos = False
 
+    def process_config(self):
+        """add user config to yt_obs"""
+        config = AppConfig().config
+        if config["downloads"]["cookie_import"]:
+            cookie_path = CookieHandler().use()
+            self.yt_obs.update({"cookiefile": cookie_path})
+
     def parse_url_list(self):
         """extract youtube ids from list"""
         self.missing_videos = []
@@ -223,16 +242,8 @@ class PendingList(PendingIndex):
 
     def get_youtube_details(self, youtube_id):
         """get details from youtubedl for single pending video"""
-        obs = {
-            "default_search": "ytsearch",
-            "quiet": True,
-            "check_formats": "selected",
-            "noplaylist": True,
-            "writethumbnail": True,
-            "simulate": True,
-        }
         try:
-            vid = yt_dlp.YoutubeDL(obs).extract_info(youtube_id)
+            vid = yt_dlp.YoutubeDL(self.yt_obs).extract_info(youtube_id)
         except yt_dlp.utils.DownloadError:
             print("failed to extract info for: " + youtube_id)
             return False

tubearchivist/home/src/download/yt_cookie.py

@@ -0,0 +1,85 @@
+"""
+functionality:
+- import yt cookie from filesystem
+- make cookie available for yt-dlp
+"""
+
+import os
+
+import yt_dlp
+from home.src.ta.config import AppConfig
+from home.src.ta.ta_redis import RedisArchivist
+
+
+class CookieHandler:
+    """handle youtube cookie for yt-dlp"""
+
+    CONFIG = AppConfig().config
+    CACHE_PATH = CONFIG["application"]["cache_dir"]
+    COOKIE_FILE_NAME = "cookies.google.txt"
+    COOKIE_KEY = "cookie"
+    COOKIE_PATH = "cookie.txt"
+
+    def import_cookie(self):
+        """import cookie from file"""
+        import_path = os.path.join(
+            self.CACHE_PATH, "import", self.COOKIE_FILE_NAME
+        )
+        with open(import_path, encoding="utf-8") as cookie_file:
+            cookie = cookie_file.read()
+
+        RedisArchivist().set_message(self.COOKIE_KEY, cookie, expire=False)
+
+        os.remove(import_path)
+        print("cookie: import successfully")
+
+    def use(self):
+        """make cookie available in FS"""
+        cookie = RedisArchivist().get_message(self.COOKIE_KEY)
+        if isinstance(cookie, dict):
+            print("no cookie imported")
+            raise FileNotFoundError
+
+        with open(self.COOKIE_PATH, "w", encoding="utf-8") as cookie_file:
+            cookie_file.write(cookie)
+
+        print("cookie: made available")
+        return self.COOKIE_PATH
+
+    def hide(self):
+        """hide cookie file if not in use"""
+        try:
+            os.remove(self.COOKIE_PATH)
+        except FileNotFoundError:
+            print("cookie: not available")
+            return
+
+        print("cookie: hidden")
+
+    def revoke(self):
+        """revoke cookie"""
+        self.hide()
+        RedisArchivist().del_message(self.COOKIE_KEY)
+        print("cookie: revoked")
+
+    def validate(self):
+        """validate cookie using the liked videos playlist"""
+        try:
+            _ = self.use()
+        except FileNotFoundError:
+            return False
+
+        url = "https://www.youtube.com/playlist?list=LL"
+        yt_obs = {
+            "quiet": True,
+            "skip_download": True,
+            "extract_flat": True,
+            "cookiefile": self.COOKIE_PATH,
+        }
+        try:
+            response = yt_dlp.YoutubeDL(yt_obs).extract_info(url)
+        except yt_dlp.utils.DownloadError:
+            print("failed to validate cookie")
+            response = False
+
+        return bool(response)

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -14,6 +14,7 @@ from time import sleep
 import yt_dlp
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import PlaylistSubscription
+from home.src.download.yt_cookie import CookieHandler
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
@@ -290,6 +291,9 @@ class VideoDownloader:
             self.obs["ratelimit"] = (
                 self.config["downloads"]["limit_speed"] * 1024
             )
+        if self.config["downloads"]["cookie_import"]:
+            cookie_path = CookieHandler().use()
+            self.obs["cookiefile"] = cookie_path
 
         throttle = self.config["downloads"]["throttledratelimit"]
         if throttle:

tubearchivist/home/src/frontend/forms.py

@@ -86,6 +86,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable subtitle index"),
     ]
 
+    COOKIE_IMPORT_CHOICES = [
+        ("", "-- change cookie settings"),
+        ("0", "disable cookie"),
+        ("1", "enable cookie"),
+    ]
+
     subscriptions_channel_size = forms.IntegerField(required=False)
     downloads_limit_count = forms.IntegerField(required=False)
     downloads_limit_speed = forms.IntegerField(required=False)
@@ -106,6 +112,9 @@ class ApplicationSettingsForm(forms.Form):
     downloads_subtitle_index = forms.ChoiceField(
         widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
     )
+    downloads_cookie_import = forms.ChoiceField(
+        widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
+    )
     downloads_integrate_ryd = forms.ChoiceField(
         widget=forms.Select, choices=RYD_CHOICES, required=False
     )

tubearchivist/home/src/index/generic.py

@@ -6,6 +6,7 @@ functionality:
 import math
 
 import yt_dlp
+from home.src.download.yt_cookie import CookieHandler
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisArchivist
@@ -37,6 +38,9 @@ class YouTubeItem:
         """read user conf"""
         self.config = AppConfig().config
         self.app_conf = self.config["application"]
+        if self.config["downloads"]["cookie_import"]:
+            cookie_path = CookieHandler().use()
+            self.yt_obs.update({"cookiefile": cookie_path})
 
     def get_from_youtube(self):
         """use yt-dlp to get meta data from youtube"""

tubearchivist/home/src/ta/config.py

@@ -83,6 +83,7 @@ class AppConfig:
 
     def update_config(self, form_post):
         """update config values from settings form"""
+        updated = []
         for key, value in form_post.items():
             if not value and not isinstance(value, int):
                 continue
@@ -96,8 +97,10 @@ class AppConfig:
 
             config_dict, config_value = key.split("_", maxsplit=1)
             self.config[config_dict][config_value] = to_write
+            updated.append((config_value, to_write))
 
         RedisArchivist().set_message("config", self.config, expire=False)
+        return updated
 
     @staticmethod
     def set_user_config(form_post, user_id):

tubearchivist/home/templates/home/settings.html

@@ -114,6 +114,14 @@
                 {{ app_form.downloads_subtitle_index }}
             </div>
         </div>
+        <div class="settings-group">
+            <h2 id="format">Cookie</h2>
+            <div class="settings-item">
+                <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
+                <i>Place your cookie file named <span class="settings-current">cookies.google.txt</span> in /cache/import before enabling.</i><br>
+                {{ app_form.downloads_cookie_import }}
+            </div>
+        </div>
         <div class="settings-group">
             <h2 id="integrations">Integrations</h2>
             <div class="settings-item">

tubearchivist/home/views.py

@@ -14,6 +14,7 @@ from django.contrib.auth.forms import AuthenticationForm
 from django.http import JsonResponse
 from django.shortcuts import redirect, render
 from django.views import View
+from home.src.download.yt_cookie import CookieHandler
 from home.src.es.index_setup import get_available_backups
 from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
@@ -791,8 +792,7 @@ class SettingsView(View):
         token = Token.objects.get_or_create(user=request.user)[0]
         return token
 
-    @staticmethod
+    def post(self, request):
-    def post(request):
         """handle form post to update settings"""
         user_form = UserSettingsForm(request.POST)
         if user_form.is_valid():
@@ -805,7 +805,8 @@ class SettingsView(View):
             app_form_post = app_form.cleaned_data
             if app_form_post:
                 print(app_form_post)
-                AppConfig().update_config(app_form_post)
+                updated = AppConfig().update_config(app_form_post)
+                self.post_process_updated(updated)
 
         scheduler_form = SchedulerSettingsForm(request.POST)
         if scheduler_form.is_valid():
@@ -817,6 +818,19 @@ class SettingsView(View):
         sleep(1)
         return redirect("settings", permanent=True)
 
+    @staticmethod
+    def post_process_updated(updated):
+        """apply changes for config"""
+        if not updated:
+            return
+
+        for config_value, updated_value in updated:
+            if config_value == "cookie_import":
+                if updated_value:
+                    CookieHandler().import_cookie()
+                else:
+                    CookieHandler().revoke()
+
 
 def progress(request):
     # pylint: disable=unused-argument