From 172ced7129e802a55f3fb23a9300873888e956b9 Mon Sep 17 00:00:00 2001
From: simon <simobilleter@gmail.com>
Date: Fri, 18 Nov 2022 08:59:04 +0700
Subject: [PATCH] configure comments extraction

---
 tubearchivist/home/config.json                 |  2 ++
 tubearchivist/home/src/frontend/forms.py       | 10 ++++++++++
 tubearchivist/home/src/index/comments.py       | 18 +++++++++++++-----
 .../home/templates/home/settings.html          | 18 ++++++++++++++++++
 4 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json
index a953a497..c8450d0d 100644
--- a/tubearchivist/home/config.json
+++ b/tubearchivist/home/config.json
@@ -27,6 +27,8 @@
         "subtitle": false,
         "subtitle_source": false,
         "subtitle_index": false,
+        "comment_max": false,
+        "comment_sort": "top",
         "cookie_import": false,
         "throttledratelimit": false,
         "integrate_ryd": false,
diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py
index 1e4365c2..79cecbef 100644
--- a/tubearchivist/home/src/frontend/forms.py
+++ b/tubearchivist/home/src/frontend/forms.py
@@ -92,6 +92,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable subtitle index"),
     ]
 
+    COMMENT_SORT_CHOICES = [
+        ("", "-- change comments sort settings --"),
+        ("top", "sort comments by top"),
+        ("new", "sort comments by new"),
+    ]
+
     COOKIE_IMPORT_CHOICES = [
         ("", "-- change cookie settings"),
         ("0", "disable cookie"),
@@ -120,6 +126,10 @@ class ApplicationSettingsForm(forms.Form):
     downloads_subtitle_index = forms.ChoiceField(
         widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
     )
+    downloads_comment_max = forms.CharField(required=False)
+    downloads_comment_sort = forms.ChoiceField(
+        widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
+    )
     downloads_cookie_import = forms.ChoiceField(
         widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
     )
diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py
index e06b9737..c907cb59 100644
--- a/tubearchivist/home/src/index/comments.py
+++ b/tubearchivist/home/src/index/comments.py
@@ -9,19 +9,21 @@ from datetime import datetime
 
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
+from home.src.ta.config import AppConfig
 
 
 class Comments:
     """hold all comments functionality"""
 
-    def __init__(self, youtube_id):
+    def __init__(self, youtube_id, config=False):
         self.youtube_id = youtube_id
         self.es_path = f"ta_comment/_doc/{youtube_id}"
-        self.max_comments = "all,100,all,30"
         self.json_data = False
+        self.config = config
 
     def build_json(self):
         """build json document for es"""
+        self._check_config()
         comments_raw = self.get_yt_comments()
         comments_format = self.format_comments(comments_raw)
 
@@ -31,13 +33,19 @@ class Comments:
             "comment_comments": comments_format,
         }
 
+    def _check_config(self):
+        """read config if not attached"""
+        if not self.config:
+            self.config = AppConfig().config
+
     def build_yt_obs(self):
         """
         get extractor config
         max-comments,max-parents,max-replies,max-replies-per-thread
         """
-        max_comments_list = [i.strip() for i in self.max_comments.split(",")]
-        comment_sort = "top"
+        max_comments = self.config["downloads"]["comment_max"]
+        max_comments_list = [i.strip() for i in max_comments.split(",")]
+        comment_sort = self.config["downloads"]["comment_sort"]
 
         yt_obs = {
             "skip_download": True,
@@ -55,7 +63,7 @@ class Comments:
 
     def get_yt_comments(self):
         """get comments from youtube"""
-        print(f"comments: get comments with format {self.max_comments}")
+        print("comments: get comments")
         yt_obs = self.build_yt_obs()
         info_json = YtWrap(yt_obs).extract(self.youtube_id)
         comments_raw = info_json.get("comments")
diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html
index 049f79c5..b02e1bdd 100644
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@@ -114,6 +114,24 @@
                 {{ app_form.downloads_subtitle_index }}
             </div>
         </div>
+        <div class="settings-group">
+            <h2 id="comments">Comments</h2>
+            <div class="settings-item">
+                <p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
+                <i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
+                <p>Example configurations:</p>
+                <ul>
+                    <li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
+                    <li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
+                </ul>
+                {{ app_form.downloads_comment_max }}</p>
+            </div>
+            <div class="settings-item">
+                <p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
+                <i>Select how many comments and threads to download:</i><br>
+                {{ app_form.downloads_comment_sort }}</p>
+            </div>
+        </div>
         <div class="settings-group">
             <h2 id="format">Cookie</h2>
             <div class="settings-item">