Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-02 09:41:07 +00:00
Code Issues Projects Releases Wiki Activity

manual import fix, #build

Browse Source 
Changed:
- fixed manual import errors, #311 #312
- fixed channel extraction to catch all alerts
- add LDAP disable cert check
- downgrade django, fix for DJANGO_DEBUG env var
This commit is contained in:
simon 2022-09-10 11:38:30 +07:00
commit 265a53ace3
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
6 changed files with 40 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml vendored
View File

@ -15,7 +15,7 @@ body:
options:
- label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki).
required: true
- label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations).
- label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and my idea is not already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap).
required: true
- type: textarea

2
README.md
View File

@ -72,7 +72,7 @@ The main Python application that displays and serves your video collection, buil
- And another volume to save application data at **/cache**.
- The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
- The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*.
- Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space.
- Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. Any wrong configurations here will result in a `Bad Request (400)` response.
- Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials.
- `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
- For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.

6
tubearchivist/home/src/download/thumbnails.py
View File

@ -124,7 +124,7 @@ class ThumbManager(ThumbManagerBase):
img_raw.convert("RGB").save(thumb_path)
def vid_thumb_path(self, absolute=False):
def vid_thumb_path(self, absolute=False, create_folder=False):
"""build expected path for video thumbnail from youtube_id"""
folder_name = self.item_id[0].lower()
folder_path = os.path.join("videos", folder_name)
@ -132,6 +132,10 @@ class ThumbManager(ThumbManagerBase):
if absolute:
thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
if create_folder:
folder_path = os.path.join(self.CACHE_DIR, folder_path)
os.makedirs(folder_path, exist_ok=True)
return thumb_path
def download_channel_art(self, urls, skip_existing=False):

15
tubearchivist/home/src/index/channel.py
View File

@ -73,13 +73,14 @@ class ChannelScraper:
def _is_deactivated(self):
"""check if channel is deactivated"""
alert_text = "This channel does not exist."
alerts = self.yt_json.get("alerts")
if alerts and alert_text in str(alerts):
print(f"{self.channel_id}: {alert_text}")
return True
if not alerts:
return False
return False
for alert in alerts:
alert_text = alert["alertRenderer"]["text"]["simpleText"]
print(f"{self.channel_id}: failed to extract, {alert_text}")
return True
def _parse_channel_main(self):
"""extract maintab values from scraped channel json data"""
@ -226,8 +227,8 @@ class YoutubeChannel(YouTubeItem):
self.json_data.update(
{
"channel_subs": content["channel_follower_count"],
"channel_description": content["description"],
"channel_subs": content.get("channel_follower_count", 0),
"channel_description": content.get("description", False),
}
)

34
tubearchivist/home/src/index/filesystem.py
View File

@ -217,15 +217,14 @@ class ImportFolderScanner:
last_base = False
for file_path in all_files:
base_name_raw, ext = os.path.splitext(file_path)
base_name, _ = os.path.splitext(base_name_raw)
base_name, ext = self._detect_base_name(file_path)
key, file_path = self._detect_type(file_path, ext)
if not key or not file_path:
continue
if base_name != last_base:
if last_base:
print(f"manual import: {current_video}")
self.to_import.append(current_video)
current_video = self._get_template()
@ -237,8 +236,21 @@ class ImportFolderScanner:
current_video[key] = file_path
if current_video.get("media"):
print(f"manual import: {current_video}")
self.to_import.append(current_video)
def _detect_base_name(self, file_path):
"""extract base_name and ext for matching"""
base_name_raw, ext = os.path.splitext(file_path)
base_name, ext2 = os.path.splitext(base_name_raw)
if ext2:
if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
# valid secondary extension
return base_name, ext
return base_name_raw, ext
def _detect_type(self, file_path, ext):
"""detect metadata type for file"""
@ -260,12 +272,12 @@ class ImportFolderScanner:
self._convert_thumb(current_video)
self._get_subtitles(current_video)
self._convert_video(current_video)
print(f"manual import: {current_video}")
ManualImport(current_video, self.CONFIG).run()
def _detect_youtube_id(self, current_video):
"""find video id from filename or json"""
print(current_video)
youtube_id = self._extract_id_from_filename(current_video["media"])
if youtube_id:
current_video["video_id"] = youtube_id
@ -276,7 +288,6 @@ class ImportFolderScanner:
current_video["video_id"] = youtube_id
return
print(current_video["media"])
raise ValueError("failed to find video id")
@staticmethod
@ -516,7 +527,8 @@ class ManualImport:
if video.offline_import and self.current_video["thumb"]:
old_path = self.current_video["thumb"]
new_path = ThumbManager(video_id).vid_thumb_path(absolute=True)
thumbs = ThumbManager(video_id)
new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
else:
url = video.json_data["vid_thumb_url"]
@ -555,11 +567,13 @@ class ManualImport:
def _cleanup(self, json_data):
"""cleanup leftover files"""
if os.path.exists(self.current_video["metadata"]):
os.remove(self.current_video["metadata"])
meta_data = self.current_video["metadata"]
if meta_data and os.path.exists(meta_data):
os.remove(meta_data)
if os.path.exists(self.current_video["thumb"]):
os.remove(self.current_video["thumb"])
thumb = self.current_video["thumb"]
if thumb and os.path.exists(thumb):
os.remove(thumb)
for subtitle_file in self.current_video["subtitle"]:
if os.path.exists(subtitle_file):

2
tubearchivist/requirements.txt
View File

@ -10,4 +10,4 @@ requests==2.28.1
ryd-client==0.0.6
uWSGI==2.0.20
whitenoise==6.2.0
yt_dlp==2022.8.19
yt_dlp==2022.9.1