@ -15,6 +15,7 @@ from home.src.download.thumbnails import ThumbManager
from home . src . download . yt_dlp_base import YtWrap
from home . src . download . yt_dlp_base import YtWrap
from home . src . es . connect import ElasticWrap , IndexPaginate
from home . src . es . connect import ElasticWrap , IndexPaginate
from home . src . index . playlist import YoutubePlaylist
from home . src . index . playlist import YoutubePlaylist
from home . src . index . video_constants import VideoTypeEnum
from home . src . ta . config import AppConfig
from home . src . ta . config import AppConfig
from home . src . ta . helper import DurationConverter
from home . src . ta . helper import DurationConverter
from home . src . ta . ta_redis import RedisArchivist
from home . src . ta . ta_redis import RedisArchivist
@ -156,7 +157,8 @@ class PendingList(PendingIndex):
def _process_entry ( self , entry ) :
def _process_entry ( self , entry ) :
""" process single entry from url list """
""" process single entry from url list """
if entry [ " type " ] == " video " :
if entry [ " type " ] == " video " :
self . _add_video ( entry [ " url " ] )
vid_type = entry . get ( " vid_type " , VideoTypeEnum . VIDEO )
self . _add_video ( entry [ " url " ] , vid_type )
elif entry [ " type " ] == " channel " :
elif entry [ " type " ] == " channel " :
self . _parse_channel ( entry [ " url " ] )
self . _parse_channel ( entry [ " url " ] )
elif entry [ " type " ] == " playlist " :
elif entry [ " type " ] == " playlist " :
@ -165,10 +167,10 @@ class PendingList(PendingIndex):
else :
else :
raise ValueError ( f " invalid url_type: { entry } " )
raise ValueError ( f " invalid url_type: { entry } " )
def _add_video ( self , url ):
def _add_video ( self , url , vid_type = VideoTypeEnum . VIDEO ):
""" add video to list """
""" add video to list """
if url not in self . missing_videos and url not in self . to_skip :
if url not in self . missing_videos and url not in self . to_skip :
self . missing_videos . append ( url )
self . missing_videos . append ( ( url , vid_type ) )
else :
else :
print ( f " { url } : skipped adding already indexed video to download. " )
print ( f " { url } : skipped adding already indexed video to download. " )
@ -177,9 +179,8 @@ class PendingList(PendingIndex):
video_results = ChannelSubscription ( ) . get_last_youtube_videos (
video_results = ChannelSubscription ( ) . get_last_youtube_videos (
url , limit = False
url , limit = False
)
)
youtube_ids = [ i [ 0 ] for i in video_results ]
for video_id , _ , vid_type in video_results :
for video_id in youtube_ids :
self . _add_video ( video_id , vid_type )
self . _add_video ( video_id )
def _parse_playlist ( self , url ) :
def _parse_playlist ( self , url ) :
""" add all videos of playlist to list """
""" add all videos of playlist to list """
@ -188,16 +189,18 @@ class PendingList(PendingIndex):
video_results = playlist . json_data . get ( " playlist_entries " )
video_results = playlist . json_data . get ( " playlist_entries " )
youtube_ids = [ i [ " youtube_id " ] for i in video_results ]
youtube_ids = [ i [ " youtube_id " ] for i in video_results ]
for video_id in youtube_ids :
for video_id in youtube_ids :
self . _add_video ( video_id )
# FIXME: This will need to be adjusted to support Live/Shorts
# from playlists
self . _add_video ( video_id , VideoTypeEnum . VIDEO )
def add_to_pending ( self , status = " pending " ) :
def add_to_pending ( self , status = " pending " ) :
""" add missing videos to pending list """
""" add missing videos to pending list """
self . get_channels ( )
self . get_channels ( )
bulk_list = [ ]
bulk_list = [ ]
for idx , youtube_id in enumerate ( self . missing_videos ) :
for idx , ( youtube_id , vid_type ) in enumerate ( self . missing_videos ) :
print ( f " { youtube_id } : add to download queue" )
print ( f " { youtube_id } ({ vid_type } ) : add to download queue" )
video_details = self . get_youtube_details ( youtube_id )
video_details = self . get_youtube_details ( youtube_id , vid_type )
if not video_details :
if not video_details :
continue
continue
@ -235,7 +238,7 @@ class PendingList(PendingIndex):
if idx + 1 % 25 == 0 :
if idx + 1 % 25 == 0 :
print ( " adding to queue progress: " + progress )
print ( " adding to queue progress: " + progress )
def get_youtube_details ( self , youtube_id ):
def get_youtube_details ( self , youtube_id , vid_type = VideoTypeEnum . VIDEO ):
""" get details from youtubedl for single pending video """
""" get details from youtubedl for single pending video """
vid = YtWrap ( self . yt_obs , self . config ) . extract ( youtube_id )
vid = YtWrap ( self . yt_obs , self . config ) . extract ( youtube_id )
if not vid :
if not vid :
@ -249,9 +252,9 @@ class PendingList(PendingIndex):
if vid [ " live_status " ] in [ " is_upcoming " , " is_live " ] :
if vid [ " live_status " ] in [ " is_upcoming " , " is_live " ] :
return False
return False
return self . _parse_youtube_details ( vid )
return self . _parse_youtube_details ( vid , vid_type )
def _parse_youtube_details ( self , vid ):
def _parse_youtube_details ( self , vid , vid_type = VideoTypeEnum . VIDEO ):
""" parse response """
""" parse response """
vid_id = vid . get ( " id " )
vid_id = vid . get ( " id " )
duration_str = DurationConverter . get_str ( vid [ " duration " ] )
duration_str = DurationConverter . get_str ( vid [ " duration " ] )
@ -271,6 +274,8 @@ class PendingList(PendingIndex):
" duration " : duration_str ,
" duration " : duration_str ,
" published " : published ,
" published " : published ,
" timestamp " : int ( datetime . now ( ) . timestamp ( ) ) ,
" timestamp " : int ( datetime . now ( ) . timestamp ( ) ) ,
# Pulling enum value out so it is serializable
" vid_type " : vid_type . value ,
}
}
if self . all_channels :
if self . all_channels :
youtube_details . update (
youtube_details . update (