From d2c8aadf799a63aaa7da81ae03052b1ec2addd20 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 1 Sep 2022 16:49:03 +0530 Subject: [PATCH] [cleanup] Misc Closes #4710, Closes #4754, Closes #4723 Authored by: pukkandan, MrRawes, DavidH-2022 --- README.md | 45 +++++++++++++-------------------- devscripts/run_tests.sh | 8 +++--- test/test_YoutubeDL.py | 2 +- test/test_jsinterp.py | 2 +- yt_dlp/YoutubeDL.py | 10 ++++---- yt_dlp/__init__.py | 2 +- yt_dlp/cookies.py | 10 ++++++-- yt_dlp/extractor/_extractors.py | 45 +++++++++++++++++---------------- yt_dlp/extractor/common.py | 2 +- yt_dlp/extractor/newspicks.py | 2 +- yt_dlp/extractor/triller.py | 2 +- yt_dlp/options.py | 8 +++--- 12 files changed, 67 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index c4667bb57..28fad2815 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ ### Standalone PyInstaller Builds On some systems, you may need to use `py` or `python` instead of `python3`. -Note that pyinstaller [does not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment. +Note that pyinstaller with versions below 4.4 [do not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment. **Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly. @@ -531,8 +531,8 @@ ## Video Selection: a file that is in the archive --break-on-reject Stop the download process when encountering a file that has been filtered out - --break-per-input Make --break-on-existing, --break-on-reject, - --max-downloads and autonumber reset per + --break-per-input --break-on-existing, --break-on-reject, + --max-downloads, and autonumber resets per input URL --no-break-per-input --break-on-existing and similar options terminates the entire download queue @@ -1238,7 +1238,6 @@ # OUTPUT TEMPLATE - `id` (string): Video identifier - `title` (string): Video title - `fulltitle` (string): Video title ignoring live timestamp and generic title - - `url` (string): Video URL - `ext` (string): Video filename extension - `alt_title` (string): A secondary title of the video - `description` (string): The description of the video @@ -1273,26 +1272,6 @@ # OUTPUT TEMPLATE - `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public" - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL - - `format` (string): A human-readable description of the format - - `format_id` (string): Format code specified by `--format` - - `format_note` (string): Additional info about the format - - `width` (numeric): Width of the video - - `height` (numeric): Height of the video - - `resolution` (string): Textual description of width and height - - `tbr` (numeric): Average bitrate of audio and video in KBit/s - - `abr` (numeric): Average audio bitrate in KBit/s - - `acodec` (string): Name of the audio codec in use - - `asr` (numeric): Audio sampling rate in Hertz - - `vbr` (numeric): Average video bitrate in KBit/s - - `fps` (numeric): Frame rate - - `dynamic_range` (string): The dynamic range of the video - - `audio_channels` (numeric): The number of audio channels - - `stretched_ratio` (float): `width:height` of the video's pixels, if not square - - `vcodec` (string): Name of the video codec in use - - `container` (string): Name of the container format - - `filesize` (numeric): The number of bytes, if known in advance - - `filesize_approx` (numeric): An estimate for the number of bytes - - `protocol` (string): The protocol that will be used for the actual download - `extractor` (string): Name of the extractor - `extractor_key` (string): Key name of the extractor - `epoch` (numeric): Unix epoch of when the information extraction was completed @@ -1311,6 +1290,8 @@ # OUTPUT TEMPLATE - `webpage_url_basename` (string): The basename of the webpage URL - `webpage_url_domain` (string): The domain of the webpage URL - `original_url` (string): The URL given by the user (or same as `webpage_url` for playlist entries) + +All the fields in [Filtering Formats](#filtering-formats) can also be used Available for the video that belongs to some logical chapter or section: @@ -1392,13 +1373,13 @@ #### Output template and Windows batch files #### Output template examples ```bash -$ yt-dlp --get-filename -o "test video.%(ext)s" BaW_jenozKc +$ yt-dlp --print filename -o "test video.%(ext)s" BaW_jenozKc test video.webm # Literal name with correct extension -$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc +$ yt-dlp --print filename -o "%(title)s.%(ext)s" BaW_jenozKc youtube-dl test video ''_ä↭𝕐.webm # All kinds of weird characters -$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames +$ yt-dlp --print filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames youtube-dl_test_video_.webm # Restricted file name # Download YouTube playlist videos in separate directory indexed by video order in a playlist @@ -1487,6 +1468,7 @@ ## Filtering Formats The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `>=`, `=` (equals), `!=` (not equals): - `filesize`: The number of bytes, if known in advance + - `filesize_approx`: An estimate for the number of bytes - `width`: Width of the video, if known - `height`: Height of the video, if known - `tbr`: Average bitrate of audio and video in KBit/s @@ -1494,16 +1476,23 @@ ## Filtering Formats - `vbr`: Average video bitrate in KBit/s - `asr`: Audio sampling rate in Hertz - `fps`: Frame rate + - `audio_channels`: The number of audio channels + - `stretched_ratio`: `width:height` of the video's pixels, if not square Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends with), `*=` (contains), `~=` (matches regex) and following string meta fields: + - `url`: Video URL - `ext`: File extension - `acodec`: Name of the audio codec in use - `vcodec`: Name of the video codec in use - `container`: Name of the container format - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`) - - `format_id`: A short description of the format - `language`: Language code + - `dynamic_range`: The dynamic range of the video + - `format_id`: A short description of the format + - `format`: A human-readable description of the format + - `format_note`: Additional info about the format + - `resolution`: Textual description of width and height Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain). The comparand of a string comparison needs to be quoted with either double or single quotes if it contains spaces or special characters other than `._-`. diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index d496a092b..faa642e96 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -if [ -z $1 ]; then +if [ -z "$1" ]; then test_set='test' -elif [ $1 = 'core' ]; then +elif [ "$1" = 'core' ]; then test_set="-m not download" -elif [ $1 = 'download' ]; then +elif [ "$1" = 'download' ]; then test_set="-m download" else - echo 'Invalid test type "'$1'". Use "core" | "download"' + echo 'Invalid test type "'"$1"'". Use "core" | "download"' exit 1 fi diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 49dc2c198..426e52305 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -668,7 +668,7 @@ def test_add_extra_info(self): def test_prepare_outtmpl_and_filename(self): def test(tmpl, expected, *, info=None, **params): params['outtmpl'] = tmpl - ydl = YoutubeDL(params) + ydl = FakeYDL(params) ydl._num_downloads = 1 self.assertEqual(ydl.validate_outtmpl(tmpl), None) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index b46d0949d..92ef532f5 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -387,7 +387,7 @@ def test_regex(self): ''') self.assertEqual(jsi.call_function('x').flags & re.I, re.I) - jsi = JSInterpreter(''' + jsi = JSInterpreter(R''' function x() { let a=/,][}",],()}(\[)/; return a; } ''') self.assertEqual(jsi.call_function('x').pattern, r',][}",],()}(\[)') diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 2b5b3fdfc..a6bbbb128 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -1044,7 +1044,7 @@ def _parse_outtmpl(self): def get_output_path(self, dir_type='', filename=None): paths = self.params.get('paths', {}) - assert isinstance(paths, dict) + assert isinstance(paths, dict), '"paths" parameter must be a dictionary' path = os.path.join( expand_path(paths.get('home', '').strip()), expand_path(paths.get(dir_type, '').strip()) if dir_type else '', @@ -2745,9 +2745,9 @@ def process_subtitles(self, video_id, normal_subtitles, automatic_captions): if lang not in available_subs: available_subs[lang] = cap_info - if (not self.params.get('writesubtitles') and not - self.params.get('writeautomaticsub') or not - available_subs): + if not available_subs or ( + not self.params.get('writesubtitles') + and not self.params.get('writeautomaticsub')): return None all_sub_langs = tuple(available_subs.keys()) @@ -2764,7 +2764,7 @@ def process_subtitles(self, video_id, normal_subtitles, automatic_captions): else: requested_langs = ['en'] if 'en' in all_sub_langs else all_sub_langs[:1] if requested_langs: - self.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs)) + self.to_screen(f'[info] {video_id}: Downloading subtitles: {", ".join(requested_langs)}') formats_query = self.params.get('subtitlesformat', 'best') formats_preference = formats_query.split('/') if formats_query else [] diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 552f29bd9..356155fcd 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -365,7 +365,7 @@ def parse_chapters(name, value): if keyring not in SUPPORTED_KEYRINGS: raise ValueError(f'unsupported keyring specified for cookies: "{keyring}". ' f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}') - opts.cookiesfrombrowser = (browser_name, profile or None, keyring, container or None) + opts.cookiesfrombrowser = (browser_name, profile, keyring, container) # MetadataParser def metadataparser_actions(f): diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index 9100f46ac..0ccd22947 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -25,7 +25,13 @@ sqlite3, ) from .minicurses import MultilinePrinter, QuietMultilinePrinter -from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path, try_call +from .utils import ( + Popen, + YoutubeDLCookieJar, + error_to_str, + expand_path, + try_call, +) CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'} SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'} @@ -138,7 +144,7 @@ def _extract_firefox_cookies(profile, container, logger): containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json') if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK): raise FileNotFoundError(f'could not read containers.json in {search_root}') - with open(containers_path, 'r') as containers: + with open(containers_path) as containers: identities = json.load(containers).get('identities', []) container_id = next((context.get('userContextId') for context in identities if container in ( context.get('name'), diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 8368e9315..82b701a5d 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1,5 +1,28 @@ # flake8: noqa: F401 +from .youtube import ( # Youtube is moved to the top to improve performance + YoutubeIE, + YoutubeClipIE, + YoutubeFavouritesIE, + YoutubeNotificationsIE, + YoutubeHistoryIE, + YoutubeTabIE, + YoutubeLivestreamEmbedIE, + YoutubePlaylistIE, + YoutubeRecommendedIE, + YoutubeSearchDateIE, + YoutubeSearchIE, + YoutubeSearchURLIE, + YoutubeMusicSearchURLIE, + YoutubeSubscriptionsIE, + YoutubeStoriesIE, + YoutubeTruncatedIDIE, + YoutubeTruncatedURLIE, + YoutubeYtBeIE, + YoutubeYtUserIE, + YoutubeWatchLaterIE, +) + from .abc import ( ABCIE, ABCIViewIE, @@ -2191,28 +2214,6 @@ from .youporn import YouPornIE from .yourporn import YourPornIE from .yourupload import YourUploadIE -from .youtube import ( - YoutubeIE, - YoutubeClipIE, - YoutubeFavouritesIE, - YoutubeNotificationsIE, - YoutubeHistoryIE, - YoutubeTabIE, - YoutubeLivestreamEmbedIE, - YoutubePlaylistIE, - YoutubeRecommendedIE, - YoutubeSearchDateIE, - YoutubeSearchIE, - YoutubeSearchURLIE, - YoutubeMusicSearchURLIE, - YoutubeSubscriptionsIE, - YoutubeStoriesIE, - YoutubeTruncatedIDIE, - YoutubeTruncatedURLIE, - YoutubeYtBeIE, - YoutubeYtUserIE, - YoutubeWatchLaterIE, -) from .zapiks import ZapiksIE from .zattoo import ( BBVTVIE, diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index b9d0305b4..c76133d8f 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -3874,7 +3874,7 @@ def extract_from_webpage(cls, ydl, url, webpage): def _extract_from_webpage(cls, url, webpage): for embed_url in orderedSet( cls._extract_embed_urls(url, webpage) or [], lazy=True): - yield cls.url_result(embed_url, cls) + yield cls.url_result(embed_url, None if cls._VALID_URL is False else cls) @classmethod def _extract_embed_urls(cls, url, webpage): diff --git a/yt_dlp/extractor/newspicks.py b/yt_dlp/extractor/newspicks.py index 0232d5357..a368ce4e0 100644 --- a/yt_dlp/extractor/newspicks.py +++ b/yt_dlp/extractor/newspicks.py @@ -5,7 +5,7 @@ class NewsPicksIE(InfoExtractor): - _VALID_URL = r'https://newspicks.com/movie-series/(?P\d+)\?movieId=(?P\d+)' + _VALID_URL = r'https://newspicks\.com/movie-series/(?P\d+)\?movieId=(?P\d+)' _TESTS = [{ 'url': 'https://newspicks.com/movie-series/11?movieId=1813', diff --git a/yt_dlp/extractor/triller.py b/yt_dlp/extractor/triller.py index c199da91d..e4123f809 100644 --- a/yt_dlp/extractor/triller.py +++ b/yt_dlp/extractor/triller.py @@ -3,13 +3,13 @@ from .common import InfoExtractor from ..utils import ( + ExtractorError, int_or_none, str_or_none, traverse_obj, unified_strdate, unified_timestamp, url_basename, - ExtractorError, ) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 0fbf1f028..4aa0acfbc 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -442,9 +442,9 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs): 'allowed_values': { 'filename', 'filename-sanitization', 'format-sort', 'abort-on-error', 'format-spec', 'no-playlist-metafiles', 'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge', - 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-attach-info-json', 'embed-metadata', - 'embed-thumbnail-atomicparsley', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi', - 'no-youtube-prefer-utc-upload-date' + 'no-attach-info-json', 'embed-metadata', 'embed-thumbnail-atomicparsley', + 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi', + 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date', }, 'aliases': { 'youtube-dl': ['all', '-multistreams'], 'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat'], @@ -634,7 +634,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs): selection.add_option( '--break-per-input', action='store_true', dest='break_per_url', default=False, - help='Make --break-on-existing, --break-on-reject, --max-downloads and autonumber reset per input URL') + help='--break-on-existing, --break-on-reject, --max-downloads, and autonumber resets per input URL') selection.add_option( '--no-break-per-input', action='store_false', dest='break_per_url',