From 76b01870f864ed91d2bbf7493d7fc0bfe81ab9d3 Mon Sep 17 00:00:00 2001 From: Adrian Heine Date: Sun, 16 Aug 2020 01:30:49 +0200 Subject: [PATCH 1/2] [ABC] Fix extraction --- youtube_dl/extractor/abc.py | 83 ++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py index 6637f4f35..ede280f04 100644 --- a/youtube_dl/extractor/abc.py +++ b/youtube_dl/extractor/abc.py @@ -12,6 +12,7 @@ js_to_json, int_or_none, parse_iso8601, + str_or_none, try_get, unescapeHTML, update_url_query, @@ -34,7 +35,7 @@ class ABCIE(InfoExtractor): 'skip': 'this video has expired', }, { 'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326', - 'md5': 'db2a5369238b51f9811ad815b69dc086', + 'md5': '4ebd61bdc82d9a8b722f64f1f4b4d121', 'info_dict': { 'id': 'NvqvPeNZsHU', 'ext': 'mp4', @@ -58,39 +59,81 @@ class ABCIE(InfoExtractor): }, { 'url': 'http://www.abc.net.au/news/2015-10-19/6866214', 'only_matching': True, + }, { + 'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074', + 'info_dict': { + 'id': '12342074', + 'ext': 'mp4', + 'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia', + 'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f', + } }] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - mobj = re.search( - r'inline(?PVideo|Audio|YouTube)Data\.push\((?P[^)]+)\);', - webpage) - if mobj is None: - expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?(.+?)', webpage, 'expired', None) - if expired: - raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True) - raise ExtractorError('Unable to extract video urls') + mobj = re.search(r'[^"]+)"\s+data-duration="\d+"\s+title="Download audio directly">', webpage) + if mobj: + urls_info = mobj.groupdict() + youtube = False + video = False + else: + mobj = re.search(r'External Link:', + webpage) + if mobj: + urls_info = mobj.groupdict() + youtube = True + video = True - urls_info = self._parse_json( - mobj.group('json_data'), video_id, transform_source=js_to_json) + if mobj is None: + mobj = re.search(r'(?P)"sources": (?P\[[^\]]+\]),', webpage) + if mobj is None: + mobj = re.search( + r'inline(?PVideo|Audio|YouTube)Data\.push\((?P[^)]+)\);', + webpage) + if mobj is None: + expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?(.+?)', webpage, 'expired', None) + if expired: + raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True) + raise ExtractorError('Unable to extract video urls') + + urls_info = self._parse_json( + mobj.group('json_data'), video_id, transform_source=js_to_json) + youtube = mobj.group('type') == 'YouTube' + video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4' if not isinstance(urls_info, list): urls_info = [urls_info] - if mobj.group('type') == 'YouTube': + if youtube: return self.playlist_result([ self.url_result(url_info['url']) for url_info in urls_info]) - formats = [{ - 'url': url_info['url'], - 'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none', - 'width': int_or_none(url_info.get('width')), - 'height': int_or_none(url_info.get('height')), - 'tbr': int_or_none(url_info.get('bitrate')), - 'filesize': int_or_none(url_info.get('filesize')), - } for url_info in urls_info] + formats = [] + for url_info in urls_info: + height = int_or_none(url_info.get('height')) + bitrate = int_or_none(url_info.get('bitrate')) + width = int_or_none(url_info.get('width')) + format_id = None + mobj = re.search(r'_(?:(?P\d+)|(?P\d+)k)\.mp4$', url_info['url']) + if mobj: + height_from_url = mobj.group('height') + if height_from_url: + height = height or int_or_none(height_from_url) + width = width or int_or_none(url_info.get('label')) + else: + bitrate = bitrate or int_or_none(mobj.group('bitrate')) + format_id = str_or_none(url_info.get('label')) + formats.append({ + 'url': url_info['url'], + 'vcodec': url_info.get('codec') if video else 'none', + 'width': width, + 'height': height, + 'tbr': bitrate, + 'filesize': int_or_none(url_info.get('filesize')), + 'format_id': format_id + }) self._sort_formats(formats) From 5630d7979425ab95ce1aaa60ce1fd82770848eb6 Mon Sep 17 00:00:00 2001 From: Adrian Heine Date: Sun, 16 Aug 2020 01:31:11 +0200 Subject: [PATCH 2/2] [ABC] Support abc.net.au/btn (closes #21254) --- youtube_dl/extractor/abc.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py index ede280f04..3e202168e 100644 --- a/youtube_dl/extractor/abc.py +++ b/youtube_dl/extractor/abc.py @@ -21,7 +21,7 @@ class ABCIE(InfoExtractor): IE_NAME = 'abc.net.au' - _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/news/(?:[^/]+/){1,2}(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn)/(?:[^/]+/){1,4}(?P\d{5,})' _TESTS = [{ 'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334', @@ -59,6 +59,14 @@ class ABCIE(InfoExtractor): }, { 'url': 'http://www.abc.net.au/news/2015-10-19/6866214', 'only_matching': True, + }, { + 'url': 'https://www.abc.net.au/btn/classroom/wwi-centenary/10527914', + 'info_dict': { + 'id': '10527914', + 'ext': 'mp4', + 'title': 'WWI Centenary', + 'description': 'md5:c2379ec0ca84072e86b446e536954546', + } }, { 'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074', 'info_dict': { @@ -67,6 +75,17 @@ class ABCIE(InfoExtractor): 'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia', 'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f', } + }, { + 'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476', + 'info_dict': { + 'id': 'tDL8Ld4dK_8', + 'ext': 'mp4', + 'title': 'Fortnite Banned From Apple and Google App Stores', + 'description': 'md5:a6df3f36ce8f816b74af4bd6462f5651', + 'upload_date': '20200813', + 'uploader': 'Behind the News', + 'uploader_id': 'behindthenews', + } }] def _real_extract(self, url): @@ -81,6 +100,8 @@ def _real_extract(self, url): else: mobj = re.search(r'External Link:', webpage) + if mobj is None: + mobj = re.search(r'