From 1d4247f64e4124476f807e9eeb58fd401ffcd8ae Mon Sep 17 00:00:00 2001 From: Petr Kutalek Date: Sun, 21 Dec 2014 01:24:05 +0100 Subject: [PATCH 1/2] [dvtv] Add support for playlists --- youtube_dl/extractor/dvtv.py | 118 ++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 29 deletions(-) diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py index af552831c..2449b03b5 100644 --- a/youtube_dl/extractor/dvtv.py +++ b/youtube_dl/extractor/dvtv.py @@ -2,62 +2,122 @@ from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( js_to_json, - unescapeHTML + unescapeHTML, + ExtractorError ) class DVTVIE(InfoExtractor): IE_NAME = 'dvtv' - IE_DESC = 'http://video.aktualne.cz/dvtv/' + IE_DESC = 'http://video.aktualne.cz/' - _VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P[a-z0-9-]+/r~[0-9a-f]{32})/?' + _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P[0-9a-f]{32})/' _TESTS = [{ 'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/', - 'md5': '75800f964fa0f82939a2914563301f72', + 'md5': '67cb83e4a955d36e1b5d31993134a0c2', 'info_dict': { - 'id': 'e5efe9ca855511e4833a0025900fea04', - 'ext': 'webm', + 'id': 'dc0768de855511e49e4b0025900fea04', + 'ext': 'mp4', 'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně' } }, { 'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/', 'md5': '6388f1941b48537dbd28791f712af8bf', 'info_dict': { - 'id': '82ed4322849211e4a10c0025900fea04', + 'id': '72c02230849211e49f60002590604f2e', 'ext': 'mp4', 'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala' } + }, { + 'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/', + 'info_dict': { + 'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci', + 'id': '973eb3bc854e11e498be002590604f2e' + }, + 'playlist': [{ + 'md5': 'da7ca6be4935532241fa9520b3ad91e4', + 'info_dict': { + 'id': 'b0b40906854d11e4bdad0025900fea04', + 'ext': 'mp4', + 'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne' + } + }, { + 'md5': '5f7652a08b05009c1292317b449ffea2', + 'info_dict': { + 'id': '420ad9ec854a11e4bdad0025900fea04', + 'ext': 'mp4', + 'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka' + } + }, { + 'md5': '498eb9dfa97169f409126c617e2a3d64', + 'info_dict': { + 'id': '95d35580846a11e4b6d20025900fea04', + 'ext': 'mp4', + 'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?' + } + }, { + 'md5': 'b8dc6b744844032dab6ba3781a7274b9', + 'info_dict': { + 'id': '6fe14d66853511e4833a0025900fea04', + 'ext': 'mp4', + 'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády' + } + }] }] + def _parse_video_metadata(self, js, video_id): + metadata = self._parse_json(js, video_id, transform_source=js_to_json) + + formats = [] + for video in metadata['sources']: + ext = video['type'][6:] + formats.append({ + 'url': video['file'], + 'ext': ext, + 'format': '%s %s' % (ext, video['label']), + 'format_id': '%s-%s' % (ext, video['label']), + 'height': int(video['label'].rstrip('p')), + 'fps': 25, + }) + + self._sort_formats(formats) + + return { + 'id': metadata['mediaid'], + 'title': unescapeHTML(metadata['title']), + 'thumbnail': 'http:%s' % metadata['image'], + 'formats': formats + } + def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - code = self._search_regex( + # singe video + items = re.findall( r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', - webpage, 'video JSON') - payload = self._parse_json(code, video_id, transform_source=js_to_json) - formats = [] - for source in payload['sources']: - ext = source['type'][6:] - formats.append({ - 'url': source['file'], - 'ext': ext, - 'format': '%s %s' % (ext, source['label']), - 'format_id': '%s-%s' % (ext, source['label']), - 'height': int(source['label'].rstrip('p')), - 'fps': 25, - }) - self._sort_formats(formats) + webpage) - return { - 'id': video_id[-32:], - 'display_id': video_id[:-35], - 'title': unescapeHTML(payload['title']), - 'thumbnail': 'http:%s' % payload['image'], - 'formats': formats - } + if items: + return self._parse_video_metadata(items[0], video_id) + + # playlist + items = re.findall( + r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);', + webpage) + + if items: + return { + '_type': 'playlist', + 'id': video_id, + 'title': self._og_search_title(webpage), + 'entries': [self._parse_video_metadata(i, video_id) for i in items] + } + + raise ExtractorError('Could not find neither video nor playlist for requested ID') From 081d6e47840fb923ed6f565fa56dd5def9b9a184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 21 Dec 2014 07:33:58 +0600 Subject: [PATCH 2/2] [dvtv] Simplify --- youtube_dl/extractor/dvtv.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py index 2449b03b5..5f8ac3e46 100644 --- a/youtube_dl/extractor/dvtv.py +++ b/youtube_dl/extractor/dvtv.py @@ -1,5 +1,4 @@ # coding: utf-8 - from __future__ import unicode_literals import re @@ -8,7 +7,7 @@ from ..utils import ( js_to_json, unescapeHTML, - ExtractorError + ExtractorError, ) @@ -16,7 +15,7 @@ class DVTVIE(InfoExtractor): IE_NAME = 'dvtv' IE_DESC = 'http://video.aktualne.cz/' - _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P[0-9a-f]{32})/' + _VALID_URL = r'http://video\.aktualne\.cz/(?:[^/]+/)+r~(?P[0-9a-f]{32})' _TESTS = [{ 'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/', @@ -24,7 +23,7 @@ class DVTVIE(InfoExtractor): 'info_dict': { 'id': 'dc0768de855511e49e4b0025900fea04', 'ext': 'mp4', - 'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně' + 'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně', } }, { 'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/', @@ -32,13 +31,13 @@ class DVTVIE(InfoExtractor): 'info_dict': { 'id': '72c02230849211e49f60002590604f2e', 'ext': 'mp4', - 'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala' + 'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala', } }, { 'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/', 'info_dict': { 'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci', - 'id': '973eb3bc854e11e498be002590604f2e' + 'id': '973eb3bc854e11e498be002590604f2e', }, 'playlist': [{ 'md5': 'da7ca6be4935532241fa9520b3ad91e4', @@ -68,7 +67,10 @@ class DVTVIE(InfoExtractor): 'ext': 'mp4', 'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády' } - }] + }], + }, { + 'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/', + 'only_matching': True, }] def _parse_video_metadata(self, js, video_id): @@ -80,7 +82,6 @@ def _parse_video_metadata(self, js, video_id): formats.append({ 'url': video['file'], 'ext': ext, - 'format': '%s %s' % (ext, video['label']), 'format_id': '%s-%s' % (ext, video['label']), 'height': int(video['label'].rstrip('p')), 'fps': 25, @@ -91,25 +92,26 @@ def _parse_video_metadata(self, js, video_id): return { 'id': metadata['mediaid'], 'title': unescapeHTML(metadata['title']), - 'thumbnail': 'http:%s' % metadata['image'], + 'thumbnail': self._proto_relative_url(metadata['image']), 'formats': formats } def _real_extract(self, url): video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) - # singe video - items = re.findall( - r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', - webpage) + # single video + item = self._search_regex( + r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});", + webpage, 'video', default=None, fatal=False) - if items: - return self._parse_video_metadata(items[0], video_id) + if item: + return self._parse_video_metadata(item, video_id) # playlist items = re.findall( - r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);', + r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);", webpage) if items: @@ -120,4 +122,4 @@ def _real_extract(self, url): 'entries': [self._parse_video_metadata(i, video_id) for i in items] } - raise ExtractorError('Could not find neither video nor playlist for requested ID') + raise ExtractorError('Could not find neither video nor playlist')