From 8011fba3ae4301995f77c64ac63a4a467ef73b7e Mon Sep 17 00:00:00 2001 From: David-Development Date: Sat, 24 Jan 2015 18:28:16 +0100 Subject: [PATCH 1/3] [rtl2] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/rtl2.py | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 youtube_dl/extractor/rtl2.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 3f7ca6f7d..03c56156a 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -349,6 +349,7 @@ from .rtbf import RTBFIE from .rte import RteIE from .rtlnl import RtlXlIE from .rtlnow import RTLnowIE +from .rtl2 import RTL2IE from .rtp import RTPIE from .rts import RTSIE from .rtve import RTVEALaCartaIE, RTVELiveIE diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py new file mode 100644 index 000000000..2bdbf2ec2 --- /dev/null +++ b/youtube_dl/extractor/rtl2.py @@ -0,0 +1,98 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + clean_html, + unified_strdate, + int_or_none, +) + + +class RTL2IE(InfoExtractor): + """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" + _VALID_URL = r'http?://(?P(?P(www\.)?rtl2\.de)/.*/(?P.*))' + _TEST = { + 'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0', + 'md5': 'dsadasdada', + 'info_dict': { + 'id': 'folge-203-0', + 'ext': 'f4v', + 'title': 'GRIP sucht den Sommerk\xf6nig', + 'description' : 'Matthias, Det und Helge treten gegeneinander an.' + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + }, + #'params': { + # rtmp download + # 'skip_download': True, + #}, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_page_url = 'http://%s/' % mobj.group('domain') + video_id = mobj.group('video_id') + + webpage = self._download_webpage('http://' + mobj.group('url'), video_id) + + vico_id = self._html_search_regex(r'vico_id: ([0-9]+)', webpage, '%s'); + vivi_id = self._html_search_regex(r'vivi_id: ([0-9]+)', webpage, '%s'); + + info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id + webpage = self._download_webpage(info_url, '') + + video_info = json.loads(webpage.decode("latin1")) + print video_info + + + #self._download_webpage('http://cp108781.edgefcs.net/crossdomain.xml', '') + + download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s'); + title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s'); + description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s'); + #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s'); + + thumbnail = video_info["video"]["image"] + + download_url = download_url.replace("\\", "") + + stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s'); + + #upload_date = self._html_search_regex(r'property=\"dc:date\".*?datatype=\"xsd:dateTime\".*?content=\"(.*?)\"', webpage, 'title') + #download_url += " -y " + stream_url + + #print stream_url + #print download_url + #print description + #print title + #print ext + + formats = [] + + fmt = { + 'url' : download_url, + #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net', + 'play_path': stream_url, + #'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf', + #'page_url': 'http://www.cbsnews.com', + #'ext': ext, + } + + formats.append(fmt) + + + return { + 'id': video_id, + 'title': title, + 'thumbnail' : thumbnail, + 'description' : description, + 'formats': formats, + } From 7906d199a11c0f8deccc0f87d7ee850106c361dc Mon Sep 17 00:00:00 2001 From: David-Development Date: Sat, 24 Jan 2015 18:28:16 +0100 Subject: [PATCH 2/3] [rtl2] Add new extractor --- youtube_dl/downloader/rtmp.py | 8 ++- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/rtl2.py | 100 +++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 youtube_dl/extractor/rtl2.py diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 6dbbc053c..6f7ad588e 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -104,7 +104,8 @@ class RtmpFD(FileDownloader): live = info_dict.get('rtmp_live', False) conn = info_dict.get('rtmp_conn', None) protocol = info_dict.get('rtmp_protocol', None) - + no_resume = info_dict.get('no_resume', False) + self.report_destination(filename) tmpfilename = self.temp_name(filename) test = self.params.get('test', False) @@ -141,7 +142,10 @@ class RtmpFD(FileDownloader): basic_args += ['--conn', conn] if protocol is not None: basic_args += ['--protocol', protocol] - args = basic_args + [[], ['--resume', '--skip', '1']][not live and self.params.get('continuedl', False)] + if not no_resume: + basic_args += ['--resume'] + + args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)] if sys.platform == 'win32' and sys.version_info < (3, 0): # Windows subprocess module does not actually support Unicode diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 3f7ca6f7d..03c56156a 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -349,6 +349,7 @@ from .rtbf import RTBFIE from .rte import RteIE from .rtlnl import RtlXlIE from .rtlnow import RTLnowIE +from .rtl2 import RTL2IE from .rtp import RTPIE from .rts import RTSIE from .rtve import RTVEALaCartaIE, RTVELiveIE diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py new file mode 100644 index 000000000..14b45e86e --- /dev/null +++ b/youtube_dl/extractor/rtl2.py @@ -0,0 +1,100 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + clean_html, + unified_strdate, + int_or_none, +) + + +class RTL2IE(InfoExtractor): + """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" + _VALID_URL = r'http?://(?P(?P(www\.)?rtl2\.de)/.*/(?P.*))' + _TESTS = [{ + 'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0', + 'info_dict': { + 'id': 'folge-203-0', + 'ext': 'f4v', + 'title': 'GRIP sucht den Sommerk\xf6nig', + 'description' : 'Matthias, Det und Helge treten gegeneinander an.' + }, + 'params': { + # rtmp download + #'skip_download': True, + }, + }, + { + 'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/', + 'info_dict': { + 'id': '21040-anna-erwischt-alex', + 'ext': 'f4v', + 'title': 'GRIP sucht den Sommerk\xf6nig', + 'description' : 'Matthias, Det und Helge treten gegeneinander an.' + }, + 'params': { + # rtmp download + #'skip_download': True, + }, + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_page_url = 'http://%s/' % mobj.group('domain') + video_id = mobj.group('video_id') + + webpage = self._download_webpage('http://' + mobj.group('url'), video_id) + + vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s'); + vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s'); + + info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id + webpage = self._download_webpage(info_url, '') + + video_info = json.loads(webpage) + + download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s'); + title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s'); + description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s'); + #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s'); + + thumbnail = video_info["video"]["image"] + + download_url = download_url.replace("\\", "") + + stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s') + + #print(download_url) + #print(stream_url) + #print(title) + #print(description) + #print(video_id) + + formats = [] + + fmt = { + 'url' : download_url, + #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net', + 'play_path': stream_url, + 'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf', + 'page_url': url, + 'flash_version' : "LNX 11,2,202,429", + 'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"], + 'no_resume' : 1, + } + + formats.append(fmt) + + return { + 'id': video_id, + 'title': title, + 'thumbnail' : thumbnail, + 'description' : description, + 'formats': formats, + } From fe41ddbb285abccc3c4d7a3ebc1238c13ec72577 Mon Sep 17 00:00:00 2001 From: David Development Date: Sun, 25 Jan 2015 11:53:53 +0100 Subject: [PATCH 3/3] refactoring - bug fixes --- youtube_dl/extractor/rtl2.py | 66 ++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py index 14b45e86e..7086d698f 100644 --- a/youtube_dl/extractor/rtl2.py +++ b/youtube_dl/extractor/rtl2.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor from ..utils import ( @@ -14,82 +13,83 @@ from ..utils import ( class RTL2IE(InfoExtractor): - """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" - _VALID_URL = r'http?://(?P(?P(www\.)?rtl2\.de)/.*/(?P.*))' + """Information Extractor for RTL2""" + _VALID_URL = r'http?://(?P(?P(www\.)?rtl2\.de)/.*/(?P.*))/' _TESTS = [{ 'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0', 'info_dict': { 'id': 'folge-203-0', 'ext': 'f4v', - 'title': 'GRIP sucht den Sommerk\xf6nig', + 'title': 'GRIP sucht den Sommerkönig', 'description' : 'Matthias, Det und Helge treten gegeneinander an.' }, 'params': { # rtmp download - #'skip_download': True, + 'skip_download': True, }, }, { 'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/', 'info_dict': { 'id': '21040-anna-erwischt-alex', - 'ext': 'f4v', - 'title': 'GRIP sucht den Sommerk\xf6nig', - 'description' : 'Matthias, Det und Helge treten gegeneinander an.' + 'ext': 'mp4', + 'title': 'Anna erwischt Alex!', + 'description' : 'Anna ist Alex\' Tochter bei Köln 50667.' }, 'params': { # rtmp download - #'skip_download': True, + 'skip_download': True, }, }, ] def _real_extract(self, url): + + #Some rtl2 urls have no slash at the end, so append it. + if not url.endswith("/"): + url += '/' + mobj = re.match(self._VALID_URL, url) - video_page_url = 'http://%s/' % mobj.group('domain') video_id = mobj.group('video_id') - - webpage = self._download_webpage('http://' + mobj.group('url'), video_id) - vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s'); - vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s'); + webpage = self._download_webpage(url, video_id) + + vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found'); + vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found'); info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id webpage = self._download_webpage(info_url, '') - video_info = json.loads(webpage) + video_info = self._download_json(info_url, video_id) - download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s'); - title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s'); - description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s'); - #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s'); + download_url = video_info["video"]["streamurl"] + title = video_info["video"]["titel"] + description = video_info["video"]["beschreibung"] thumbnail = video_info["video"]["image"] download_url = download_url.replace("\\", "") stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s') - - #print(download_url) - #print(stream_url) - #print(title) - #print(description) - #print(video_id) - formats = [] - - fmt = { - 'url' : download_url, + #Debug output + #print('URL: ' + url) + #print('DL URL: ' + download_url) + #print('Stream URL: ' + stream_url) + #print('Title: ' + title) + #print('Description: '+ description) + #print('Video ID: ' + video_id) + + formats = [{ + 'url' : download_url, #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net', 'play_path': stream_url, 'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf', 'page_url': url, 'flash_version' : "LNX 11,2,202,429", 'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"], - 'no_resume' : 1, - } - - formats.append(fmt) + 'no_resume' : True, + }] return { 'id': video_id,