From f3eaa8dd1c3949b7165157b306ad47df338eaa06 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 5 Mar 2021 10:07:32 +0000 Subject: [PATCH] [Youtube] Extract alerts from continuation (#144) Related: #143 Authored by: colethedj --- yt_dlp/extractor/youtube.py | 57 +++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index b8ff485b3..f9323d292 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -2796,7 +2796,11 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): # See: https://github.com/ytdl-org/youtube-dl/issues/28194 if response.get('continuationContents') or response.get('onResponseReceivedActions'): break - last_error = 'Incomplete data recieved' + + # Youtube may send alerts if there was an issue with the continuation page + self._extract_alerts(response, expected=False) + + last_error = 'Incomplete data received' if count >= retries: self._downloader.report_error(last_error) @@ -2982,23 +2986,35 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): self._extract_mix_playlist(playlist, playlist_id), playlist_id=playlist_id, playlist_title=title) - @staticmethod - def _extract_alerts(data): - for alert_dict in try_get(data, lambda x: x['alerts'], list) or []: - if not isinstance(alert_dict, dict): - continue - for renderer in alert_dict: - alert = alert_dict[renderer] - alert_type = alert.get('type') - if not alert_type: + def _extract_alerts(self, data, expected=False): + + def _real_extract_alerts(): + for alert_dict in try_get(data, lambda x: x['alerts'], list) or []: + if not isinstance(alert_dict, dict): continue - message = try_get(alert, lambda x: x['text']['simpleText'], compat_str) - if message: - yield alert_type, message - for run in try_get(alert, lambda x: x['text']['runs'], list) or []: - message = try_get(run, lambda x: x['text'], compat_str) + for alert in alert_dict.values(): + alert_type = alert.get('type') + if not alert_type: + continue + message = try_get(alert, lambda x: x['text']['simpleText'], compat_str) if message: yield alert_type, message + for run in try_get(alert, lambda x: x['text']['runs'], list) or []: + message = try_get(run, lambda x: x['text'], compat_str) + if message: + yield alert_type, message + + err_msg = None + for alert_type, alert_message in _real_extract_alerts(): + if alert_type.lower() == 'error': + if err_msg: + self._downloader.report_warning('YouTube said: %s - %s' % ('ERROR', err_msg)) + err_msg = alert_message + else: + self._downloader.report_warning('YouTube said: %s - %s' % (alert_type, alert_message)) + + if err_msg: + raise ExtractorError('YouTube said: %s' % err_msg, expected=expected) def _extract_identity_token(self, webpage, item_id): ytcfg = self._extract_ytcfg(item_id, webpage) @@ -3024,16 +3040,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): url, item_id, 'Downloading webpage%s' % (' (retry #%d)' % count if count else '')) data = self._extract_yt_initial_data(item_id, webpage) - err_msg = None - for alert_type, alert_message in self._extract_alerts(data): - if alert_type.lower() == 'error': - if err_msg: - self._downloader.report_warning('YouTube said: %s - %s' % ('ERROR', err_msg)) - err_msg = alert_message - else: - self._downloader.report_warning('YouTube said: %s - %s' % (alert_type, alert_message)) - if err_msg: - raise ExtractorError('YouTube said: %s' % err_msg, expected=True) + self._extract_alerts(data, expected=True) if data.get('contents') or data.get('currentVideoEndpoint'): break if count >= retries: