From 6face215b83a268962f2fcc0a1b72719d6bff8c7 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Sat, 9 Jul 2022 23:08:08 +0200 Subject: [PATCH] bypass google consent with ucbcb=1 --- searx/engines/google.py | 1 + searx/engines/google_images.py | 9 +-------- searx/engines/google_news.py | 10 +--------- searx/engines/google_play_apps.py | 2 +- searx/engines/google_scholar.py | 10 +--------- searx/engines/google_videos.py | 10 +--------- searx/engines/youtube_noapi.py | 4 +--- 7 files changed, 7 insertions(+), 39 deletions(-) diff --git a/searx/engines/google.py b/searx/engines/google.py index acc87307b..4468558ac 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -287,6 +287,7 @@ def request(query, params): 'oe': "utf8", 'start': offset, 'filter': '0', + 'ucbcb': 1, **additional_parameters, } ) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index fc192d62d..b5c7b5302 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -133,14 +133,7 @@ def request(query, params): + '/search' + "?" + urlencode( - { - 'q': query, - 'tbm': "isch", - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'num': 30, - } + {'q': query, 'tbm': "isch", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'num': 30, 'ucbcb': 1} ) ) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 162e4348e..2aef517f6 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -14,7 +14,6 @@ ignores some parameters from the common :ref:`google API`: # pylint: disable=invalid-name import binascii -from datetime import datetime import re from urllib.parse import urlencode from base64 import b64decode @@ -99,13 +98,7 @@ def request(query, params): + '/search' + "?" + urlencode( - { - 'q': query, - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'gl': lang_info['country'], - } + {'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'gl': lang_info['country'], 'ucbcb': 1} ) + ('&ceid=%s' % ceid) ) # ceid includes a ':' character which must not be urlencoded @@ -113,7 +106,6 @@ def request(query, params): params['headers'].update(lang_info['headers']) params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' - params['headers']['Cookie'] = "CONSENT=YES+cb.%s-14-p0.en+F+941;" % datetime.now().strftime("%Y%m%d") return params diff --git a/searx/engines/google_play_apps.py b/searx/engines/google_play_apps.py index 577e4e114..c80a1f270 100644 --- a/searx/engines/google_play_apps.py +++ b/searx/engines/google_play_apps.py @@ -22,7 +22,7 @@ about = { } categories = ["files", "apps"] -search_url = "https://play.google.com/store/search?{query}&c=apps" +search_url = "https://play.google.com/store/search?{query}&c=apps&ucbcb=1" def request(query, params): diff --git a/searx/engines/google_scholar.py b/searx/engines/google_scholar.py index e0700957c..e4c2d7273 100644 --- a/searx/engines/google_scholar.py +++ b/searx/engines/google_scholar.py @@ -85,15 +85,7 @@ def request(query, params): + lang_info['subdomain'] + '/scholar' + "?" - + urlencode( - { - 'q': query, - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'start': offset, - } - ) + + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset, 'ucbcb': 1}) ) query_url += time_range_url(params) diff --git a/searx/engines/google_videos.py b/searx/engines/google_videos.py index 06aac8ae1..3d3f6d93c 100644 --- a/searx/engines/google_videos.py +++ b/searx/engines/google_videos.py @@ -118,15 +118,7 @@ def request(query, params): + lang_info['subdomain'] + '/search' + "?" - + urlencode( - { - 'q': query, - 'tbm': "vid", - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - } - ) + + urlencode({'q': query, 'tbm': "vid", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'ucbcb': 1}) ) if params['time_range'] in time_range_dict: diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py index 406314684..30f89d72f 100644 --- a/searx/engines/youtube_noapi.py +++ b/searx/engines/youtube_noapi.py @@ -3,7 +3,6 @@ Youtube (Videos) """ -from datetime import datetime from functools import reduce from json import loads, dumps from urllib.parse import quote_plus @@ -26,7 +25,7 @@ time_range_support = True # search-url base_url = 'https://www.youtube.com/results' -search_url = base_url + '?search_query={query}&page={page}' +search_url = base_url + '?search_query={query}&page={page}&ucbcb=1' time_range_url = '&sp=EgII{time_range}%253D%253D' # the key seems to be constant next_page_url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' @@ -52,7 +51,6 @@ def request(query, params): ) params['headers']['Content-Type'] = 'application/json' - params['headers']['Cookie'] = "CONSENT=YES+cb.%s-17-p0.en+F+941;" % datetime.now().strftime("%Y%m%d") return params