From 4a28b593c2656e690ce7f0c46dd5a53a0c55b971 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 19 Feb 2022 14:03:23 +0100 Subject: [PATCH] [fix] google images engine: Fix 'scrap_img_by_id' function The 'scrap_img_by_id' function didn't return any longer anything useful. This fix allows the google images engine to present the full source image instead of only the thumbnail. The function scrap_img_by_id() is rpelaced by a fully rewrite to parse image URLs by a regular expression. The new function parse_urls_img_from_js(dom) returns a mapping of data-id to image URL. Closes: https://github.com/searxng/searxng/issues/909 Signed-off-by: Markus Heiser --- searx/engines/google_images.py | 65 +++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 2855860d8..fc192d62d 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -13,6 +13,7 @@ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs """ +import re from urllib.parse import urlencode, unquote from lxml import html @@ -68,15 +69,55 @@ def scrap_out_thumbs(dom): return ret_val -def scrap_img_by_id(script, data_id): - """Get full image URL by data-id in parent element""" - img_url = '' - _script = script.split('\n') - for i, line in enumerate(_script): - if 'gstatic.com/images' in line and data_id in line and i + 1 < len(_script): - url_line = _script[i + 1] - img_url = url_line.split('"')[1] - img_url = unquote(img_url.replace(r'\u00', r'%')) +# [0, "-H96xjSoW5DsgM", ["https://encrypted-tbn0.gstatic.com/images?q...", 155, 324] +# , ["https://assets.cdn.moviepilot.de/files/d3bf..", 576, 1200], +_RE_JS_IMAGE_URL = re.compile( + r'"' + r'([^"]*)' # -H96xjSoW5DsgM + r'",\s*\["' + r'https://[^\.]*\.gstatic.com/images[^"]*' # https://encrypted-tbn0.gstatic.com/images?q... + r'[^\[]*\["' + r'(https?://[^"]*)' # https://assets.cdn.moviepilot.de/files/d3bf... +) + + +def parse_urls_img_from_js(dom): + + # There are two HTML script tags starting with a JS function + # 'AF_initDataCallback(...)' + # + # + # + # + # The second script contains the URLs of the images. + + # The AF_initDataCallback(..) is called with very large dictionary, that + # looks like JSON but it is not JSON since it contains JS variables and + # constants like 'null' (we can't use a JSON parser for). + # + # The alternative is to parse the entire