|
|
@ -14,7 +14,7 @@ from lxml import html, etree
|
|
|
|
from searx.engines.xpath import extract_text, extract_url
|
|
|
|
from searx.engines.xpath import extract_text, extract_url
|
|
|
|
from searx import logger
|
|
|
|
from searx import logger
|
|
|
|
from searx.url_utils import urlencode, urlparse, parse_qsl
|
|
|
|
from searx.url_utils import urlencode, urlparse, parse_qsl
|
|
|
|
from searx.utils import match_language
|
|
|
|
from searx.utils import match_language, eval_xpath
|
|
|
|
|
|
|
|
|
|
|
|
logger = logger.getChild('google engine')
|
|
|
|
logger = logger.getChild('google engine')
|
|
|
|
|
|
|
|
|
|
|
@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):
|
|
|
|
|
|
|
|
|
|
|
|
# returns extract_text on the first result selected by the xpath or None
|
|
|
|
# returns extract_text on the first result selected by the xpath or None
|
|
|
|
def extract_text_from_dom(result, xpath):
|
|
|
|
def extract_text_from_dom(result, xpath):
|
|
|
|
r = result.xpath(xpath)
|
|
|
|
r = eval_xpath(result, xpath)
|
|
|
|
if len(r) > 0:
|
|
|
|
if len(r) > 0:
|
|
|
|
return extract_text(r[0])
|
|
|
|
return extract_text(r[0])
|
|
|
|
return None
|
|
|
|
return None
|
|
|
@ -227,21 +227,21 @@ def response(resp):
|
|
|
|
# convert the text to dom
|
|
|
|
# convert the text to dom
|
|
|
|
dom = html.fromstring(resp.text)
|
|
|
|
dom = html.fromstring(resp.text)
|
|
|
|
|
|
|
|
|
|
|
|
instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
|
|
|
|
instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')
|
|
|
|
if instant_answer:
|
|
|
|
if instant_answer:
|
|
|
|
results.append({'answer': u' '.join(instant_answer)})
|
|
|
|
results.append({'answer': u' '.join(instant_answer)})
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
|
|
|
|
results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]
|
|
|
|
.split()[1].replace(',', ''))
|
|
|
|
.split()[1].replace(',', ''))
|
|
|
|
results.append({'number_of_results': results_num})
|
|
|
|
results.append({'number_of_results': results_num})
|
|
|
|
except:
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
# parse results
|
|
|
|
# parse results
|
|
|
|
for result in dom.xpath(results_xpath):
|
|
|
|
for result in eval_xpath(dom, results_xpath):
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
title = extract_text(result.xpath(title_xpath)[0])
|
|
|
|
title = extract_text(eval_xpath(result, title_xpath)[0])
|
|
|
|
url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
|
|
|
|
url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)
|
|
|
|
parsed_url = urlparse(url, google_hostname)
|
|
|
|
parsed_url = urlparse(url, google_hostname)
|
|
|
|
|
|
|
|
|
|
|
|
# map result
|
|
|
|
# map result
|
|
|
@ -250,7 +250,7 @@ def response(resp):
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
# if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
|
|
|
|
# if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
|
|
|
|
# print "yooooo"*30
|
|
|
|
# print "yooooo"*30
|
|
|
|
# x = result.xpath(map_near)
|
|
|
|
# x = eval_xpath(result, map_near)
|
|
|
|
# if len(x) > 0:
|
|
|
|
# if len(x) > 0:
|
|
|
|
# # map : near the location
|
|
|
|
# # map : near the location
|
|
|
|
# results = results + parse_map_near(parsed_url, x, google_hostname)
|
|
|
|
# results = results + parse_map_near(parsed_url, x, google_hostname)
|
|
|
@ -287,11 +287,11 @@ def response(resp):
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
# parse suggestion
|
|
|
|
# parse suggestion
|
|
|
|
for suggestion in dom.xpath(suggestion_xpath):
|
|
|
|
for suggestion in eval_xpath(dom, suggestion_xpath):
|
|
|
|
# append suggestion
|
|
|
|
# append suggestion
|
|
|
|
results.append({'suggestion': extract_text(suggestion)})
|
|
|
|
results.append({'suggestion': extract_text(suggestion)})
|
|
|
|
|
|
|
|
|
|
|
|
for correction in dom.xpath(spelling_suggestion_xpath):
|
|
|
|
for correction in eval_xpath(dom, spelling_suggestion_xpath):
|
|
|
|
results.append({'correction': extract_text(correction)})
|
|
|
|
results.append({'correction': extract_text(correction)})
|
|
|
|
|
|
|
|
|
|
|
|
# return results
|
|
|
|
# return results
|
|
|
@ -300,9 +300,9 @@ def response(resp):
|
|
|
|
|
|
|
|
|
|
|
|
def parse_images(result, google_hostname):
|
|
|
|
def parse_images(result, google_hostname):
|
|
|
|
results = []
|
|
|
|
results = []
|
|
|
|
for image in result.xpath(images_xpath):
|
|
|
|
for image in eval_xpath(result, images_xpath):
|
|
|
|
url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
|
|
|
|
url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)
|
|
|
|
img_src = extract_text(image.xpath(image_img_src_xpath)[0])
|
|
|
|
img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])
|
|
|
|
|
|
|
|
|
|
|
|
# append result
|
|
|
|
# append result
|
|
|
|
results.append({'url': url,
|
|
|
|
results.append({'url': url,
|
|
|
@ -389,10 +389,10 @@ def attributes_to_html(attributes):
|
|
|
|
def _fetch_supported_languages(resp):
|
|
|
|
def _fetch_supported_languages(resp):
|
|
|
|
supported_languages = {}
|
|
|
|
supported_languages = {}
|
|
|
|
dom = html.fromstring(resp.text)
|
|
|
|
dom = html.fromstring(resp.text)
|
|
|
|
options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
|
|
|
|
options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')
|
|
|
|
for option in options:
|
|
|
|
for option in options:
|
|
|
|
code = option.xpath('./@value')[0].split('_')[-1]
|
|
|
|
code = eval_xpath(option, './@value')[0].split('_')[-1]
|
|
|
|
name = option.xpath('./@data-name')[0].title()
|
|
|
|
name = eval_xpath(option, './@data-name')[0].title()
|
|
|
|
supported_languages[code] = {"name": name}
|
|
|
|
supported_languages[code] = {"name": name}
|
|
|
|
|
|
|
|
|
|
|
|
return supported_languages
|
|
|
|
return supported_languages
|
|
|
|