searxng/searx/engines/swisscows.py

122 lines
3.7 KiB
Python
Raw Normal View History

"""
Swisscows (Web, Images)
@website https://swisscows.ch
@provide-api no
@using-api no
@results HTML (using search portal)
@stable no (HTML can change)
@parse url, title, content
"""
from json import loads
import re
from lxml.html import fromstring
2016-11-30 17:43:03 +00:00
from searx.url_utils import unquote, urlencode
from searx.utils import match_language
# engine dependent config
categories = ['general', 'images']
paging = True
language_support = True
# search-url
base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}'
supported_languages_url = base_url
# regex
2016-11-30 17:43:03 +00:00
regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile(b'^initialData:\s*')
regex_json_remove_end = re.compile(b',\s*environment$')
regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
# do search-request
def request(query, params):
region = match_language(params['language'], supported_languages, language_aliases)
ui_language = region.split('-')[0]
search_path = search_string.format(
2016-11-30 17:43:03 +00:00
query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
page=params['pageno']
)
# image search query is something like 'image?{query}&page={page}'
if params['category'] == 'images':
search_path = 'image' + search_path
params['url'] = base_url + search_path
return params
# get response from search-request
def response(resp):
results = []
2016-11-30 17:43:03 +00:00
json_regex = regex_json.search(resp.text)
# check if results are returned
if not json_regex:
return []
2016-11-30 17:43:03 +00:00
json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
json = loads(json_raw.decode('utf-8'))
# parse results
for result in json['Results'].get('items', []):
result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')
# parse image results
if result.get('ContentType', '').startswith('image'):
2016-11-30 17:43:03 +00:00
img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
# append result
results.append({'url': result['SourceUrl'],
'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
# parse general results
else:
result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')
# append result
results.append({'url': result_url,
'title': result_title,
'content': result_content})
# parse images
for result in json.get('Images', []):
# decode image url
2016-11-30 17:43:03 +00:00
img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
# append result
results.append({'url': result['SourceUrl'],
'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
# return results
return results
# get supported languages from their site
def _fetch_supported_languages(resp):
supported_languages = []
dom = fromstring(resp.text)
options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
for option in options:
code = option.xpath('./@data-search-language')[0]
if code.startswith('nb-'):
code = code.replace('nb', 'no', 1)
supported_languages.append(code)
return supported_languages