From 41800835f9ea82f4b5342525df1c8174eddb9efc Mon Sep 17 00:00:00 2001 From: Marc Abonce Seguin Date: Mon, 14 Sep 2020 00:06:58 -0700 Subject: [PATCH] fetch supported languages for startpage engine --- searx/engines/startpage.py | 69 ++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 953734934..b6d0c24f7 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -14,9 +14,12 @@ from lxml import html from dateutil import parser from datetime import datetime, timedelta import re +from unicodedata import normalize, combining +from babel import Locale +from babel.localedata import locale_identifiers from searx.engines.xpath import extract_text from searx.languages import language_codes -from searx.utils import eval_xpath +from searx.utils import eval_xpath, match_language # engine dependent config categories = ['general'] @@ -26,6 +29,7 @@ categories = ['general'] paging = True language_support = True +supported_languages_url = 'https://www.startpage.com/do/settings' # search-url base_url = 'https://startpage.com/' @@ -54,12 +58,11 @@ def request(query, params): # set language if specified if params['language'] != 'all': - language = 'english' - for lc, _, _, lang in language_codes: - if lc == params['language']: - language = lang - params['data']['language'] = language - params['data']['lui'] = language + lang_code = match_language(params['language'], supported_languages, fallback=None) + if lang_code: + language_name = supported_languages[lang_code]['alias'] + params['data']['language'] = language_name + params['data']['lui'] = language_name return params @@ -132,3 +135,55 @@ def response(resp): # return results return results + + +# get supported languages from their site +def _fetch_supported_languages(resp): + # startpage's language selector is a mess + # each option has a displayed name and a value, either of which may represent the language name + # in the native script, the language name in English, an English transliteration of the native name, + # the English name of the writing script used by the language, or occasionally something else entirely. + + # this cases are so special they need to be hardcoded, a couple of them are mispellings + language_names = { + 'english_uk': 'en-GB', + 'fantizhengwen': ['zh-TW', 'zh-HK'], + 'hangul': 'ko', + 'malayam': 'ml', + 'norsk': 'nb', + 'sinhalese': 'si', + 'sudanese': 'su' + } + + # get the English name of every language known by babel + language_names.update({name.lower(): lang_code for lang_code, name in Locale('en')._data['languages'].items()}) + + # get the native name of every language known by babel + for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()): + native_name = Locale(lang_code).get_language_name().lower() + # add native name exactly as it is + language_names[native_name] = lang_code + + # add "normalized" language name (i.e. français becomes francais and español becomes espanol) + unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name))) + if len(unaccented_name) == len(unaccented_name.encode()): + # add only if result is ascii (otherwise "normalization" didn't work) + language_names[unaccented_name] = lang_code + + dom = html.fromstring(resp.text) + sp_lang_names = [] + for option in dom.xpath('//form[@id="settings-form"]//select[@name="language"]/option'): + sp_lang_names.append((option.get('value'), extract_text(option).lower())) + + supported_languages = {} + for sp_option_value, sp_option_text in sp_lang_names: + lang_code = language_names.get(sp_option_value) or language_names.get(sp_option_text) + if isinstance(lang_code, str): + supported_languages[lang_code] = {'alias': sp_option_value} + elif isinstance(lang_code, list): + for lc in lang_code: + supported_languages[lc] = {'alias': sp_option_value} + else: + print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text)) + + return supported_languages