From a11ceb0a57124d5a6b4ac9a74b4cfd4bf8578cd0 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Tue, 12 May 2020 17:15:53 -0600 Subject: [PATCH] Feature: language config (#27) * Added language configuration support Main page now has a dropdown for selecting preferred language of results. Refactored config to be its own model with language constants. * Added more language support Interface language is now updated using the "hl" arg Fixed chinese traditional and simplified values Updated decoding of characters to gb2312 * Updated to use conditional decoding dependent on language * Updated filter to not rely on valid config to work properly --- app/filter.py | 2 +- app/models/__init__.py | 0 app/models/config.py | 74 ++++++++++++++++++++++++++++++++++++++++ app/request.py | 16 +++++++-- app/routes.py | 24 ++++++++----- app/templates/index.html | 13 +++++++ 6 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 app/models/__init__.py create mode 100644 app/models/config.py diff --git a/app/filter.py b/app/filter.py index 64e35e7..a77116c 100644 --- a/app/filter.py +++ b/app/filter.py @@ -19,7 +19,7 @@ class Filter: if config is None: config = {} - self.near = config['near'] if 'near' in config else None + self.near = config['near'] if 'near' in config else '' self.dark = config['dark'] if 'dark' in config else False self.nojs = config['nojs'] if 'nojs' in config else False self.mobile = mobile diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models/config.py b/app/models/config.py new file mode 100644 index 0000000..ee3c4ac --- /dev/null +++ b/app/models/config.py @@ -0,0 +1,74 @@ + +class Config: + # Derived from here: + # https://sites.google.com/site/tomihasa/google-language-codes#searchlanguage + LANGUAGES = [ + {'name': 'English', 'value': 'lang_en'}, + {'name': 'Afrikaans', 'value': 'lang_af'}, + {'name': 'Arabic', 'value': 'lang_ar'}, + {'name': 'Armenian', 'value': 'lang_hy'}, + {'name': 'Belarusian', 'value': 'lang_be'}, + {'name': 'Bulgarian', 'value': 'lang_bg'}, + {'name': 'Catalan', 'value': 'lang_ca'}, + {'name': 'Chinese (Simplified)', 'value': 'lang_zh-CN'}, + {'name': 'Chinese (Traditional)', 'value': 'lang_zh-TW'}, + {'name': 'Croatian', 'value': 'lang_hr'}, + {'name': 'Czech', 'value': 'lang_cs'}, + {'name': 'Danish', 'value': 'lang_da'}, + {'name': 'Dutch', 'value': 'lang_nl'}, + {'name': 'Esperanto', 'value': 'lang_eo'}, + {'name': 'Estonian', 'value': 'lang_et'}, + {'name': 'Filipino', 'value': 'lang_tl'}, + {'name': 'Finnish', 'value': 'lang_fi'}, + {'name': 'French', 'value': 'lang_fr'}, + {'name': 'German', 'value': 'lang_de'}, + {'name': 'Greek', 'value': 'lang_el'}, + {'name': 'Hebrew', 'value': 'lang_iw'}, + {'name': 'Hindi', 'value': 'lang_hi'}, + {'name': 'Hungarian', 'value': 'lang_hu'}, + {'name': 'Icelandic', 'value': 'lang_is'}, + {'name': 'Indonesian', 'value': 'lang_id'}, + {'name': 'Italian', 'value': 'lang_it'}, + {'name': 'Japanese', 'value': 'lang_ja'}, + {'name': 'Korean', 'value': 'lang_ko'}, + {'name': 'Latvian', 'value': 'lang_lv'}, + {'name': 'Lithuanian', 'value': 'lang_lt'}, + {'name': 'Norwegian', 'value': 'lang_no'}, + {'name': 'Persian', 'value': 'lang_fa'}, + {'name': 'Polish', 'value': 'lang_pl'}, + {'name': 'Portuguese', 'value': 'lang_pt'}, + {'name': 'Romanian', 'value': 'lang_ro'}, + {'name': 'Russian', 'value': 'lang_ru'}, + {'name': 'Serbian', 'value': 'lang_sr'}, + {'name': 'Slovak', 'value': 'lang_sk'}, + {'name': 'Slovenian', 'value': 'lang_sl'}, + {'name': 'Spanish', 'value': 'lang_es'}, + {'name': 'Swahili', 'value': 'lang_sw'}, + {'name': 'Swedish', 'value': 'lang_sv'}, + {'name': 'Thai', 'value': 'lang_th'}, + {'name': 'Turkish', 'value': 'lang_tr'}, + {'name': 'Ukrainian', 'value': 'lang_uk'}, + {'name': 'Vietnamese', 'value': 'lang_vi'}, + ] + + def __init__(self, **kwargs): + self.url = '' + self.lang = 'lang_en' + self.dark = False + self.nojs = False + self.near = '' + + for key, value in kwargs.items(): + setattr(self, key, value) + + def __getitem__(self, name): + return getattr(self, name) + + def __setitem__(self, name, value): + return setattr(self, name, value) + + def __delitem__(self, name): + return delattr(self, name) + + def __contains__(self, name): + return hasattr(self, name) \ No newline at end of file diff --git a/app/request.py b/app/request.py index aebc716..f50bfe4 100644 --- a/app/request.py +++ b/app/request.py @@ -26,7 +26,7 @@ def gen_user_agent(normal_ua): return DESKTOP_UA.format(mozilla, linux, firefox) -def gen_query(query, args, near_city=None): +def gen_query(query, args, near_city=None, language='lang_en'): param_dict = {key: '' for key in VALID_PARAMS} # Use :past(hour/day/week/month/year) if available # example search "new restaurants :past month" @@ -49,6 +49,9 @@ def gen_query(query, args, near_city=None): if near_city is not None: param_dict['near'] = '&near=' + urlparse.quote(near_city) + # Set language for results (lr) and interface (hl) + param_dict['lr'] = '&lr=' + language + '&hl=' + language.replace('lang_', '') + for val in param_dict.values(): if not val or val is None: continue @@ -58,12 +61,19 @@ def gen_query(query, args, near_city=None): class Request: - def __init__(self, normal_ua): + def __init__(self, normal_ua, language='lang_en'): self.modified_user_agent = gen_user_agent(normal_ua) + self.language = language def __getitem__(self, name): return getattr(self, name) + def get_decode_value(self): + if 'lang_zh' in self.language: + return 'gb2312' + else: + return 'unicode-escape' + def send(self, base_url=SEARCH_URL, query='', return_bytes=False): response_header = [] @@ -80,4 +90,4 @@ class Request: if return_bytes: return b_obj.getvalue() else: - return b_obj.getvalue().decode('unicode-escape', 'ignore') + return b_obj.getvalue().decode(self.get_decode_value(), 'ignore') diff --git a/app/routes.py b/app/routes.py index 7886de9..ba2394c 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,5 +1,6 @@ from app import app from app.filter import Filter +from app.models.config import Config from app.request import Request, gen_query import argparse from bs4 import BeautifulSoup @@ -19,13 +20,14 @@ CONFIG_PATH = app.config['STATIC_FOLDER'] + '/config.json' @app.before_request def before_request_func(): - g.user_request = Request(request.headers.get('User-Agent')) - g.user_config = json.load(open(CONFIG_PATH)) if os.path.exists(CONFIG_PATH) else {'url': request.url_root} + json_config = json.load(open(CONFIG_PATH)) if os.path.exists(CONFIG_PATH) else {'url': request.url_root} + g.user_config = Config(**json_config) - if 'url' not in g.user_config or not g.user_config['url']: - g.user_config['url'] = request.url_root + if not g.user_config.url: + g.user_config.url = request.url_root - g.app_location = g.user_config['url'] + g.user_request = Request(request.headers.get('User-Agent'), language=g.user_config.lang) + g.app_location = g.user_config.url @app.errorhandler(404) @@ -35,8 +37,12 @@ def unknown_page(e): @app.route('/', methods=['GET']) def index(): - bg = '#000' if 'dark' in g.user_config and g.user_config['dark'] else '#fff' - return render_template('index.html', bg=bg, ua=g.user_request.modified_user_agent) + bg = '#000' if g.user_config.dark else '#fff' + return render_template('index.html', + bg=bg, + ua=g.user_request.modified_user_agent, + languages=Config.LANGUAGES, + current_lang=g.user_config.lang) @app.route('/opensearch.xml', methods=['GET']) @@ -69,7 +75,7 @@ def search(): mobile = 'Android' in user_agent or 'iPhone' in user_agent content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key) - full_query = gen_query(q, request_params, content_filter.near) + full_query = gen_query(q, request_params, content_filter.near, language=g.user_config.lang) get_body = g.user_request.send(query=full_query) results = content_filter.reskin(get_body) @@ -81,7 +87,7 @@ def search(): @app.route('/config', methods=['GET', 'POST']) def config(): if request.method == 'GET': - return json.dumps(g.user_config) + return json.dumps(g.user_config.__dict__) else: config_data = request.form.to_dict() if 'url' not in config_data or not config_data['url']: diff --git a/app/templates/index.html b/app/templates/index.html index 8332a51..89c38d3 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -41,6 +41,19 @@ User Agent: {{ ua }} +
+ + +