Feature: language config (#27)

* Added language configuration support

Main page now has a dropdown for selecting preferred language of
results.

Refactored config to be its own model with language constants.

* Added more language support

Interface language is now updated using the "hl" arg

Fixed chinese traditional and simplified values

Updated decoding of characters to gb2312

* Updated to use conditional decoding dependent on language

* Updated filter to not rely on valid config to work properly

app/filter.py

@@ -19,7 +19,7 @@ class Filter:
         if config is None:
             config = {}
 
-        self.near = config['near'] if 'near' in config else None
+        self.near = config['near'] if 'near' in config else ''
         self.dark = config['dark'] if 'dark' in config else False
         self.nojs = config['nojs'] if 'nojs' in config else False
         self.mobile = mobile

app/models/config.py

@@ -0,0 +1,74 @@
+
+class Config:
+    # Derived from here:
+    # https://sites.google.com/site/tomihasa/google-language-codes#searchlanguage
+    LANGUAGES = [
+        {'name': 'English', 'value': 'lang_en'},
+        {'name': 'Afrikaans', 'value': 'lang_af'},
+        {'name': 'Arabic', 'value': 'lang_ar'},
+        {'name': 'Armenian', 'value': 'lang_hy'},
+        {'name': 'Belarusian', 'value': 'lang_be'},
+        {'name': 'Bulgarian', 'value': 'lang_bg'},
+        {'name': 'Catalan', 'value': 'lang_ca'},
+        {'name': 'Chinese (Simplified)', 'value': 'lang_zh-CN'},
+        {'name': 'Chinese (Traditional)', 'value': 'lang_zh-TW'},
+        {'name': 'Croatian', 'value': 'lang_hr'},
+        {'name': 'Czech', 'value': 'lang_cs'},
+        {'name': 'Danish', 'value': 'lang_da'},
+        {'name': 'Dutch', 'value': 'lang_nl'},
+        {'name': 'Esperanto', 'value': 'lang_eo'},
+        {'name': 'Estonian', 'value': 'lang_et'},
+        {'name': 'Filipino', 'value': 'lang_tl'},
+        {'name': 'Finnish', 'value': 'lang_fi'},
+        {'name': 'French', 'value': 'lang_fr'},
+        {'name': 'German', 'value': 'lang_de'},
+        {'name': 'Greek', 'value': 'lang_el'},
+        {'name': 'Hebrew', 'value': 'lang_iw'},
+        {'name': 'Hindi', 'value': 'lang_hi'},
+        {'name': 'Hungarian', 'value': 'lang_hu'},
+        {'name': 'Icelandic', 'value': 'lang_is'},
+        {'name': 'Indonesian',  'value': 'lang_id'},
+        {'name': 'Italian',  'value': 'lang_it'},
+        {'name': 'Japanese',  'value': 'lang_ja'},
+        {'name': 'Korean',  'value': 'lang_ko'},
+        {'name': 'Latvian',  'value': 'lang_lv'},
+        {'name': 'Lithuanian',  'value': 'lang_lt'},
+        {'name': 'Norwegian',  'value': 'lang_no'},
+        {'name': 'Persian',  'value': 'lang_fa'},
+        {'name': 'Polish', 'value': 'lang_pl'},
+        {'name': 'Portuguese', 'value': 'lang_pt'},
+        {'name': 'Romanian', 'value': 'lang_ro'},
+        {'name': 'Russian', 'value': 'lang_ru'},
+        {'name': 'Serbian', 'value': 'lang_sr'},
+        {'name': 'Slovak', 'value': 'lang_sk'},
+        {'name': 'Slovenian', 'value': 'lang_sl'},
+        {'name': 'Spanish', 'value': 'lang_es'},
+        {'name': 'Swahili', 'value': 'lang_sw'},
+        {'name': 'Swedish', 'value': 'lang_sv'},
+        {'name': 'Thai', 'value': 'lang_th'},
+        {'name': 'Turkish', 'value': 'lang_tr'},
+        {'name': 'Ukrainian', 'value': 'lang_uk'},
+        {'name': 'Vietnamese', 'value': 'lang_vi'},
+    ]
+
+    def __init__(self, **kwargs):
+        self.url = ''
+        self.lang = 'lang_en'
+        self.dark = False
+        self.nojs = False
+        self.near = ''
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    def __getitem__(self, name):
+        return getattr(self, name)
+
+    def __setitem__(self, name, value):
+        return setattr(self, name, value)
+
+    def __delitem__(self, name):
+        return delattr(self, name)
+
+    def __contains__(self, name):
+        return hasattr(self, name)

app/request.py

@@ -26,7 +26,7 @@ def gen_user_agent(normal_ua):
         return DESKTOP_UA.format(mozilla, linux, firefox)
 
 
-def gen_query(query, args, near_city=None):
+def gen_query(query, args, near_city=None, language='lang_en'):
     param_dict = {key: '' for key in VALID_PARAMS}
     # Use :past(hour/day/week/month/year) if available
     # example search "new restaurants :past month"
@@ -49,6 +49,9 @@ def gen_query(query, args, near_city=None):
     if near_city is not None:
         param_dict['near'] = '&near=' + urlparse.quote(near_city)
 
+    # Set language for results (lr) and interface (hl)
+    param_dict['lr'] = '&lr=' + language + '&hl=' + language.replace('lang_', '')
+
     for val in param_dict.values():
         if not val or val is None:
             continue
@@ -58,12 +61,19 @@ def gen_query(query, args, near_city=None):
 
 
 class Request:
-    def __init__(self, normal_ua):
+    def __init__(self, normal_ua, language='lang_en'):
         self.modified_user_agent = gen_user_agent(normal_ua)
+        self.language = language
 
     def __getitem__(self, name):
         return getattr(self, name)
 
+    def get_decode_value(self):
+        if 'lang_zh' in self.language:
+            return 'gb2312'
+        else:
+            return 'unicode-escape'
+
     def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
         response_header = []
 
@@ -80,4 +90,4 @@ class Request:
         if return_bytes:
             return b_obj.getvalue()
         else:
-            return b_obj.getvalue().decode('unicode-escape', 'ignore')
+            return b_obj.getvalue().decode(self.get_decode_value(), 'ignore')

app/routes.py

@@ -1,5 +1,6 @@
 from app import app
 from app.filter import Filter
+from app.models.config import Config
 from app.request import Request, gen_query
 import argparse
 from bs4 import BeautifulSoup
@@ -19,13 +20,14 @@ CONFIG_PATH = app.config['STATIC_FOLDER'] + '/config.json'
 
 @app.before_request
 def before_request_func():
-    g.user_request = Request(request.headers.get('User-Agent'))
-    g.user_config = json.load(open(CONFIG_PATH)) if os.path.exists(CONFIG_PATH) else {'url': request.url_root}
+    json_config = json.load(open(CONFIG_PATH)) if os.path.exists(CONFIG_PATH) else {'url': request.url_root}
+    g.user_config = Config(**json_config)
 
-    if 'url' not in g.user_config or not g.user_config['url']:
-        g.user_config['url'] = request.url_root
+    if not g.user_config.url:
+        g.user_config.url = request.url_root
 
-    g.app_location = g.user_config['url']
+    g.user_request = Request(request.headers.get('User-Agent'), language=g.user_config.lang)
+    g.app_location = g.user_config.url
 
 
 @app.errorhandler(404)
@@ -35,8 +37,12 @@ def unknown_page(e):
 
 @app.route('/', methods=['GET'])
 def index():
-    bg = '#000' if 'dark' in g.user_config and g.user_config['dark'] else '#fff'
-    return render_template('index.html', bg=bg, ua=g.user_request.modified_user_agent)
+    bg = '#000' if g.user_config.dark else '#fff'
+    return render_template('index.html',
+                           bg=bg,
+                           ua=g.user_request.modified_user_agent,
+                           languages=Config.LANGUAGES,
+                           current_lang=g.user_config.lang)
 
 
 @app.route('/opensearch.xml', methods=['GET'])
@@ -69,7 +75,7 @@ def search():
     mobile = 'Android' in user_agent or 'iPhone' in user_agent
 
     content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key)
-    full_query = gen_query(q, request_params, content_filter.near)
+    full_query = gen_query(q, request_params, content_filter.near, language=g.user_config.lang)
     get_body = g.user_request.send(query=full_query)
 
     results = content_filter.reskin(get_body)
@@ -81,7 +87,7 @@ def search():
 @app.route('/config', methods=['GET', 'POST'])
 def config():
     if request.method == 'GET':
-        return json.dumps(g.user_config)
+        return json.dumps(g.user_config.__dict__)
     else:
         config_data = request.form.to_dict()
         if 'url' not in config_data or not config_data['url']:

app/templates/index.html

@@ -41,6 +41,19 @@
                             <!-- TODO: Add option to regenerate user agent? -->
                             <span class="ua-span">User Agent: {{ ua }}</span>
                         </div>
+                        <div class="config-div">
+                            <label for="config-lang">Language: </label>
+                            <select name="lang" id="config-lang">
+                                {% for lang in languages %}
+                                <option value="{{ lang.value }}"
+                                    {% if lang.value in current_lang %}
+                                        selected
+                                    {% endif %}>
+                                    {{ lang.name }}
+                                </option>
+                                {% endfor %}
+                            </select>
+                        </div>
                         <div class="config-div">
                             <label for="config-near">Near: </label>
                             <input type="text" name="near" id="config-near" placeholder="City Name">