From 0c0ebb8917d73ec3bbfa6274b5e141fac56cc222 Mon Sep 17 00:00:00 2001
From: Ben Busby <ben.busby7@gmail.com>
Date: Tue, 28 Apr 2020 18:19:34 -0600
Subject: [PATCH] Added POST search, encrypted query strings, refactoring

The implementation of POST search support comes with a few benefits. The
most apparent is the avoidance of search queries appearing in web server
logs -- instead of the prior GET approach (i.e.
/search?q=my+search+query), using POST requests with the query stored in
the request body creates logs that simply appear as "/search".

Since a lot of relative links are generated in the results page, I came
up with a way to generate a unique key at run time that is used to
encrypt any query strings before sending to the user. This benefits both
regular text queries as well as fetching of image links and means that
web logs will only show an encrypted string where a link or query
string might slip through.

Unfortunately, GET search requests still need to be supported, as it
doesn't seem that Firefox (on iOS) supports loading search engines by
their opensearch.xml file, but instead relies on manual entry of a
search query string. Once this is updated, I'll probably remove GET
request search support.
---
 README.md                    |   5 +-
 app/__init__.py              |   2 +
 app/filter.py                | 225 +++++++++++++++++++----------------
 app/request.py               |   3 +
 app/routes.py                |  25 ++--
 app/static/css/main.css      |   3 +-
 app/static/js/controller.js  |   4 -
 app/templates/index.html     |  10 +-
 app/templates/opensearch.xml |   4 +-
 9 files changed, 158 insertions(+), 123 deletions(-)

diff --git a/README.md b/README.md
index 341166d..616ccc4 100644
--- a/README.md
+++ b/README.md
@@ -63,7 +63,7 @@ pip install -r requirements.txt
 ./run
 ```
 
-### C) Manual Setup (Docker) 
+### C) Manual Setup (Docker)
 1. Ensure the Docker daemon is running, and is accessible by your user account
   - To add user permissions, you can execute `sudo usermod -aG docker yourusername`
   - Running `docker ps` should return something besides an error. If you encounter an error saying the daemon isn't running, try `sudo systemctl start docker` (Linux) or ensure the docker tool is running (Windows/macOS).
@@ -102,10 +102,11 @@ To filter by a range of time, append ":past <time>" to the end of your search, w
 Update browser settings:
   - Firefox (Desktop)
     - Navigate to your app's url, and click the 3 dot menu in the address bar. At the bottom, there should be an option to "Add Search Engine". Once you've clicked this, open your Firefox Preferences menu, click "Search" in the left menu, and use the available dropdown to select "Shoogle" from the list.
-  - Firefox (Mobile)
+  - Firefox (iOS)
     - In the mobile app Settings page, tap "Search" within the "General" section. There should be an option titled "Add Search Engine" to select. It should prompt you to enter a title and search query url - use the following elements to fill out the form:
       - Title: "Shoogle"
       - URL: "https://\<your shoogle url\>/search?q=%s"
+  - Others (TODO)
 
 ### Customizing and Configuration
 Shoogle currently allows a few minor configuration settings, accessible from the home page:
diff --git a/app/__init__.py b/app/__init__.py
index 6f344fd..82b63a3 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,6 +1,8 @@
+from cryptography.fernet import Fernet
 from flask import Flask
 import os
 
 app = Flask(__name__, static_folder=os.path.dirname(os.path.abspath(__file__)) + '/static')
+app.secret_key = Fernet.generate_key()
 
 from app import routes
diff --git a/app/filter.py b/app/filter.py
index 6708368..5f3f1e3 100644
--- a/app/filter.py
+++ b/app/filter.py
@@ -1,4 +1,6 @@
+from app.request import VALID_PARAMS
 from bs4 import BeautifulSoup
+from cryptography.fernet import Fernet
 import re
 import urllib.parse as urlparse
 from urllib.parse import parse_qs
@@ -7,7 +9,7 @@ SKIP_ARGS = ['ref_src', 'utm']
 
 
 class Filter:
-    def __init__(self, mobile=False, config=None):
+    def __init__(self, mobile=False, config=None, secret_key=''):
         if config is None:
             config = {}
 
@@ -15,6 +17,7 @@ class Filter:
         self.dark = config['dark'] if 'dark' in config else False
         self.nojs = config['nojs'] if 'nojs' in config else False
         self.mobile = mobile
+        self.secret_key = secret_key
 
     def __getitem__(self, name):
         return getattr(self, name)
@@ -30,104 +33,124 @@ class Filter:
         return page
 
     def clean(self, soup):
-        def remove_ads():
-            main_divs = soup.find('div', {'id': 'main'})
-            if main_divs is None:
-                return
-
-            result_divs = main_divs.findAll('div', recursive=False)
-
-            # Only ads/sponsored content use classes in the list of result divs
-            ad_divs = [ad_div for ad_div in result_divs if 'class' in ad_div.attrs]
-            for div in ad_divs:
-                div.decompose()
-
-        def sync_images():
-            for img in soup.find_all('img'):
-                if img['src'].startswith('//'):
-                    img['src'] = 'https:' + img['src']
-
-                img['src'] = '/tmp?image_url=' + img['src']
-
-        def update_styling():
-            # Remove unnecessary button(s)
-            for button in soup.find_all('button'):
-                button.decompose()
-
-            # Remove svg logos
-            for svg in soup.find_all('svg'):
-                svg.decompose()
-
-            # Update logo
-            logo = soup.find('a', {'class': 'l'})
-            if logo and self.mobile:
-                logo['style'] = 'display:flex; justify-content:center; align-items:center; color:#685e79; ' \
-                                'font-size:18px; '
-
-            # Fix search bar length on mobile
-            try:
-                search_bar = soup.find('header').find('form').find('div')
-                search_bar['style'] = 'width: 100%;'
-            except AttributeError:
-                pass
-
-            # Set up dark mode if active
-            if self.dark:
-                soup.find('html')['style'] = 'scrollbar-color: #333 #111;'
-                for input_element in soup.findAll('input'):
-                    input_element['style'] = 'color:#fff;'
-
-        def update_links():
-            # Replace hrefs with only the intended destination (no "utm" type tags)
-            for a in soup.find_all('a', href=True):
-                href = a['href']
-                if '/advanced_search' in href:
-                    a.decompose()
-                    continue
-
-                if 'url?q=' in href:
-                    # Strip unneeded arguments
-                    result_link = urlparse.urlparse(href)
-                    result_link = parse_qs(result_link.query)['q'][0]
-
-                    parsed_link = urlparse.urlparse(result_link)
-                    link_args = parse_qs(parsed_link.query)
-                    safe_args = {}
-
-                    for arg in link_args.keys():
-                        if arg in SKIP_ARGS:
-                            continue
-
-                        safe_args[arg] = link_args[arg]
-
-                    # Remove original link query and replace with filtered args
-                    result_link = result_link.replace(parsed_link.query, '')
-                    if len(safe_args) > 1:
-                        result_link = result_link + urlparse.urlencode(safe_args)
-                    else:
-                        result_link = result_link.replace('?', '')
-
-                    a['href'] = result_link
-
-                    # Add no-js option
-                    if self.nojs:
-                        nojs_link = soup.new_tag('a')
-                        nojs_link['href'] = '/window?location=' + result_link
-                        nojs_link['style'] = 'display:block;width:100%;'
-                        nojs_link.string = 'NoJS Link: ' + nojs_link['href']
-                        a.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
-                        a.append(nojs_link)
-
-            # Ensure no extra scripts passed through
-            try:
-                for script in soup('script'):
-                    script.decompose()
-                soup.find('div', id='sfooter').decompose()
-            except Exception:
-                pass
-
-        remove_ads()
-        sync_images()
-        update_styling()
-        update_links()
+        self.remove_ads(soup)
+        self.sync_images(soup)
+        self.update_styling(soup)
+        self.update_links(soup)
+
+        input_form = soup.find('form')
+        input_form['method'] = 'POST'
+
         return soup
+
+    def remove_ads(self, soup):
+        main_divs = soup.find('div', {'id': 'main'})
+        if main_divs is None:
+            return
+
+        result_divs = main_divs.findAll('div', recursive=False)
+
+        # Only ads/sponsored content use classes in the list of result divs
+        ad_divs = [ad_div for ad_div in result_divs if 'class' in ad_div.attrs]
+        for div in ad_divs:
+            div.decompose()
+
+    def sync_images(self, soup):
+        for img in [_ for _ in soup.find_all('img') if 'src' in _]:
+            img_src = img['src']
+            if img_src.startswith('//'):
+                img_src = 'https:' + img_src
+
+            enc_src = Fernet(self.secret_key).encrypt(img_src.encode())
+            img['src'] = '/tmp?image_url=' + enc_src.decode()
+
+    def update_styling(self, soup):
+        # Remove unnecessary button(s)
+        for button in soup.find_all('button'):
+            button.decompose()
+
+        # Remove svg logos
+        for svg in soup.find_all('svg'):
+            svg.decompose()
+
+        # Update logo
+        logo = soup.find('a', {'class': 'l'})
+        if logo and self.mobile:
+            logo['style'] = 'display:flex; justify-content:center; align-items:center; color:#685e79; ' \
+                            'font-size:18px; '
+
+        # Fix search bar length on mobile
+        try:
+            search_bar = soup.find('header').find('form').find('div')
+            search_bar['style'] = 'width: 100%;'
+        except AttributeError:
+            pass
+
+        # Set up dark mode if active
+        if self.dark:
+            soup.find('html')['style'] = 'scrollbar-color: #333 #111;'
+            for input_element in soup.findAll('input'):
+                input_element['style'] = 'color:#fff;'
+
+    def update_links(self, soup):
+        # Replace hrefs with only the intended destination (no "utm" type tags)
+        for a in soup.find_all('a', href=True):
+            href = a['href']
+            if '/advanced_search' in href:
+                a.decompose()
+                continue
+
+            if '?q=' not in href:
+                continue
+
+            result_link = urlparse.urlparse(href)
+            query_link = parse_qs(result_link.query)['q'][0]
+
+            if '/search?q=' in href:
+                enc_result = Fernet(self.secret_key).encrypt(query_link.encode())
+                new_search = '/search?q=' + enc_result.decode()
+
+                for param in VALID_PARAMS:
+                    if param in parse_qs(result_link.query):
+                        new_search += '&' + param + '=' + parse_qs(result_link.query)[param][0]
+                a['href'] = new_search
+                continue
+
+            if 'url?q=' in href:
+                # Strip unneeded arguments
+                parsed_link = urlparse.urlparse(query_link)
+                link_args = parse_qs(parsed_link.query)
+                safe_args = {}
+
+                for arg in link_args.keys():
+                    if arg in SKIP_ARGS:
+                        continue
+
+                    safe_args[arg] = link_args[arg]
+
+                # Remove original link query and replace with filtered args
+                query_link = query_link.replace(parsed_link.query, '')
+                if len(safe_args) > 1:
+                    query_link = query_link + urlparse.urlencode(safe_args)
+                else:
+                    query_link = query_link.replace('?', '')
+
+                a['href'] = query_link
+
+                # Add no-js option
+                if self.nojs:
+                    nojs_link = soup.new_tag('a')
+                    nojs_link['href'] = '/window?location=' + query_link
+                    nojs_link['style'] = 'display:block;width:100%;'
+                    nojs_link.string = 'NoJS Link: ' + nojs_link['href']
+                    a.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
+                    a.append(nojs_link)
+
+        # Ensure no extra scripts passed through
+        try:
+            for script in soup('script'):
+                script.decompose()
+            soup.find('div', id='sfooter').decompose()
+        except Exception:
+            pass
+
diff --git a/app/request.py b/app/request.py
index bb3be7a..c02cb00 100644
--- a/app/request.py
+++ b/app/request.py
@@ -9,6 +9,9 @@ SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
 MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
 DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
 
+# Valid query params
+VALID_PARAMS = ['tbs', 'tbm', 'start', 'near']
+
 
 def gen_user_agent(normal_ua):
     is_mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
diff --git a/app/routes.py b/app/routes.py
index db66656..cfafd6d 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -2,6 +2,7 @@ from app import app
 from app.filter import Filter
 from app.request import Request, gen_query
 from bs4 import BeautifulSoup
+from cryptography.fernet import Fernet, InvalidToken
 from flask import g, make_response, request, redirect, render_template, send_file
 import io
 import json
@@ -19,11 +20,6 @@ def before_request_func():
     g.user_request = Request(request.headers.get('User-Agent'))
 
 
-# @app.after_request
-# def after_request(response):
-#     return response
-
-
 @app.route('/', methods=['GET'])
 def index():
     bg = '#000' if 'dark' in user_config and user_config['dark'] else '#fff'
@@ -42,16 +38,25 @@ def opensearch():
     return response
 
 
-@app.route('/search', methods=['GET'])
+@app.route('/search', methods=['GET', 'POST'])
 def search():
-    q = request.args.get('q')
+    q = None
+    if request.method == 'GET':
+        q = request.args.get('q')
+        try:
+            q = Fernet(app.secret_key).decrypt(q.encode()).decode()
+        except InvalidToken:
+            pass
+    else:
+        q = request.form['q']
+
     if q is None or len(q) <= 0:
         return render_template('error.html')
 
     user_agent = request.headers.get('User-Agent')
     mobile = 'Android' in user_agent or 'iPhone' in user_agent
 
-    content_filter = Filter(mobile, user_config)
+    content_filter = Filter(mobile, user_config, secret_key=app.secret_key)
     full_query = gen_query(q, request.args, content_filter.near)
     get_body = g.user_request.send(query=full_query)
 
@@ -95,7 +100,9 @@ def imgres():
 
 @app.route('/tmp')
 def tmp():
-    file_data = g.user_request.send(base_url=request.args.get('image_url'), return_bytes=True)
+    cipher_suite = Fernet(app.secret_key)
+    img_url = cipher_suite.decrypt(request.args.get('image_url').encode()).decode()
+    file_data = g.user_request.send(base_url=img_url, return_bytes=True)
     tmp_mem = io.BytesIO()
     tmp_mem.write(file_data)
     tmp_mem.seek(0)
diff --git a/app/static/css/main.css b/app/static/css/main.css
index f019a15..98f3ed8 100644
--- a/app/static/css/main.css
+++ b/app/static/css/main.css
@@ -57,6 +57,7 @@ body {
     margin: auto;
     border-radius: 0 0 10px 10px;
     max-width: 600px;
+    -webkit-appearance: none;
 }
 
 .config-div {
@@ -117,4 +118,4 @@ button::-moz-focus-inner {
     color: white;
     -webkit-box-decoration-break: clone;
     box-decoration-break: clone;
-}
\ No newline at end of file
+}
diff --git a/app/static/js/controller.js b/app/static/js/controller.js
index 094cf67..bc28231 100644
--- a/app/static/js/controller.js
+++ b/app/static/js/controller.js
@@ -18,10 +18,6 @@ document.addEventListener("DOMContentLoaded", function() {
         }
     });
 
-    searchBtn.onclick = function() {
-        window.location.href = '/search?q=' + encodeURI(searchBar.value);
-    };
-
     // Setup shoogle config
     const collapsible = document.getElementById("config-collapsible");
     collapsible.addEventListener("click", function() {
diff --git a/app/templates/index.html b/app/templates/index.html
index e8a0fdb..915af1a 100644
--- a/app/templates/index.html
+++ b/app/templates/index.html
@@ -25,10 +25,12 @@
     <body id="main" style="display: none; background-color: {{ bg }}">
         <div class="search-container">
             <img class="logo" src="/static/img/logo.png">
-            <div class="search-fields">
-                <input type="text" id="search-bar">
-                <button type="submit" id="search-submit">Search</button>
-            </div>
+            <form action="/search" method="post">
+                <div class="search-fields">
+                    <input type="text" name="q" id="search-bar">
+                    <input type="submit" id="search-submit" value="Submit">
+                </div>
+            </form>
             <br/>
             <button id="config-collapsible" class="collapsible">Configuration</button>
             <div class="content">
diff --git a/app/templates/opensearch.xml b/app/templates/opensearch.xml
index 250c998..6c9b22c 100644
--- a/app/templates/opensearch.xml
+++ b/app/templates/opensearch.xml
@@ -1,10 +1,10 @@
 <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"
                        xmlns:moz="http://www.mozilla.org/2006/browser/search/">
-  <ShortName>Shoogle</ShortName>
+  <ShortName>ShoogleTEST</ShortName>
   <Description>Shoogle: A lightweight, deployable Google search proxy for desktop/mobile that removes Javascript, AMP links, and ads</Description>
   <InputEncoding>UTF-8</InputEncoding>
   <Image width="32" height="32" type="image/x-icon">/static/img/favicon/favicon-32x32.png</Image>
-  <Url type="text/html" template="{{ shoogle_url }}/search">
+  <Url type="text/html" method="post" template="{{ shoogle_url }}/search">
     <Param name="q" value="{searchTerms}"/>
   </Url>
   <Url type="application/x-suggestions+json" template="{{ shoogle_url }}/search"/>