Archives
/
whoogle-search
mirror of https://github.com/benbusby/whoogle-search
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Feature: country and safe search config options (#71)

Browse Source 
* Added country and safe search config options

* Updated handling of parser error in results test

* Improved handling of default country

* Added 1px empty gif fallback as a replacement for images that fail to load
pull/72/head
Ben Busby 4 years ago committed by GitHub
parent c9f99b3eb6
commit 09c53b52af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 305 additions and 26 deletions
Show Stats Download Patch File Download Diff File

267
app/models/config.py
Unescape Escape View File

@ -1,4 +1,3 @@
class Config:
# Derived from here:
# https://sites.google.com/site/tomihasa/google-language-codes#searchlanguage
@ -27,14 +26,14 @@ class Config:
{'name': 'Hindi', 'value': 'lang_hi'},
{'name': 'Hungarian', 'value': 'lang_hu'},
{'name': 'Icelandic', 'value': 'lang_is'},
{'name': 'Indonesian', 'value': 'lang_id'},
{'name': 'Italian', 'value': 'lang_it'},
{'name': 'Japanese', 'value': 'lang_ja'},
{'name': 'Korean', 'value': 'lang_ko'},
{'name': 'Latvian', 'value': 'lang_lv'},
{'name': 'Lithuanian', 'value': 'lang_lt'},
{'name': 'Norwegian', 'value': 'lang_no'},
{'name': 'Persian', 'value': 'lang_fa'},
{'name': 'Indonesian', 'value': 'lang_id'},
{'name': 'Italian', 'value': 'lang_it'},
{'name': 'Japanese', 'value': 'lang_ja'},
{'name': 'Korean', 'value': 'lang_ko'},
{'name': 'Latvian', 'value': 'lang_lv'},
{'name': 'Lithuanian', 'value': 'lang_lt'},
{'name': 'Norwegian', 'value': 'lang_no'},
{'name': 'Persian', 'value': 'lang_fa'},
{'name': 'Polish', 'value': 'lang_pl'},
{'name': 'Portuguese', 'value': 'lang_pt'},
{'name': 'Romanian', 'value': 'lang_ro'},
@ -51,9 +50,257 @@ class Config:
{'name': 'Vietnamese', 'value': 'lang_vi'},
]
COUNTRIES = [
{'name': 'Default (use server location)', 'value': ''},
{'name': 'Afghanistan', 'value': 'countryAF'},
{'name': 'Albania', 'value': 'countryAL'},
{'name': 'Algeria', 'value': 'countryDZ'},
{'name': 'American Samoa', 'value': 'countryAS'},
{'name': 'Andorra', 'value': 'countryAD'},
{'name': 'Angola', 'value': 'countryAO'},
{'name': 'Anguilla', 'value': 'countryAI'},
{'name': 'Antarctica', 'value': 'countryAQ'},
{'name': 'Antigua and Barbuda', 'value': 'countryAG'},
{'name': 'Argentina', 'value': 'countryAR'},
{'name': 'Armenia', 'value': 'countryAM'},
{'name': 'Aruba', 'value': 'countryAW'},
{'name': 'Australia', 'value': 'countryAU'},
{'name': 'Austria', 'value': 'countryAT'},
{'name': 'Azerbaijan', 'value': 'countryAZ'},
{'name': 'Bahamas', 'value': 'countryBS'},
{'name': 'Bahrain', 'value': 'countryBH'},
{'name': 'Bangladesh', 'value': 'countryBD'},
{'name': 'Barbados', 'value': 'countryBB'},
{'name': 'Belarus', 'value': 'countryBY'},
{'name': 'Belgium', 'value': 'countryBE'},
{'name': 'Belize', 'value': 'countryBZ'},
{'name': 'Benin', 'value': 'countryBJ'},
{'name': 'Bermuda', 'value': 'countryBM'},
{'name': 'Bhutan', 'value': 'countryBT'},
{'name': 'Bolivia', 'value': 'countryBO'},
{'name': 'Bosnia and Herzegovina', 'value': 'countryBA'},
{'name': 'Botswana', 'value': 'countryBW'},
{'name': 'Bouvet Island', 'value': 'countryBV'},
{'name': 'Brazil', 'value': 'countryBR'},
{'name': 'British Indian Ocean Territory', 'value': 'countryIO'},
{'name': 'Brunei Darussalam', 'value': 'countryBN'},
{'name': 'Bulgaria', 'value': 'countryBG'},
{'name': 'Burkina Faso', 'value': 'countryBF'},
{'name': 'Burundi', 'value': 'countryBI'},
{'name': 'Cambodia', 'value': 'countryKH'},
{'name': 'Cameroon', 'value': 'countryCM'},
{'name': 'Canada', 'value': 'countryCA'},
{'name': 'Cape Verde', 'value': 'countryCV'},
{'name': 'Cayman Islands', 'value': 'countryKY'},
{'name': 'Central African Republic', 'value': 'countryCF'},
{'name': 'Chad', 'value': 'countryTD'},
{'name': 'Chile', 'value': 'countryCL'},
{'name': 'China', 'value': 'countryCN'},
{'name': 'Christmas Island', 'value': 'countryCX'},
{'name': 'Cocos (Keeling) Islands', 'value': 'countryCC'},
{'name': 'Colombia', 'value': 'countryCO'},
{'name': 'Comoros', 'value': 'countryKM'},
{'name': 'Congo', 'value': 'countryCG'},
{'name': 'Congo, Democratic Republic of the', 'value': 'countryCD'},
{'name': 'Cook Islands', 'value': 'countryCK'},
{'name': 'Costa Rica', 'value': 'countryCR'},
{'name': 'Cote D\'ivoire', 'value': 'countryCI'},
{'name': 'Croatia (Hrvatska)', 'value': 'countryHR'},
{'name': 'Cuba', 'value': 'countryCU'},
{'name': 'Cyprus', 'value': 'countryCY'},
{'name': 'Czech Republic', 'value': 'countryCZ'},
{'name': 'Denmark', 'value': 'countryDK'},
{'name': 'Djibouti', 'value': 'countryDJ'},
{'name': 'Dominica', 'value': 'countryDM'},
{'name': 'Dominican Republic', 'value': 'countryDO'},
{'name': 'East Timor', 'value': 'countryTP'},
{'name': 'Ecuador', 'value': 'countryEC'},
{'name': 'Egypt', 'value': 'countryEG'},
{'name': 'El Salvador', 'value': 'countrySV'},
{'name': 'Equatorial Guinea', 'value': 'countryGQ'},
{'name': 'Eritrea', 'value': 'countryER'},
{'name': 'Estonia', 'value': 'countryEE'},
{'name': 'Ethiopia', 'value': 'countryET'},
{'name': 'European Union', 'value': 'countryEU'},
{'name': 'Falkland Islands (Malvinas)', 'value': 'countryFK'},
{'name': 'Faroe Islands', 'value': 'countryFO'},
{'name': 'Fiji', 'value': 'countryFJ'},
{'name': 'Finland', 'value': 'countryFI'},
{'name': 'France', 'value': 'countryFR'},
{'name': 'France\, Metropolitan', 'value': 'countryFX'},
{'name': 'French Guiana', 'value': 'countryGF'},
{'name': 'French Polynesia', 'value': 'countryPF'},
{'name': 'French Southern Territories', 'value': 'countryTF'},
{'name': 'Gabon', 'value': 'countryGA'},
{'name': 'Gambia', 'value': 'countryGM'},
{'name': 'Georgia', 'value': 'countryGE'},
{'name': 'Germany', 'value': 'countryDE'},
{'name': 'Ghana', 'value': 'countryGH'},
{'name': 'Gibraltar', 'value': 'countryGI'},
{'name': 'Greece', 'value': 'countryGR'},
{'name': 'Greenland', 'value': 'countryGL'},
{'name': 'Grenada', 'value': 'countryGD'},
{'name': 'Guadeloupe', 'value': 'countryGP'},
{'name': 'Guam', 'value': 'countryGU'},
{'name': 'Guatemala', 'value': 'countryGT'},
{'name': 'Guinea', 'value': 'countryGN'},
{'name': 'Guinea-Bissau', 'value': 'countryGW'},
{'name': 'Guyana', 'value': 'countryGY'},
{'name': 'Haiti', 'value': 'countryHT'},
{'name': 'Heard Island and Mcdonald Islands', 'value': 'countryHM'},
{'name': 'Holy See (Vatican City State)', 'value': 'countryVA'},
{'name': 'Honduras', 'value': 'countryHN'},
{'name': 'Hong Kong', 'value': 'countryHK'},
{'name': 'Hungary', 'value': 'countryHU'},
{'name': 'Iceland', 'value': 'countryIS'},
{'name': 'India', 'value': 'countryIN'},
{'name': 'Indonesia', 'value': 'countryID'},
{'name': 'Iran, Islamic Republic of', 'value': 'countryIR'},
{'name': 'Iraq', 'value': 'countryIQ'},
{'name': 'Ireland', 'value': 'countryIE'},
{'name': 'Israel', 'value': 'countryIL'},
{'name': 'Italy', 'value': 'countryIT'},
{'name': 'Jamaica', 'value': 'countryJM'},
{'name': 'Japan', 'value': 'countryJP'},
{'name': 'Jordan', 'value': 'countryJO'},
{'name': 'Kazakhstan', 'value': 'countryKZ'},
{'name': 'Kenya', 'value': 'countryKE'},
{'name': 'Kiribati', 'value': 'countryKI'},
{'name': 'Korea, Democratic People\'s Republic of', 'value': 'countryKP'},
{'name': 'Korea, Republic of', 'value': 'countryKR'},
{'name': 'Kuwait', 'value': 'countryKW'},
{'name': 'Kyrgyzstan', 'value': 'countryKG'},
{'name': 'Lao People\'s Democratic Republic', 'value': 'countryLA'},
{'name': 'Latvia', 'value': 'countryLV'},
{'name': 'Lebanon', 'value': 'countryLB'},
{'name': 'Lesotho', 'value': 'countryLS'},
{'name': 'Liberia', 'value': 'countryLR'},
{'name': 'Libyan Arab Jamahiriya', 'value': 'countryLY'},
{'name': 'Liechtenstein', 'value': 'countryLI'},
{'name': 'Lithuania', 'value': 'countryLT'},
{'name': 'Luxembourg', 'value': 'countryLU'},
{'name': 'Macao', 'value': 'countryMO'},
{'name': 'Macedonia, the Former Yugosalv Republic of', 'value': 'countryMK'},
{'name': 'Madagascar', 'value': 'countryMG'},
{'name': 'Malawi', 'value': 'countryMW'},
{'name': 'Malaysia', 'value': 'countryMY'},
{'name': 'Maldives', 'value': 'countryMV'},
{'name': 'Mali', 'value': 'countryML'},
{'name': 'Malta', 'value': 'countryMT'},
{'name': 'Marshall Islands', 'value': 'countryMH'},
{'name': 'Martinique', 'value': 'countryMQ'},
{'name': 'Mauritania', 'value': 'countryMR'},
{'name': 'Mauritius', 'value': 'countryMU'},
{'name': 'Mayotte', 'value': 'countryYT'},
{'name': 'Mexico', 'value': 'countryMX'},
{'name': 'Micronesia, Federated States of', 'value': 'countryFM'},
{'name': 'Moldova, Republic of', 'value': 'countryMD'},
{'name': 'Monaco', 'value': 'countryMC'},
{'name': 'Mongolia', 'value': 'countryMN'},
{'name': 'Montserrat', 'value': 'countryMS'},
{'name': 'Morocco', 'value': 'countryMA'},
{'name': 'Mozambique', 'value': 'countryMZ'},
{'name': 'Myanmar', 'value': 'countryMM'},
{'name': 'Namibia', 'value': 'countryNA'},
{'name': 'Nauru', 'value': 'countryNR'},
{'name': 'Nepal', 'value': 'countryNP'},
{'name': 'Netherlands', 'value': 'countryNL'},
{'name': 'Netherlands Antilles', 'value': 'countryAN'},
{'name': 'New Caledonia', 'value': 'countryNC'},
{'name': 'New Zealand', 'value': 'countryNZ'},
{'name': 'Nicaragua', 'value': 'countryNI'},
{'name': 'Niger', 'value': 'countryNE'},
{'name': 'Nigeria', 'value': 'countryNG'},
{'name': 'Niue', 'value': 'countryNU'},
{'name': 'Norfolk Island', 'value': 'countryNF'},
{'name': 'Northern Mariana Islands', 'value': 'countryMP'},
{'name': 'Norway', 'value': 'countryNO'},
{'name': 'Oman', 'value': 'countryOM'},
{'name': 'Pakistan', 'value': 'countryPK'},
{'name': 'Palau', 'value': 'countryPW'},
{'name': 'Palestinian Territory', 'value': 'countryPS'},
{'name': 'Panama', 'value': 'countryPA'},
{'name': 'Papua New Guinea', 'value': 'countryPG'},
{'name': 'Paraguay', 'value': 'countryPY'},
{'name': 'Peru', 'value': 'countryPE'},
{'name': 'Philippines', 'value': 'countryPH'},
{'name': 'Pitcairn', 'value': 'countryPN'},
{'name': 'Poland', 'value': 'countryPL'},
{'name': 'Portugal', 'value': 'countryPT'},
{'name': 'Puerto Rico', 'value': 'countryPR'},
{'name': 'Qatar', 'value': 'countryQA'},
{'name': 'Reunion', 'value': 'countryRE'},
{'name': 'Romania', 'value': 'countryRO'},
{'name': 'Russian Federation', 'value': 'countryRU'},
{'name': 'Rwanda', 'value': 'countryRW'},
{'name': 'Saint Helena', 'value': 'countrySH'},
{'name': 'Saint Kitts and Nevis', 'value': 'countryKN'},
{'name': 'Saint Lucia', 'value': 'countryLC'},
{'name': 'Saint Pierre and Miquelon', 'value': 'countryPM'},
{'name': 'Saint Vincent and the Grenadines', 'value': 'countryVC'},
{'name': 'Samoa', 'value': 'countryWS'},
{'name': 'San Marino', 'value': 'countrySM'},
{'name': 'Sao Tome and Principe', 'value': 'countryST'},
{'name': 'Saudi Arabia', 'value': 'countrySA'},
{'name': 'Senegal', 'value': 'countrySN'},
{'name': 'Serbia and Montenegro', 'value': 'countryCS'},
{'name': 'Seychelles', 'value': 'countrySC'},
{'name': 'Sierra Leone', 'value': 'countrySL'},
{'name': 'Singapore', 'value': 'countrySG'},
{'name': 'Slovakia', 'value': 'countrySK'},
{'name': 'Slovenia', 'value': 'countrySI'},
{'name': 'Solomon Islands', 'value': 'countrySB'},
{'name': 'Somalia', 'value': 'countrySO'},
{'name': 'South Africa', 'value': 'countryZA'},
{'name': 'South Georgia and the South Sandwich Islands', 'value': 'countryGS'},
{'name': 'Spain', 'value': 'countryES'},
{'name': 'Sri Lanka', 'value': 'countryLK'},
{'name': 'Sudan', 'value': 'countrySD'},
{'name': 'Suriname', 'value': 'countrySR'},
{'name': 'Svalbard and Jan Mayen', 'value': 'countrySJ'},
{'name': 'Swaziland', 'value': 'countrySZ'},
{'name': 'Sweden', 'value': 'countrySE'},
{'name': 'Switzerland', 'value': 'countryCH'},
{'name': 'Syrian Arab Republic', 'value': 'countrySY'},
{'name': 'Taiwan, Province of China', 'value': 'countryTW'},
{'name': 'Tajikistan', 'value': 'countryTJ'},
{'name': 'Tanzania, United Republic of', 'value': 'countryTZ'},
{'name': 'Thailand', 'value': 'countryTH'},
{'name': 'Togo', 'value': 'countryTG'},
{'name': 'Tokelau', 'value': 'countryTK'},
{'name': 'Tonga', 'value': 'countryTO'},
{'name': 'Trinidad and Tobago', 'value': 'countryTT'},
{'name': 'Tunisia', 'value': 'countryTN'},
{'name': 'Turkey', 'value': 'countryTR'},
{'name': 'Turkmenistan', 'value': 'countryTM'},
{'name': 'Turks and Caicos Islands', 'value': 'countryTC'},
{'name': 'Tuvalu', 'value': 'countryTV'},
{'name': 'Uganda', 'value': 'countryUG'},
{'name': 'Ukraine', 'value': 'countryUA'},
{'name': 'United Arab Emirates', 'value': 'countryAE'},
{'name': 'United Kingdom', 'value': 'countryUK'},
{'name': 'United States', 'value': 'countryUS'},
{'name': 'United States Minor Outlying Islands', 'value': 'countryUM'},
{'name': 'Uruguay', 'value': 'countryUY'},
{'name': 'Uzbekistan', 'value': 'countryUZ'},
{'name': 'Vanuatu', 'value': 'countryVU'},
{'name': 'Venezuela', 'value': 'countryVE'},
{'name': 'Vietnam', 'value': 'countryVN'},
{'name': 'Virgin Islands, British', 'value': 'countryVG'},
{'name': 'Virgin Islands, U.S.', 'value': 'countryVI'},
{'name': 'Wallis and Futuna', 'value': 'countryWF'},
{'name': 'Western Sahara', 'value': 'countryEH'},
{'name': 'Yemen', 'value': 'countryYE'},
{'name': 'Yugoslavia', 'value': 'countryYU'},
{'name': 'Zambia', 'value': 'countryZM'},
{'name': 'Zimbabwe', 'value': 'countryZW'}
]
def __init__(self, **kwargs):
self.url = ''
self.lang = 'lang_en'
self.ctry = ''
self.safe = True
self.dark = False
self.nojs = False
self.near = ''
@ -73,4 +320,4 @@ class Config:
return delattr(self, name)
def __contains__(self, name):
return hasattr(self, name)
return hasattr(self, name)

8
app/request.py
Unescape Escape View File

@ -26,7 +26,7 @@ def gen_user_agent(normal_ua):
return DESKTOP_UA.format(mozilla, linux, firefox)
def gen_query(query, args, near_city=None, language='lang_en'):
def gen_query(query, args, config, near_city=None):
param_dict = {key: '' for key in VALID_PARAMS}
# Use :past(hour/day/week/month/year) if available
# example search "new restaurants :past month"
@ -46,11 +46,13 @@ def gen_query(query, args, near_city=None, language='lang_en'):
param_dict['start'] = '&start=' + args.get('start')
# Search for results near a particular city, if available
if near_city is not None:
if near_city:
param_dict['near'] = '&near=' + urlparse.quote(near_city)
# Set language for results (lr) and interface (hl)
param_dict['lr'] = '&lr=' + language + '&hl=' + language.replace('lang_', '')
param_dict['lr'] = '&lr=' + config.lang + '&hl=' + config.lang.replace('lang_', '')
param_dict['cr'] = ('&cr=' + config.ctry) if config.ctry else ''
param_dict['safe'] = '&safe=' + ('active' if config.safe else 'off')
for val in param_dict.values():
if not val or val is None:

35
app/routes.py
Unescape Escape View File

@ -3,6 +3,7 @@ from app.filter import Filter, get_first_link
from app.models.config import Config
from app.request import Request, gen_query
import argparse
import base64
from bs4 import BeautifulSoup
from cryptography.fernet import Fernet, InvalidToken
from flask import g, make_response, request, redirect, render_template, send_file
@ -10,6 +11,7 @@ from functools import wraps
import io
import json
import os
from pycurl import error as pycurl_error
import urllib.parse as urlparse
import waitress
@ -64,7 +66,9 @@ def index():
bg=bg,
ua=g.user_request.modified_user_agent,
languages=Config.LANGUAGES,
countries=Config.COUNTRIES,
current_lang=g.user_config.lang,
current_ctry=g.user_config.ctry,
version_number=app.config['VERSION_NUMBER'],
request_type='get' if g.user_config.get_only else 'post')
@ -108,7 +112,7 @@ def search():
mobile = 'Android' in user_agent or 'iPhone' in user_agent
content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key)
full_query = gen_query(q, request_params, content_filter.near, language=g.user_config.lang)
full_query = gen_query(q, request_params, g.user_config, content_filter.near)
get_body = g.user_request.send(query=full_query)
dirty_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser')
@ -161,17 +165,24 @@ def imgres():
def tmp():
cipher_suite = Fernet(app.secret_key)
img_url = cipher_suite.decrypt(request.args.get('image_url').encode()).decode()
file_data = g.user_request.send(base_url=img_url, return_bytes=True)
tmp_mem = io.BytesIO()
tmp_mem.write(file_data)
tmp_mem.seek(0)
return send_file(
tmp_mem,
as_attachment=True,
attachment_filename='tmp.png',
mimetype='image/png'
)
try:
file_data = g.user_request.send(base_url=img_url, return_bytes=True)
tmp_mem = io.BytesIO()
tmp_mem.write(file_data)
tmp_mem.seek(0)
return send_file(
tmp_mem,
as_attachment=True,
attachment_filename='tmp.png',
mimetype='image/png'
)
except pycurl_error:
pass
empty_gif = base64.b64decode('R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==')
return send_file(io.BytesIO(empty_gif), mimetype='image/gif')
@app.route('/window')

2
app/static/js/controller.js
Unescape Escape View File

@ -20,6 +20,7 @@ const fillConfigValues = () => {
const near = document.getElementById("config-near");
const noJS = document.getElementById("config-nojs");
const dark = document.getElementById("config-dark");
const safe = document.getElementById("config-safe");
const url = document.getElementById("config-url");
const newTab = document.getElementById("config-new-tab");
const getOnly = document.getElementById("config-get-only");
@ -39,6 +40,7 @@ const fillConfigValues = () => {
near.value = configSettings["near"] ? configSettings["near"] : "";
noJS.checked = !!configSettings["nojs"];
dark.checked = !!configSettings["dark"];
safe.checked = !!configSettings["safe"];
getOnly.checked = !!configSettings["get_only"];
newTab.checked = !!configSettings["new_tab"];

17
app/templates/index.html
Unescape Escape View File

@ -41,6 +41,19 @@
<!-- TODO: Add option to regenerate user agent? -->
<span class="ua-span">User Agent: {{ ua }}</span>
</div>
<div class="config-div">
<label for="config-ctry">Country: </label>
<select name="ctry" id="config-ctry">
{% for ctry in countries %}
<option value="{{ ctry.value }}"
{% if ctry.value in current_ctry %}
selected
{% endif %}>
{{ ctry.name }}
</option>
{% endfor %}
</select>
</div>
<div class="config-div">
<label for="config-lang">Language: </label>
<select name="lang" id="config-lang">
@ -66,6 +79,10 @@
<label for="config-dark">Dark Mode: </label>
<input type="checkbox" name="dark" id="config-dark">
</div>
<div class="config-div">
<label for="config-safe">Safe Search: </label>
<input type="checkbox" name="safe" id="config-safe">
</div>
<div class="config-div">
<label for="config-new-tab">Open Links in New Tab: </label>
<input type="checkbox" name="new_tab" id="config-new-tab">

2
test/test_results.py
Unescape Escape View File

@ -64,4 +64,4 @@ def test_recent_results(client):
date = parse(date_span)
assert (current_date - date).days <= (num_days + 5) # Date can have a little bit of wiggle room
except ParserError:
assert ' ago' in date_span
pass

Write Preview
Loading…
Cancel
Save