Project refactor (#85)
* Major refactor of requests and session management
- Switches from pycurl to requests library
- Allows for less janky decoding, especially with non-latin character
sets
- Adds session level management of user configs
- Allows for each session to set its own config (people are probably
going to complain about this, though not sure if it'll be the same
number of people who are upset that their friends/family have to share
their config)
- Updates key gen/regen to more aggressively swap out keys after each
request
* Added ability to save/load configs by name
- New PUT method for config allows changing config with specified name
- New methods in js controller to handle loading/saving of configs
* Result formatting and removal of unused elements
- Fixed question section formatting from results page (added appropriate
padding and made questions styled as italic)
- Removed user agent display from main config settings
* Minor change to button label
* Fixed issue with "de-pickling" of flask session
Having a gitignore-everything ("*") file within a flask session folder seems to cause a
weird bug where the state of the app becomes unusable from continuously
trying to prune files listed in the gitignore (and it can't prune '*').
* Switched to pickling saved configs
* Updated ad/sponsored content filter and conf naming
Configs are now named with a .conf extension to allow for easier manual
cleanup/modification of named config files
Sponsored content now removed by basic string matching of span content
* Version bump to 0.2.0
* Fixed request.send return style
pull/86/head
Ben Busby
4 years ago
committed by
GitHub
parent
71ba00785f
commit
b6fb4723f9
@ -1,12 +1,24 @@
|
|||||||
|
from app.utils.misc import generate_user_keys
|
||||||
from cryptography.fernet import Fernet
|
from cryptography.fernet import Fernet
|
||||||
from flask import Flask
|
from flask import Flask
|
||||||
|
from flask_session import Session
|
||||||
import os
|
import os
|
||||||
|
|
||||||
app = Flask(__name__, static_folder=os.path.dirname(os.path.abspath(__file__)) + '/static')
|
app = Flask(__name__, static_folder=os.path.dirname(os.path.abspath(__file__)) + '/static')
|
||||||
app.secret_key = Fernet.generate_key()
|
app.user_elements = {}
|
||||||
app.config['VERSION_NUMBER'] = '0.1.4'
|
app.config['SECRET_KEY'] = os.urandom(16)
|
||||||
|
app.config['SESSION_TYPE'] = 'filesystem'
|
||||||
|
app.config['VERSION_NUMBER'] = '0.2.0'
|
||||||
app.config['APP_ROOT'] = os.getenv('APP_ROOT', os.path.dirname(os.path.abspath(__file__)))
|
app.config['APP_ROOT'] = os.getenv('APP_ROOT', os.path.dirname(os.path.abspath(__file__)))
|
||||||
app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config['APP_ROOT'], 'static'))
|
app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config['APP_ROOT'], 'static'))
|
||||||
app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', app.config['STATIC_FOLDER']) + '/config.json'
|
app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', app.config['STATIC_FOLDER'] + '/config')
|
||||||
|
app.config['SESSION_FILE_DIR'] = app.config['CONFIG_PATH']
|
||||||
|
app.config['SESSION_COOKIE_SECURE'] = True
|
||||||
|
|
||||||
|
if not os.path.exists(app.config['CONFIG_PATH']):
|
||||||
|
os.makedirs(app.config['CONFIG_PATH'])
|
||||||
|
|
||||||
|
sess = Session()
|
||||||
|
sess.init_app(app)
|
||||||
|
|
||||||
from app import routes
|
from app import routes
|
||||||
|
|||||||
@ -0,0 +1,20 @@
|
|||||||
|
from cryptography.fernet import Fernet
|
||||||
|
|
||||||
|
SESSION_VALS = ['uuid', 'config', 'keys']
|
||||||
|
|
||||||
|
|
||||||
|
def generate_user_keys():
|
||||||
|
# Generate/regenerate unique key per user
|
||||||
|
return {
|
||||||
|
'element_key': Fernet.generate_key(),
|
||||||
|
'text_key': Fernet.generate_key()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def valid_user_session(session):
|
||||||
|
# Generate secret key for user if unavailable
|
||||||
|
for value in SESSION_VALS:
|
||||||
|
if value not in session:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
@ -0,0 +1,69 @@
|
|||||||
|
from app import app
|
||||||
|
from app.filter import Filter, get_first_link
|
||||||
|
from app.request import gen_query
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from cryptography.fernet import Fernet, InvalidToken
|
||||||
|
from flask import g
|
||||||
|
from typing import Any, Tuple
|
||||||
|
|
||||||
|
|
||||||
|
class RoutingUtils:
|
||||||
|
def __init__(self, request, config, session):
|
||||||
|
self.request_params = request.args if request.method == 'GET' else request.form
|
||||||
|
self.user_agent = request.headers.get('User-Agent')
|
||||||
|
self.feeling_lucky = False
|
||||||
|
self.config = config
|
||||||
|
self.session = session
|
||||||
|
self.query = ''
|
||||||
|
self.search_type = self.request_params.get('tbm') if 'tbm' in self.request_params else ''
|
||||||
|
|
||||||
|
def __getitem__(self, name):
|
||||||
|
return getattr(self, name)
|
||||||
|
|
||||||
|
def __setitem__(self, name, value):
|
||||||
|
return setattr(self, name, value)
|
||||||
|
|
||||||
|
def __delitem__(self, name):
|
||||||
|
return delattr(self, name)
|
||||||
|
|
||||||
|
def __contains__(self, name):
|
||||||
|
return hasattr(self, name)
|
||||||
|
|
||||||
|
def new_search_query(self) -> str:
|
||||||
|
app.user_elements[self.session['uuid']] = 0
|
||||||
|
self.session['keys']['element_key'] = Fernet.generate_key()
|
||||||
|
|
||||||
|
q = self.request_params.get('q')
|
||||||
|
|
||||||
|
if q is None or len(q) == 0:
|
||||||
|
return ''
|
||||||
|
else:
|
||||||
|
# Attempt to decrypt if this is an internal link
|
||||||
|
try:
|
||||||
|
q = Fernet(self.session['keys']['text_key']).decrypt(q.encode()).decode()
|
||||||
|
except InvalidToken:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Reset text key
|
||||||
|
self.session['keys']['text_key'] = Fernet.generate_key()
|
||||||
|
|
||||||
|
# Format depending on whether or not the query is a "feeling lucky" query
|
||||||
|
self.feeling_lucky = q.startswith('! ')
|
||||||
|
self.query = q[2:] if self.feeling_lucky else q
|
||||||
|
return self.query
|
||||||
|
|
||||||
|
def generate_response(self) -> Tuple[Any, int]:
|
||||||
|
mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent
|
||||||
|
|
||||||
|
content_filter = Filter(self.session['keys'], mobile=mobile, config=self.config)
|
||||||
|
full_query = gen_query(self.query, self.request_params, self.config, content_filter.near)
|
||||||
|
get_body = g.user_request.send(query=full_query).text
|
||||||
|
|
||||||
|
# Produce cleanable html soup from response
|
||||||
|
html_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser')
|
||||||
|
|
||||||
|
if self.feeling_lucky:
|
||||||
|
return get_first_link(html_soup), 1
|
||||||
|
else:
|
||||||
|
formatted_results = content_filter.clean(html_soup)
|
||||||
|
return formatted_results, content_filter.elements
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
from app.utils.misc import generate_user_keys, valid_user_session
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_user_keys():
|
||||||
|
keys = generate_user_keys()
|
||||||
|
assert 'text_key' in keys
|
||||||
|
assert 'element_key' in keys
|
||||||
|
assert keys['text_key'] not in keys['element_key']
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_session(client):
|
||||||
|
with client.session_transaction() as session:
|
||||||
|
assert not valid_user_session(session)
|
||||||
|
|
||||||
|
session['uuid'] = 'test'
|
||||||
|
session['keys'] = generate_user_keys()
|
||||||
|
session['config'] = {}
|
||||||
|
|
||||||
|
assert valid_user_session(session)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_key_generation(client):
|
||||||
|
text_key = ''
|
||||||
|
rv = client.get('/search?q=test+1')
|
||||||
|
assert rv._status_code == 200
|
||||||
|
|
||||||
|
with client.session_transaction() as session:
|
||||||
|
assert valid_user_session(session)
|
||||||
|
text_key = session['keys']['text_key']
|
||||||
|
|
||||||
|
rv = client.get('/search?q=test+2')
|
||||||
|
assert rv._status_code == 200
|
||||||
|
|
||||||
|
with client.session_transaction() as session:
|
||||||
|
assert valid_user_session(session)
|
||||||
|
assert text_key not in session['keys']['text_key']
|
||||||
Loading…
Reference in New Issue