@ -1,16 +1,17 @@
import argparse
import base64
import html
import io
import json
import pickle
import urllib . parse as urlparse
import uuid
from datetime import timedelta
from functools import wraps
import waitress
from app import app
from app . models . config import Config
from app . models . endpoint import Endpoint
from app . request import Request , TorError
from app . utils . bangs import resolve_bang
from app . utils . misc import read_config_bool , get_client_ip
@ -22,6 +23,7 @@ from bs4 import BeautifulSoup as bsoup
from flask import jsonify , make_response , request , redirect , render_template , \
send_file , session , url_for
from requests import exceptions , get
from requests . models import PreparedRequest
# Load DDG bang json files only on init
bang_json = json . load ( open ( app . config [ ' BANG_FILE ' ] ) )
@ -57,23 +59,79 @@ def auth_required(f):
return decorated
def session_required ( f ) :
@wraps ( f )
def decorated ( * args , * * kwargs ) :
if ( valid_user_session ( session ) and
' cookies_disabled ' not in request . args ) :
g . session_key = session [ ' key ' ]
else :
session . pop ( ' _permanent ' , None )
g . session_key = app . default_key
# Clear out old sessions
invalid_sessions = [ ]
for user_session in os . listdir ( app . config [ ' SESSION_FILE_DIR ' ] ) :
session_path = os . path . join (
app . config [ ' SESSION_FILE_DIR ' ] ,
user_session )
try :
with open ( session_path , ' rb ' ) as session_file :
_ = pickle . load ( session_file )
data = pickle . load ( session_file )
if isinstance ( data , dict ) and ' valid ' in data :
continue
invalid_sessions . append ( session_path )
except ( EOFError , FileNotFoundError ) :
pass
for invalid_session in invalid_sessions :
os . remove ( invalid_session )
return f ( * args , * * kwargs )
return decorated
@app.before_request
def before_request_func ( ) :
g . request_params = (
request . args if request . method == ' GET ' else request . form
)
g . cookies_disabled = False
# Skip pre-request actions if verifying session
if ' /session ' in request . path and not valid_user_session ( session ) :
return
default_config = json . load ( open ( app . config [ ' DEFAULT_CONFIG ' ] ) ) \
if os . path . exists ( app . config [ ' DEFAULT_CONFIG ' ] ) else { }
# Generate session values for user if unavailable
if not valid_user_session ( session ) :
session [ ' config ' ] = json . load ( open ( app . config [ ' DEFAULT_CONFIG ' ] ) ) \
if os . path . exists ( app . config [ ' DEFAULT_CONFIG ' ] ) else { }
if ( not valid_user_session ( session ) and
' cookies_disabled ' not in request . args ) :
session [ ' config ' ] = default_config
session [ ' uuid ' ] = str ( uuid . uuid4 ( ) )
session [ ' key ' ] = generate_user_key ( True )
# Flag cookies as possibly disabled in order to prevent against
# unnecessary session directory expansion
g . cookies_disabled = True
session [ ' key ' ] = generate_user_key ( )
# Skip checking for session on /autocomplete searches,
# since they can be done from the browser search bar (aka
# no ability to initialize a session)
if not Endpoint . autocomplete . in_path ( request . path ) :
return redirect ( url_for (
' session_check ' ,
session_id = session [ ' uuid ' ] ,
follow = request . url ) , code = 307 )
else :
g . user_config = Config ( * * session [ ' config ' ] )
elif ' cookies_disabled ' not in request . args :
# Set session as permanent
session . permanent = True
app . permanent_session_lifetime = timedelta ( days = 365 )
g . user_config = Config ( * * session [ ' config ' ] )
else :
# User has cookies disabled, fall back to immutable default config
session . pop ( ' _permanent ' , None )
g . user_config = Config ( * * default_config )
# Handle https upgrade
if needs_https ( request . url ) :
@ -81,8 +139,6 @@ def before_request_func():
request . url . replace ( ' http:// ' , ' https:// ' , 1 ) ,
code = 308 )
g . user_config = Config ( * * session [ ' config ' ] )
if not g . user_config . url :
g . user_config . url = request . url_root . replace (
' http:// ' ,
@ -98,19 +154,6 @@ def before_request_func():
@app.after_request
def after_request_func ( resp ) :
# Check if address consistently has cookies blocked,
# in which case start removing session files after creation.
#
# Note: This is primarily done to prevent overpopulation of session
# directories, since browsers that block cookies will still trigger
# Flask's session creation routine with every request.
if g . cookies_disabled and request . remote_addr not in app . no_cookie_ips :
app . no_cookie_ips . append ( request . remote_addr )
elif g . cookies_disabled and request . remote_addr in app . no_cookie_ips :
session_list = list ( session . keys ( ) )
for key in session_list :
session . pop ( key )
resp . headers [ ' Content-Security-Policy ' ] = app . config [ ' CSP ' ]
if os . environ . get ( ' HTTPS_ONLY ' , False ) :
resp . headers [ ' Content-Security-Policy ' ] + = ' upgrade-insecure-requests '
@ -124,22 +167,28 @@ def unknown_page(e):
return redirect ( g . app_location )
@app.route ( ' / healthz' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . } ' , methods = [ ' GET ' ] )
def healthz ( ) :
return ' '
@app.route ( ' /home ' , methods = [ ' GET ' ] )
def home ( ) :
return redirect ( url_for ( ' .index ' ) )
@app.route ( f ' / { Endpoint . session } /<session_id> ' , methods = [ ' GET ' , ' PUT ' , ' POST ' ] )
def session_check ( session_id ) :
if ' uuid ' in session and session [ ' uuid ' ] == session_id :
session [ ' valid ' ] = True
return redirect ( request . args . get ( ' follow ' ) , code = 307 )
else :
follow_url = request . args . get ( ' follow ' )
req = PreparedRequest ( )
req . prepare_url ( follow_url , { ' cookies_disabled ' : 1 } )
session . pop ( ' _permanent ' , None )
return redirect ( req . url , code = 307 )
@app.route ( ' / ' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . home } ' , methods = [ ' GET ' ] )
@auth_required
def index ( ) :
# Reset keys
session [ ' key ' ] = generate_user_key ( g . cookies_disabled )
# Redirect if an error was raised
if ' error_message ' in session and session [ ' error_message ' ] :
error_message = session [ ' error_message ' ]
@ -157,13 +206,16 @@ def index():
logo = render_template (
' logo.html ' ,
dark = g . user_config . dark ) ,
config_disabled = app . config [ ' CONFIG_DISABLE ' ] ,
config_disabled = (
app . config [ ' CONFIG_DISABLE ' ] or
not valid_user_session ( session ) or
' cookies_disabled ' in request . args ) ,
config = g . user_config ,
tor_available = int ( os . environ . get ( ' TOR_AVAILABLE ' ) ) ,
version_number = app . config [ ' VERSION_NUMBER ' ] )
@app.route ( ' /opensearch.xml ' , methods = [ ' GET ' ] )
@app.route ( f' / { Endpoint . opensearch } ' , methods = [ ' GET ' ] )
def opensearch ( ) :
opensearch_url = g . app_location
if opensearch_url . endswith ( ' / ' ) :
@ -183,7 +235,7 @@ def opensearch():
) , 200 , { ' Content-Disposition ' : ' attachment; filename= " opensearch.xml " ' }
@app.route ( ' /search.html ' , methods = [ ' GET ' ] )
@app.route ( f' / { Endpoint . search_html } ' , methods = [ ' GET ' ] )
def search_html ( ) :
search_url = g . app_location
if search_url . endswith ( ' / ' ) :
@ -191,7 +243,7 @@ def search_html():
return render_template ( ' search.html ' , url = search_url )
@app.route ( ' / autocomplete' , methods = [ ' GET ' , ' POST ' ] )
@app.route ( f ' / { Endpoint . } ' , methods = [ ' GET ' , ' POST ' ] )
def autocomplete ( ) :
ac_var = ' WHOOGLE_AUTOCOMPLETE '
if os . getenv ( ac_var ) and not read_config_bool ( ac_var ) :
@ -224,14 +276,14 @@ def autocomplete():
] )
@app.route ( ' /search ' , methods = [ ' GET ' , ' POST ' ] )
@app.route ( f ' / { Endpoint . search } ' , methods = [ ' GET ' , ' POST ' ] )
@session_required
@auth_required
def search ( ) :
# Update user config if specified in search args
g . user_config = g . user_config . from_params ( g . request_params )
search_util = Search ( request , g . user_config , session ,
cookies_disabled = g . cookies_disabled )
search_util = Search ( request , g . user_config , g . session_key )
query = search_util . new_search_query ( )
bang = resolve_bang ( query = query , bangs_dict = bang_json )
@ -240,7 +292,7 @@ def search():
# Redirect to home if invalid/blank search
if not query :
return redirect ( ' / ' )
return redirect ( url_for ( ' .index ' ) )
# Generate response and number of external elements from the page
try :
@ -300,10 +352,13 @@ def search():
search_util . search_type else ' ' ) ) , resp_code
@app.route ( ' /config ' , methods = [ ' GET ' , ' POST ' , ' PUT ' ] )
@app.route ( f ' / { Endpoint . config } ' , methods = [ ' GET ' , ' POST ' , ' PUT ' ] )
@session_required
@auth_required
def config ( ) :
config_disabled = app . config [ ' CONFIG_DISABLE ' ]
config_disabled = (
app . config [ ' CONFIG_DISABLE ' ] or
not valid_user_session ( session ) )
if request . method == ' GET ' :
return json . dumps ( g . user_config . __dict__ )
elif request . method == ' PUT ' and not config_disabled :
@ -330,18 +385,14 @@ def config():
app . config [ ' CONFIG_PATH ' ] ,
request . args . get ( ' name ' ) ) , ' wb ' ) )
# Overwrite default config if user has cookies disabled
if g . cookies_disabled :
open ( app . config [ ' DEFAULT_CONFIG ' ] , ' w ' ) . write (
json . dumps ( config_data , indent = 4 ) )
session [ ' config ' ] = config_data
return redirect ( config_data [ ' url ' ] )
else :
return redirect ( url_for ( ' .index ' ) , code = 403 )
@app.route ( ' /url ' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . url } ' , methods = [ ' GET ' ] )
@session_required
@auth_required
def url ( ) :
if ' url ' in request . args :
@ -356,16 +407,18 @@ def url():
error_message = ' Unable to resolve query: ' + q )
@app.route ( ' /imgres ' )
@app.route ( f ' / { Endpoint . imgres } ' )
@session_required
@auth_required
def imgres ( ) :
return redirect ( request . args . get ( ' imgurl ' ) )
@app.route ( ' /element ' )
@app.route ( f ' / { Endpoint . element } ' )
@session_required
@auth_required
def element ( ) :
cipher_suite = Fernet ( session[ ' key ' ] )
cipher_suite = Fernet ( g. session_key )
src_url = cipher_suite . decrypt ( request . args . get ( ' url ' ) . encode ( ) ) . decode ( )
src_type = request . args . get ( ' type ' )
@ -384,7 +437,7 @@ def element():
return send_file ( io . BytesIO ( empty_gif ) , mimetype = ' image/gif ' )
@app.route ( ' / window' )
@app.route ( f ' / { Endpoint . } ' )
@auth_required
def window ( ) :
get_body = g . user_request . send ( base_url = request . args . get ( ' location ' ) ) . text
Ben Busby
2 years ago
committed by
GitHub