@ -1,16 +1,17 @@
import argparse
import argparse
import base64
import base64
import html
import io
import io
import json
import json
import pickle
import pickle
import urllib . parse as urlparse
import urllib . parse as urlparse
import uuid
import uuid
from datetime import timedelta
from functools import wraps
from functools import wraps
import waitress
import waitress
from app import app
from app import app
from app . models . config import Config
from app . models . config import Config
from app . models . endpoint import Endpoint
from app . request import Request , TorError
from app . request import Request , TorError
from app . utils . bangs import resolve_bang
from app . utils . bangs import resolve_bang
from app . utils . misc import read_config_bool , get_client_ip
from app . utils . misc import read_config_bool , get_client_ip
@ -22,6 +23,7 @@ from bs4 import BeautifulSoup as bsoup
from flask import jsonify , make_response , request , redirect , render_template , \
from flask import jsonify , make_response , request , redirect , render_template , \
send_file , session , url_for
send_file , session , url_for
from requests import exceptions , get
from requests import exceptions , get
from requests . models import PreparedRequest
# Load DDG bang json files only on init
# Load DDG bang json files only on init
bang_json = json . load ( open ( app . config [ ' BANG_FILE ' ] ) )
bang_json = json . load ( open ( app . config [ ' BANG_FILE ' ] ) )
@ -57,23 +59,79 @@ def auth_required(f):
return decorated
return decorated
def session_required ( f ) :
@wraps ( f )
def decorated ( * args , * * kwargs ) :
if ( valid_user_session ( session ) and
' cookies_disabled ' not in request . args ) :
g . session_key = session [ ' key ' ]
else :
session . pop ( ' _permanent ' , None )
g . session_key = app . default_key
# Clear out old sessions
invalid_sessions = [ ]
for user_session in os . listdir ( app . config [ ' SESSION_FILE_DIR ' ] ) :
session_path = os . path . join (
app . config [ ' SESSION_FILE_DIR ' ] ,
user_session )
try :
with open ( session_path , ' rb ' ) as session_file :
_ = pickle . load ( session_file )
data = pickle . load ( session_file )
if isinstance ( data , dict ) and ' valid ' in data :
continue
invalid_sessions . append ( session_path )
except ( EOFError , FileNotFoundError ) :
pass
for invalid_session in invalid_sessions :
os . remove ( invalid_session )
return f ( * args , * * kwargs )
return decorated
@app.before_request
@app.before_request
def before_request_func ( ) :
def before_request_func ( ) :
g . request_params = (
g . request_params = (
request . args if request . method == ' GET ' else request . form
request . args if request . method == ' GET ' else request . form
)
)
g . cookies_disabled = False
# Skip pre-request actions if verifying session
if ' /session ' in request . path and not valid_user_session ( session ) :
return
default_config = json . load ( open ( app . config [ ' DEFAULT_CONFIG ' ] ) ) \
if os . path . exists ( app . config [ ' DEFAULT_CONFIG ' ] ) else { }
# Generate session values for user if unavailable
# Generate session values for user if unavailable
if not valid_user_session ( session ) :
if ( not valid_user_session ( session ) and
session [ ' config ' ] = json . load ( open ( app . config [ ' DEFAULT_CONFIG ' ] ) ) \
' cookies_disabled ' not in request . args ) :
if os . path . exists ( app . config [ ' DEFAULT_CONFIG ' ] ) else { }
session [ ' config ' ] = default_config
session [ ' uuid ' ] = str ( uuid . uuid4 ( ) )
session [ ' uuid ' ] = str ( uuid . uuid4 ( ) )
session [ ' key ' ] = generate_user_key ( True )
session [ ' key ' ] = generate_user_key ( )
# Flag cookies as possibly disabled in order to prevent against
# Skip checking for session on /autocomplete searches,
# unnecessary session directory expansion
# since they can be done from the browser search bar (aka
g . cookies_disabled = True
# no ability to initialize a session)
if not Endpoint . autocomplete . in_path ( request . path ) :
return redirect ( url_for (
' session_check ' ,
session_id = session [ ' uuid ' ] ,
follow = request . url ) , code = 307 )
else :
g . user_config = Config ( * * session [ ' config ' ] )
elif ' cookies_disabled ' not in request . args :
# Set session as permanent
session . permanent = True
app . permanent_session_lifetime = timedelta ( days = 365 )
g . user_config = Config ( * * session [ ' config ' ] )
else :
# User has cookies disabled, fall back to immutable default config
session . pop ( ' _permanent ' , None )
g . user_config = Config ( * * default_config )
# Handle https upgrade
# Handle https upgrade
if needs_https ( request . url ) :
if needs_https ( request . url ) :
@ -81,8 +139,6 @@ def before_request_func():
request . url . replace ( ' http:// ' , ' https:// ' , 1 ) ,
request . url . replace ( ' http:// ' , ' https:// ' , 1 ) ,
code = 308 )
code = 308 )
g . user_config = Config ( * * session [ ' config ' ] )
if not g . user_config . url :
if not g . user_config . url :
g . user_config . url = request . url_root . replace (
g . user_config . url = request . url_root . replace (
' http:// ' ,
' http:// ' ,
@ -98,19 +154,6 @@ def before_request_func():
@app.after_request
@app.after_request
def after_request_func ( resp ) :
def after_request_func ( resp ) :
# Check if address consistently has cookies blocked,
# in which case start removing session files after creation.
#
# Note: This is primarily done to prevent overpopulation of session
# directories, since browsers that block cookies will still trigger
# Flask's session creation routine with every request.
if g . cookies_disabled and request . remote_addr not in app . no_cookie_ips :
app . no_cookie_ips . append ( request . remote_addr )
elif g . cookies_disabled and request . remote_addr in app . no_cookie_ips :
session_list = list ( session . keys ( ) )
for key in session_list :
session . pop ( key )
resp . headers [ ' Content-Security-Policy ' ] = app . config [ ' CSP ' ]
resp . headers [ ' Content-Security-Policy ' ] = app . config [ ' CSP ' ]
if os . environ . get ( ' HTTPS_ONLY ' , False ) :
if os . environ . get ( ' HTTPS_ONLY ' , False ) :
resp . headers [ ' Content-Security-Policy ' ] + = ' upgrade-insecure-requests '
resp . headers [ ' Content-Security-Policy ' ] + = ' upgrade-insecure-requests '
@ -124,22 +167,28 @@ def unknown_page(e):
return redirect ( g . app_location )
return redirect ( g . app_location )
@app.route ( ' / healthz' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . healthz} ' , methods = [ ' GET ' ] )
def healthz ( ) :
def healthz ( ) :
return ' '
return ' '
@app.route ( ' /home ' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . session } /<session_id> ' , methods = [ ' GET ' , ' PUT ' , ' POST ' ] )
def home ( ) :
def session_check ( session_id ) :
return redirect ( url_for ( ' .index ' ) )
if ' uuid ' in session and session [ ' uuid ' ] == session_id :
session [ ' valid ' ] = True
return redirect ( request . args . get ( ' follow ' ) , code = 307 )
else :
follow_url = request . args . get ( ' follow ' )
req = PreparedRequest ( )
req . prepare_url ( follow_url , { ' cookies_disabled ' : 1 } )
session . pop ( ' _permanent ' , None )
return redirect ( req . url , code = 307 )
@app.route ( ' / ' , methods = [ ' GET ' ] )
@app.route ( ' / ' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . home } ' , methods = [ ' GET ' ] )
@auth_required
@auth_required
def index ( ) :
def index ( ) :
# Reset keys
session [ ' key ' ] = generate_user_key ( g . cookies_disabled )
# Redirect if an error was raised
# Redirect if an error was raised
if ' error_message ' in session and session [ ' error_message ' ] :
if ' error_message ' in session and session [ ' error_message ' ] :
error_message = session [ ' error_message ' ]
error_message = session [ ' error_message ' ]
@ -157,13 +206,16 @@ def index():
logo = render_template (
logo = render_template (
' logo.html ' ,
' logo.html ' ,
dark = g . user_config . dark ) ,
dark = g . user_config . dark ) ,
config_disabled = app . config [ ' CONFIG_DISABLE ' ] ,
config_disabled = (
app . config [ ' CONFIG_DISABLE ' ] or
not valid_user_session ( session ) or
' cookies_disabled ' in request . args ) ,
config = g . user_config ,
config = g . user_config ,
tor_available = int ( os . environ . get ( ' TOR_AVAILABLE ' ) ) ,
tor_available = int ( os . environ . get ( ' TOR_AVAILABLE ' ) ) ,
version_number = app . config [ ' VERSION_NUMBER ' ] )
version_number = app . config [ ' VERSION_NUMBER ' ] )
@app.route ( ' /opensearch.xml ' , methods = [ ' GET ' ] )
@app.route ( f' / { Endpoint . opensearch } ' , methods = [ ' GET ' ] )
def opensearch ( ) :
def opensearch ( ) :
opensearch_url = g . app_location
opensearch_url = g . app_location
if opensearch_url . endswith ( ' / ' ) :
if opensearch_url . endswith ( ' / ' ) :
@ -183,7 +235,7 @@ def opensearch():
) , 200 , { ' Content-Disposition ' : ' attachment; filename= " opensearch.xml " ' }
) , 200 , { ' Content-Disposition ' : ' attachment; filename= " opensearch.xml " ' }
@app.route ( ' /search.html ' , methods = [ ' GET ' ] )
@app.route ( f' / { Endpoint . search_html } ' , methods = [ ' GET ' ] )
def search_html ( ) :
def search_html ( ) :
search_url = g . app_location
search_url = g . app_location
if search_url . endswith ( ' / ' ) :
if search_url . endswith ( ' / ' ) :
@ -191,7 +243,7 @@ def search_html():
return render_template ( ' search.html ' , url = search_url )
return render_template ( ' search.html ' , url = search_url )
@app.route ( ' / autocomplete' , methods = [ ' GET ' , ' POST ' ] )
@app.route ( f ' / { Endpoint . autocomplete} ' , methods = [ ' GET ' , ' POST ' ] )
def autocomplete ( ) :
def autocomplete ( ) :
ac_var = ' WHOOGLE_AUTOCOMPLETE '
ac_var = ' WHOOGLE_AUTOCOMPLETE '
if os . getenv ( ac_var ) and not read_config_bool ( ac_var ) :
if os . getenv ( ac_var ) and not read_config_bool ( ac_var ) :
@ -224,14 +276,14 @@ def autocomplete():
] )
] )
@app.route ( ' /search ' , methods = [ ' GET ' , ' POST ' ] )
@app.route ( f ' / { Endpoint . search } ' , methods = [ ' GET ' , ' POST ' ] )
@session_required
@auth_required
@auth_required
def search ( ) :
def search ( ) :
# Update user config if specified in search args
# Update user config if specified in search args
g . user_config = g . user_config . from_params ( g . request_params )
g . user_config = g . user_config . from_params ( g . request_params )
search_util = Search ( request , g . user_config , session ,
search_util = Search ( request , g . user_config , g . session_key )
cookies_disabled = g . cookies_disabled )
query = search_util . new_search_query ( )
query = search_util . new_search_query ( )
bang = resolve_bang ( query = query , bangs_dict = bang_json )
bang = resolve_bang ( query = query , bangs_dict = bang_json )
@ -240,7 +292,7 @@ def search():
# Redirect to home if invalid/blank search
# Redirect to home if invalid/blank search
if not query :
if not query :
return redirect ( ' / ' )
return redirect ( url_for ( ' .index ' ) )
# Generate response and number of external elements from the page
# Generate response and number of external elements from the page
try :
try :
@ -300,10 +352,13 @@ def search():
search_util . search_type else ' ' ) ) , resp_code
search_util . search_type else ' ' ) ) , resp_code
@app.route ( ' /config ' , methods = [ ' GET ' , ' POST ' , ' PUT ' ] )
@app.route ( f ' / { Endpoint . config } ' , methods = [ ' GET ' , ' POST ' , ' PUT ' ] )
@session_required
@auth_required
@auth_required
def config ( ) :
def config ( ) :
config_disabled = app . config [ ' CONFIG_DISABLE ' ]
config_disabled = (
app . config [ ' CONFIG_DISABLE ' ] or
not valid_user_session ( session ) )
if request . method == ' GET ' :
if request . method == ' GET ' :
return json . dumps ( g . user_config . __dict__ )
return json . dumps ( g . user_config . __dict__ )
elif request . method == ' PUT ' and not config_disabled :
elif request . method == ' PUT ' and not config_disabled :
@ -330,18 +385,14 @@ def config():
app . config [ ' CONFIG_PATH ' ] ,
app . config [ ' CONFIG_PATH ' ] ,
request . args . get ( ' name ' ) ) , ' wb ' ) )
request . args . get ( ' name ' ) ) , ' wb ' ) )
# Overwrite default config if user has cookies disabled
if g . cookies_disabled :
open ( app . config [ ' DEFAULT_CONFIG ' ] , ' w ' ) . write (
json . dumps ( config_data , indent = 4 ) )
session [ ' config ' ] = config_data
session [ ' config ' ] = config_data
return redirect ( config_data [ ' url ' ] )
return redirect ( config_data [ ' url ' ] )
else :
else :
return redirect ( url_for ( ' .index ' ) , code = 403 )
return redirect ( url_for ( ' .index ' ) , code = 403 )
@app.route ( ' /url ' , methods = [ ' GET ' ] )
@app.route ( f ' / { Endpoint . url } ' , methods = [ ' GET ' ] )
@session_required
@auth_required
@auth_required
def url ( ) :
def url ( ) :
if ' url ' in request . args :
if ' url ' in request . args :
@ -356,16 +407,18 @@ def url():
error_message = ' Unable to resolve query: ' + q )
error_message = ' Unable to resolve query: ' + q )
@app.route ( ' /imgres ' )
@app.route ( f ' / { Endpoint . imgres } ' )
@session_required
@auth_required
@auth_required
def imgres ( ) :
def imgres ( ) :
return redirect ( request . args . get ( ' imgurl ' ) )
return redirect ( request . args . get ( ' imgurl ' ) )
@app.route ( ' /element ' )
@app.route ( f ' / { Endpoint . element } ' )
@session_required
@auth_required
@auth_required
def element ( ) :
def element ( ) :
cipher_suite = Fernet ( session[ ' key ' ] )
cipher_suite = Fernet ( g. session_key )
src_url = cipher_suite . decrypt ( request . args . get ( ' url ' ) . encode ( ) ) . decode ( )
src_url = cipher_suite . decrypt ( request . args . get ( ' url ' ) . encode ( ) ) . decode ( )
src_type = request . args . get ( ' type ' )
src_type = request . args . get ( ' type ' )
@ -384,7 +437,7 @@ def element():
return send_file ( io . BytesIO ( empty_gif ) , mimetype = ' image/gif ' )
return send_file ( io . BytesIO ( empty_gif ) , mimetype = ' image/gif ' )
@app.route ( ' / window' )
@app.route ( f ' / { Endpoint . window} ' )
@auth_required
@auth_required
def window ( ) :
def window ( ) :
get_body = g . user_request . send ( base_url = request . args . get ( ' location ' ) ) . text
get_body = g . user_request . send ( base_url = request . args . get ( ' location ' ) ) . text