searxng/searx/webapp.py

576 lines
19 KiB
Python
Raw Normal View History

2013-10-14 21:09:13 +00:00
#!/usr/bin/env python
2013-10-14 22:33:18 +00:00
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
2014-02-14 15:16:20 +00:00
if __name__ == '__main__':
from sys import path
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__))+'/../'))
2014-01-21 20:28:54 +00:00
import json
import cStringIO
import os
2014-03-14 08:55:04 +00:00
from datetime import datetime, timedelta
2014-03-08 13:00:20 +00:00
from itertools import chain
2014-02-05 19:24:31 +00:00
from flask import (
Flask, request, render_template, url_for, Response, make_response,
redirect, send_from_directory
)
2014-03-15 19:20:41 +00:00
from flask.ext.babel import Babel, gettext, format_date
2014-01-30 18:02:23 +00:00
from searx import settings, searx_dir
2014-02-05 19:24:31 +00:00
from searx.engines import (
2014-07-07 11:59:27 +00:00
categories, engines, get_engines_stats, engine_shortcuts
2014-02-05 19:24:31 +00:00
)
2014-04-24 23:46:40 +00:00
from searx.utils import (
UnicodeWriter, highlight_content, html_to_text, get_themes
)
2014-11-18 10:37:42 +00:00
from searx.version import VERSION_STRING
2014-06-24 14:30:04 +00:00
from searx.https_rewrite import https_rules
2014-01-31 03:35:23 +00:00
from searx.languages import language_codes
from searx.search import Search
from searx.query import Query
2014-03-29 15:29:19 +00:00
from searx.autocomplete import backends as autocomplete_backends
2013-12-01 22:52:49 +00:00
from urlparse import urlparse
import re
2013-10-14 21:09:13 +00:00
2014-06-24 14:30:04 +00:00
static_path, templates_path, themes =\
get_themes(settings['themes_path']
if settings.get('themes_path')
else searx_dir)
2014-10-09 17:26:02 +00:00
default_theme = settings['server'].get('default_theme', 'default')
2014-04-24 23:46:40 +00:00
app = Flask(
__name__,
2014-04-24 23:46:40 +00:00
static_folder=static_path,
template_folder=templates_path
)
app.secret_key = settings['server']['secret_key']
2014-01-14 17:17:19 +00:00
babel = Babel(app)
2014-01-19 22:04:09 +00:00
#TODO configurable via settings.yml
2014-10-03 11:16:30 +00:00
favicons = ['wikipedia', 'youtube', 'vimeo', 'dailymotion', 'soundcloud',
'twitter', 'stackoverflow', 'github', 'deviantart']
2014-12-13 20:37:28 +00:00
global_favicons = []
for indice,theme in enumerate(themes):
global_favicons.append([])
theme_img_path = searx_dir+"/static/"+theme+"/img/"
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
global_favicons[indice].extend(filenames)
2014-01-19 22:04:09 +00:00
cookie_max_age = 60 * 60 * 24 * 365 * 23 # 23 years
2014-01-19 22:04:09 +00:00
@babel.localeselector
def get_locale():
2014-01-21 23:59:18 +00:00
locale = request.accept_languages.best_match(settings['locales'].keys())
if request.cookies.get('locale', '') in settings['locales']:
locale = request.cookies.get('locale', '')
if 'locale' in request.args\
and request.args['locale'] in settings['locales']:
locale = request.args['locale']
if 'locale' in request.form\
and request.form['locale'] in settings['locales']:
locale = request.form['locale']
return locale
2014-01-14 17:17:19 +00:00
def get_base_url():
if settings['server']['base_url']:
hostname = settings['server']['base_url']
2014-01-14 17:17:19 +00:00
else:
scheme = 'http'
if request.is_secure:
scheme = 'https'
hostname = url_for('index', _external=True, _scheme=scheme)
return hostname
2014-04-24 23:46:40 +00:00
def get_current_theme_name(override=None):
"""Returns theme name.
Checks in this order:
1. override
2. cookies
3. settings"""
if override and override in themes:
return override
2014-09-02 22:57:09 +00:00
theme_name = request.args.get('theme',
request.cookies.get('theme',
default_theme))
2014-04-24 23:46:40 +00:00
if theme_name not in themes:
theme_name = default_theme
return theme_name
def url_for_theme(endpoint, override_theme=None, **values):
if endpoint == 'static' and values.get('filename', None):
theme_name = get_current_theme_name(override=override_theme)
values['filename'] = "{}/{}".format(theme_name, values['filename'])
return url_for(endpoint, **values)
def render(template_name, override_theme=None, **kwargs):
2014-03-08 13:00:20 +00:00
blocked_engines = request.cookies.get('blocked_engines', '').split(',')
2014-03-29 15:29:19 +00:00
autocomplete = request.cookies.get('autocomplete')
if autocomplete not in autocomplete_backends:
autocomplete = None
2014-03-08 13:00:20 +00:00
nonblocked_categories = (engines[e].categories
for e in engines
if e not in blocked_engines)
2014-03-29 15:29:19 +00:00
2014-03-08 13:00:20 +00:00
nonblocked_categories = set(chain.from_iterable(nonblocked_categories))
2014-03-29 15:29:19 +00:00
2014-10-19 10:41:04 +00:00
if 'categories' not in kwargs:
2014-03-08 13:00:20 +00:00
kwargs['categories'] = ['general']
kwargs['categories'].extend(x for x in
sorted(categories.keys())
2014-03-08 13:03:42 +00:00
if x != 'general'
2014-03-08 13:00:20 +00:00
and x in nonblocked_categories)
2014-03-29 15:29:19 +00:00
2014-10-19 10:41:04 +00:00
if 'selected_categories' not in kwargs:
kwargs['selected_categories'] = []
2014-09-06 13:21:29 +00:00
for arg in request.args:
if arg.startswith('category_'):
c = arg.split('_', 1)[1]
if c in categories:
kwargs['selected_categories'].append(c)
if not kwargs['selected_categories']:
cookie_categories = request.cookies.get('categories', '').split(',')
for ccateg in cookie_categories:
if ccateg in categories:
kwargs['selected_categories'].append(ccateg)
2014-09-06 13:21:29 +00:00
if not kwargs['selected_categories']:
kwargs['selected_categories'] = ['general']
2014-03-29 15:29:19 +00:00
2014-10-19 10:41:04 +00:00
if 'autocomplete' not in kwargs:
2014-03-29 15:29:19 +00:00
kwargs['autocomplete'] = autocomplete
2014-11-18 10:37:42 +00:00
kwargs['searx_version'] = VERSION_STRING
kwargs['method'] = request.cookies.get('method', 'POST')
2014-04-24 23:46:40 +00:00
# override url_for function in templates
kwargs['url_for'] = url_for_theme
kwargs['theme'] = get_current_theme_name(override=override_theme)
2014-11-18 18:55:39 +00:00
kwargs['template_name'] = template_name
2014-04-24 23:46:40 +00:00
return render_template(
'{}/{}'.format(kwargs['theme'], template_name), **kwargs)
2013-10-15 18:50:12 +00:00
@app.route('/search', methods=['GET', 'POST'])
2014-01-14 17:19:21 +00:00
@app.route('/', methods=['GET', 'POST'])
2013-10-14 21:09:13 +00:00
def index():
2014-01-31 06:08:24 +00:00
"""Render index page.
Supported outputs: html, json, csv, rss.
"""
2013-11-03 23:18:07 +00:00
2014-02-07 02:15:34 +00:00
if not request.args and not request.form:
2014-03-21 10:11:31 +00:00
return render(
'index.html',
)
2014-02-07 02:15:34 +00:00
try:
search = Search(request)
except:
2014-03-21 10:11:31 +00:00
return render(
'index.html',
)
2014-01-29 19:52:04 +00:00
2014-10-19 10:41:04 +00:00
search.results, search.suggestions,\
search.answers, search.infoboxes = search.search(request)
for result in search.results:
2014-06-24 14:30:04 +00:00
if not search.paging and engines[result['engine']].paging:
search.paging = True
2014-06-24 14:30:04 +00:00
2014-10-19 10:41:04 +00:00
# check if HTTPS rewrite is required
2014-06-24 14:30:04 +00:00
if settings['server']['https_rewrite']\
and result['parsed_url'].scheme == 'http':
skip_https_rewrite = False
# check if HTTPS rewrite is possible
for target, rules, exclusions in https_rules:
# check if target regex match with url
if target.match(result['url']):
# process exclusions
for exclusion in exclusions:
# check if exclusion match with url
if exclusion.match(result['url']):
skip_https_rewrite = True
break
# skip https rewrite if required
if skip_https_rewrite:
break
# process rules
for rule in rules:
2014-10-15 12:47:03 +00:00
try:
# TODO, precompile rule
p = re.compile(rule[0])
2014-10-19 10:41:04 +00:00
2014-10-15 12:47:03 +00:00
# rewrite url if possible
new_result_url = p.sub(rule[1], result['url'])
except:
break
# parse new url
new_parsed_url = urlparse(new_result_url)
# continiue if nothing was rewritten
if result['url'] == new_result_url:
continue
# get domainname from result
2014-10-19 10:41:04 +00:00
# TODO, does only work correct with TLD's like
# asdf.com, not for asdf.com.de
# TODO, using publicsuffix instead of this rewrite rule
2014-10-19 10:41:04 +00:00
old_result_domainname = '.'.join(
result['parsed_url'].hostname.split('.')[-2:])
new_result_domainname = '.'.join(
new_parsed_url.hostname.split('.')[-2:])
2014-10-19 10:41:04 +00:00
# check if rewritten hostname is the same,
# to protect against wrong or malicious rewrite rules
if old_result_domainname == new_result_domainname:
# set new url
result['url'] = new_result_url
2014-06-24 14:30:04 +00:00
2014-10-19 10:41:04 +00:00
# target has matched, do not search over the other rules
2014-06-24 14:30:04 +00:00
break
if search.request_data.get('format', 'html') == 'html':
2014-01-10 22:38:08 +00:00
if 'content' in result:
2014-02-07 01:45:12 +00:00
result['content'] = highlight_content(result['content'],
search.query.encode('utf-8')) # noqa
result['title'] = highlight_content(result['title'],
search.query.encode('utf-8'))
2014-01-10 22:38:08 +00:00
else:
if 'content' in result:
result['content'] = html_to_text(result['content']).strip()
# removing html content and whitespace duplications
2014-02-05 07:20:24 +00:00
result['title'] = ' '.join(html_to_text(result['title'])
.strip().split())
2014-06-24 14:30:04 +00:00
2013-10-25 00:14:26 +00:00
if len(result['url']) > 74:
2014-01-20 01:31:20 +00:00
url_parts = result['url'][:35], result['url'][-35:]
2014-02-15 06:28:21 +00:00
result['pretty_url'] = u'{0}[...]{1}'.format(*url_parts)
2013-10-25 00:14:26 +00:00
else:
result['pretty_url'] = result['url']
2013-11-15 18:28:30 +00:00
for engine in result['engines']:
2014-01-19 22:04:09 +00:00
if engine in favicons:
result['favicon'] = engine
2014-03-14 08:55:04 +00:00
# TODO, check if timezone is calculated right
if 'publishedDate' in result:
result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
2014-03-15 19:20:41 +00:00
minutes = int((timedifference.seconds / 60) % 60)
hours = int(timedifference.seconds / 60 / 60)
2014-03-15 18:13:57 +00:00
if hours == 0:
2014-03-15 19:20:41 +00:00
result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes) # noqa
2014-03-14 08:55:04 +00:00
else:
2014-03-15 19:20:41 +00:00
result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa
2014-03-14 08:55:04 +00:00
else:
result['publishedDate'] = format_date(result['publishedDate'])
2014-03-14 08:55:04 +00:00
if search.request_data.get('format') == 'json':
2014-02-07 01:45:12 +00:00
return Response(json.dumps({'query': search.query,
'results': search.results}),
2014-01-20 01:31:20 +00:00
mimetype='application/json')
elif search.request_data.get('format') == 'csv':
2013-11-15 17:55:18 +00:00
csv = UnicodeWriter(cStringIO.StringIO())
2013-12-02 20:36:09 +00:00
keys = ('title', 'url', 'content', 'host', 'engine', 'score')
2014-02-11 12:13:51 +00:00
if search.results:
2013-11-15 17:55:18 +00:00
csv.writerow(keys)
for row in search.results:
2013-12-02 20:36:09 +00:00
row['host'] = row['parsed_url'].netloc
csv.writerow([row.get(key, '') for key in keys])
2014-02-11 12:13:51 +00:00
csv.stream.seek(0)
2013-11-15 18:28:30 +00:00
response = Response(csv.stream.read(), mimetype='application/csv')
2014-02-07 01:45:12 +00:00
cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search.query)
response.headers.add('Content-Disposition', cont_disp)
2013-11-15 17:55:18 +00:00
return response
elif search.request_data.get('format') == 'rss':
response_rss = render(
'opensearch_response_rss.xml',
results=search.results,
q=search.request_data['q'],
number_of_results=len(search.results),
base_url=get_base_url()
)
2014-01-14 21:18:21 +00:00
return Response(response_rss, mimetype='text/xml')
2014-01-14 17:17:19 +00:00
return render(
'results.html',
results=search.results,
q=search.request_data['q'],
selected_categories=search.categories,
paging=search.paging,
pageno=search.pageno,
2014-03-04 17:53:56 +00:00
base_url=get_base_url(),
2014-04-24 23:46:40 +00:00
suggestions=search.suggestions,
2014-09-28 14:51:41 +00:00
answers=search.answers,
infoboxes=search.infoboxes,
theme=get_current_theme_name(),
2014-12-13 20:37:28 +00:00
favicons=global_favicons[themes.index(get_current_theme_name())]
)
2014-01-01 21:16:53 +00:00
2013-10-14 21:09:13 +00:00
2013-10-20 22:28:48 +00:00
@app.route('/about', methods=['GET'])
def about():
2014-01-31 06:08:24 +00:00
"""Render about page"""
2014-03-21 11:19:48 +00:00
return render(
'about.html',
)
2014-01-17 15:23:23 +00:00
@app.route('/autocompleter', methods=['GET', 'POST'])
def autocompleter():
"""Return autocompleter results"""
request_data = {}
2014-03-29 15:29:19 +00:00
# select request method
if request.method == 'POST':
request_data = request.form
else:
request_data = request.args
2014-03-29 15:29:19 +00:00
# set blocked engines
if request.cookies.get('blocked_engines'):
blocked_engines = request.cookies['blocked_engines'].split(',') # noqa
else:
blocked_engines = []
# parse query
query = Query(request_data.get('q', '').encode('utf-8'), blocked_engines)
query.parse_query()
2014-03-29 15:29:19 +00:00
# check if search query is set
if not query.getSearchQuery():
2014-03-29 15:29:19 +00:00
return
# run autocompleter
2014-03-29 15:29:19 +00:00
completer = autocomplete_backends.get(request.cookies.get('autocomplete'))
# check if valid autocompleter is selected
2014-03-29 15:29:19 +00:00
if not completer:
return
# run autocompletion
raw_results = completer(query.getSearchQuery())
# parse results (write :language and !engine back to result string)
results = []
for result in raw_results:
result_query = query
result_query.changeSearchQuery(result)
# add parsed result
results.append(result_query.getFullQuery())
2014-03-29 15:29:19 +00:00
# return autocompleter results
if request_data.get('format') == 'x-suggestions':
2014-12-08 23:57:04 +00:00
return Response(json.dumps([query.query, results]),
2014-03-29 15:29:19 +00:00
mimetype='application/json')
else:
2014-03-29 15:29:19 +00:00
return Response(json.dumps(results),
mimetype='application/json')
2014-01-01 21:16:53 +00:00
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
2014-01-31 06:08:24 +00:00
"""Render preferences page.
Settings that are going to be saved as cookies."""
2014-01-31 03:35:23 +00:00
lang = None
if request.cookies.get('language')\
and request.cookies['language'] in (x[0] for x in language_codes):
lang = request.cookies['language']
2014-01-01 21:16:53 +00:00
2014-02-06 23:35:15 +00:00
blocked_engines = []
if request.method == 'GET':
blocked_engines = request.cookies.get('blocked_engines', '').split(',')
2014-04-24 23:46:40 +00:00
else: # on save
2014-01-01 21:16:53 +00:00
selected_categories = []
2014-01-21 23:59:18 +00:00
locale = None
2014-03-29 15:29:19 +00:00
autocomplete = ''
method = 'POST'
for pd_name, pd in request.form.items():
2014-01-01 21:16:53 +00:00
if pd_name.startswith('category_'):
category = pd_name[9:]
2014-10-19 10:41:04 +00:00
if category not in categories:
2014-01-01 21:16:53 +00:00
continue
selected_categories.append(category)
2014-01-21 23:59:18 +00:00
elif pd_name == 'locale' and pd in settings['locales']:
locale = pd
2014-03-29 15:29:19 +00:00
elif pd_name == 'autocomplete':
autocomplete = pd
2014-01-31 03:35:23 +00:00
elif pd_name == 'language' and (pd == 'all' or
pd in (x[0] for
x in language_codes)):
lang = pd
elif pd_name == 'method':
method = pd
2014-02-06 23:35:15 +00:00
elif pd_name.startswith('engine_'):
engine_name = pd_name.replace('engine_', '', 1)
if engine_name in engines:
blocked_engines.append(engine_name)
2014-04-24 23:46:40 +00:00
elif pd_name == 'theme':
theme = pd if pd in themes else default_theme
2014-01-21 23:59:18 +00:00
2014-03-17 17:30:08 +00:00
resp = make_response(redirect(url_for('index')))
2014-01-21 23:59:18 +00:00
2014-02-06 23:42:09 +00:00
user_blocked_engines = request.cookies.get('blocked_engines', '').split(',') # noqa
2014-02-06 23:35:15 +00:00
if sorted(blocked_engines) != sorted(user_blocked_engines):
resp.set_cookie(
'blocked_engines', ','.join(blocked_engines),
max_age=cookie_max_age
2014-02-06 23:35:15 +00:00
)
2014-01-21 23:59:18 +00:00
if locale:
resp.set_cookie(
'locale', locale,
max_age=cookie_max_age
2014-01-21 23:59:18 +00:00
)
2014-01-31 03:35:23 +00:00
if lang:
resp.set_cookie(
'language', lang,
max_age=cookie_max_age
2014-01-31 03:35:23 +00:00
)
2014-01-01 21:16:53 +00:00
if selected_categories:
# cookie max age: 4 weeks
resp.set_cookie(
'categories', ','.join(selected_categories),
2014-03-29 15:29:19 +00:00
max_age=cookie_max_age
)
2014-03-29 15:29:19 +00:00
resp.set_cookie(
'autocomplete', autocomplete,
max_age=cookie_max_age
)
resp.set_cookie('method', method, max_age=cookie_max_age)
2014-04-24 23:46:40 +00:00
resp.set_cookie(
'theme', theme, max_age=cookie_max_age)
2014-01-21 23:59:18 +00:00
return resp
2014-01-22 00:20:38 +00:00
return render('preferences.html',
locales=settings['locales'],
2014-01-31 03:35:23 +00:00
current_locale=get_locale(),
current_language=lang or 'all',
2014-02-06 23:35:15 +00:00
language_codes=language_codes,
categs=categories.items(),
blocked_engines=blocked_engines,
2014-03-29 15:29:19 +00:00
autocomplete_backends=autocomplete_backends,
2014-04-24 23:46:40 +00:00
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
theme=get_current_theme_name())
2014-01-01 21:16:53 +00:00
2013-10-26 23:03:05 +00:00
@app.route('/stats', methods=['GET'])
def stats():
2014-01-31 06:08:24 +00:00
"""Render engine statistics page."""
2013-10-26 23:03:05 +00:00
stats = get_engines_stats()
2014-03-21 11:19:48 +00:00
return render(
'stats.html',
stats=stats,
)
2013-10-26 23:03:05 +00:00
2014-01-01 21:16:53 +00:00
2013-12-01 15:10:38 +00:00
@app.route('/robots.txt', methods=['GET'])
def robots():
return Response("""User-agent: *
Allow: /
Allow: /about
Disallow: /stats
2014-02-07 17:43:05 +00:00
Disallow: /preferences
2013-12-01 15:10:38 +00:00
""", mimetype='text/plain')
2014-01-01 21:16:53 +00:00
2013-10-15 22:01:08 +00:00
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
2013-10-20 20:37:55 +00:00
method = 'post'
2013-10-20 22:28:48 +00:00
# chrome/chromium only supports HTTP GET....
2013-10-20 20:37:55 +00:00
if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
method = 'get'
2014-03-29 15:29:19 +00:00
ret = render('opensearch.xml',
opensearch_method=method,
2014-03-29 15:40:43 +00:00
host=get_base_url())
2014-03-29 15:29:19 +00:00
2013-10-15 22:01:08 +00:00
resp = Response(response=ret,
2014-01-20 01:31:20 +00:00
status=200,
mimetype="application/xml")
2013-10-15 22:01:08 +00:00
return resp
2013-12-01 22:52:49 +00:00
@app.route('/favicon.ico')
def favicon():
2014-04-24 23:46:40 +00:00
return send_from_directory(os.path.join(app.root_path,
'static',
get_current_theme_name(),
'img'),
2014-01-20 01:31:20 +00:00
'favicon.png',
mimetype='image/vnd.microsoft.icon')
2013-12-01 22:52:49 +00:00
def run():
app.run(
debug=settings['server']['debug'],
use_debugger=settings['server']['debug'],
port=settings['server']['port']
)
2014-07-03 20:02:53 +00:00
application = app
if __name__ == "__main__":
run()