Merge branch 'master' into searchpy2

pull/1/head
Alexandre Flament 8 years ago committed by GitHub
commit e48f07a367

@ -0,0 +1,46 @@
from os import listdir
from os.path import realpath, dirname, join, isdir
from searx.utils import load_module
from collections import defaultdict
answerers_dir = dirname(realpath(__file__))
def load_answerers():
answerers = []
for filename in listdir(answerers_dir):
if not isdir(join(answerers_dir, filename)):
continue
module = load_module('answerer.py', join(answerers_dir, filename))
if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
exit(2)
answerers.append(module)
return answerers
def get_answerers_by_keywords(answerers):
by_keyword = defaultdict(list)
for answerer in answerers:
for keyword in answerer.keywords:
for keyword in answerer.keywords:
by_keyword[keyword].append(answerer.answer)
return by_keyword
def ask(query):
results = []
query_parts = filter(None, query.query.split())
if query_parts[0] not in answerers_by_keywords:
return results
for answerer in answerers_by_keywords[query_parts[0]]:
result = answerer(query)
if result:
results.append(result)
return results
answerers = load_answerers()
answerers_by_keywords = get_answerers_by_keywords(answerers)

@ -0,0 +1,50 @@
import random
import string
from flask_babel import gettext
# required answerer attribute
# specifies which search query keywords triggers this answerer
keywords = ('random',)
random_int_max = 2**31
random_string_letters = string.lowercase + string.digits + string.uppercase
def random_string():
return u''.join(random.choice(random_string_letters)
for _ in range(random.randint(8, 32)))
def random_float():
return unicode(random.random())
def random_int():
return unicode(random.randint(-random_int_max, random_int_max))
random_types = {u'string': random_string,
u'int': random_int,
u'float': random_float}
# required answerer function
# can return a list of results (any result type) for a given query
def answer(query):
parts = query.query.split()
if len(parts) != 2:
return []
if parts[1] not in random_types:
return []
return [{'answer': random_types[parts[1]]()}]
# required answerer function
# returns information about the answerer
def self_info():
return {'name': gettext('Random value generator'),
'description': gettext('Generate different random values'),
'examples': [u'random {}'.format(x) for x in random_types]}

@ -0,0 +1,51 @@
from functools import reduce
from operator import mul
from flask_babel import gettext
keywords = ('min',
'max',
'avg',
'sum',
'prod')
# required answerer function
# can return a list of results (any result type) for a given query
def answer(query):
parts = query.query.split()
if len(parts) < 2:
return []
try:
args = map(float, parts[1:])
except:
return []
func = parts[0]
answer = None
if func == 'min':
answer = min(args)
elif func == 'max':
answer = max(args)
elif func == 'avg':
answer = sum(args) / len(args)
elif func == 'sum':
answer = sum(args)
elif func == 'prod':
answer = reduce(mul, args, 1)
if answer is None:
return []
return [{'answer': unicode(answer)}]
# required answerer function
# returns information about the answerer
def self_info():
return {'name': gettext('Statistics functions'),
'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),
'examples': ['avg 123 548 2.04 24.2']}

@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com> (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
''' '''
from os.path import realpath, dirname, splitext, join from os.path import realpath, dirname
import sys import sys
from imp import load_source
from flask_babel import gettext from flask_babel import gettext
from operator import itemgetter from operator import itemgetter
from searx import settings from searx import settings
from searx import logger from searx import logger
from searx.utils import load_module
logger = logger.getChild('engines') logger = logger.getChild('engines')
@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__))
engines = {} engines = {}
categories = {'general': []} categories = {'general': []}
_initialized = False
engine_shortcuts = {} engine_shortcuts = {}
engine_default_args = {'paging': False, engine_default_args = {'paging': False,
@ -46,16 +47,6 @@ engine_default_args = {'paging': False,
'time_range_support': False} 'time_range_support': False}
def load_module(filename):
modname = splitext(filename)[0]
if modname in sys.modules:
del sys.modules[modname]
filepath = join(engine_dir, filename)
module = load_source(modname, filepath)
module.name = modname
return module
def load_engine(engine_data): def load_engine(engine_data):
if '_' in engine_data['name']: if '_' in engine_data['name']:
@ -65,7 +56,7 @@ def load_engine(engine_data):
engine_module = engine_data['engine'] engine_module = engine_data['engine']
try: try:
engine = load_module(engine_module + '.py') engine = load_module(engine_module + '.py', engine_dir)
except: except:
logger.exception('Cannot load engine "{}"'.format(engine_module)) logger.exception('Cannot load engine "{}"'.format(engine_module))
return None return None

@ -12,7 +12,6 @@
""" """
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
@ -135,7 +134,7 @@ def response(resp):
for result in dom.xpath(xpath_results): for result in dom.xpath(xpath_results):
link = result.xpath(xpath_link)[0] link = result.xpath(xpath_link)[0]
href = urljoin(base_url, link.attrib.get('href')) href = urljoin(base_url, link.attrib.get('href'))
title = escape(extract_text(link)) title = extract_text(link)
results.append({'url': href, results.append({'url': href,
'title': title}) 'title': title})

@ -16,7 +16,6 @@
from lxml import etree from lxml import etree
from urllib import urlencode from urllib import urlencode
from searx.utils import searx_useragent from searx.utils import searx_useragent
from cgi import escape
from datetime import datetime from datetime import datetime
import re import re
@ -94,7 +93,7 @@ def response(resp):
url = item.text url = item.text
elif item.attrib["name"] == "dcdescription": elif item.attrib["name"] == "dcdescription":
content = escape(item.text[:300]) content = item.text[:300]
if len(item.text) > 300: if len(item.text) > 300:
content += "..." content += "..."

@ -14,7 +14,6 @@
""" """
from urllib import urlencode from urllib import urlencode
from cgi import escape
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
@ -32,18 +31,14 @@ search_string = 'search?{query}&first={offset}'
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1 offset = (params['pageno'] - 1) * 10 + 1
if params['language'] == 'all': if params['language'] != 'all':
language = 'en-US' query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
else: query.decode('utf-8')).encode('utf-8')
language = params['language'].replace('_', '-')
search_path = search_string.format( search_path = search_string.format(
query=urlencode({'q': query, 'setmkt': language}), query=urlencode({'q': query}),
offset=offset) offset=offset)
params['cookies']['SRCHHPGUSR'] = \
'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]
params['url'] = base_url + search_path params['url'] = base_url + search_path
return params return params
@ -65,7 +60,7 @@ def response(resp):
link = result.xpath('.//h3/a')[0] link = result.xpath('.//h3/a')[0]
url = link.attrib.get('href') url = link.attrib.get('href')
title = extract_text(link) title = extract_text(link)
content = escape(extract_text(result.xpath('.//p'))) content = extract_text(result.xpath('.//p'))
# append result # append result
results.append({'url': url, results.append({'url': url,
@ -77,7 +72,7 @@ def response(resp):
link = result.xpath('.//h2/a')[0] link = result.xpath('.//h2/a')[0]
url = link.attrib.get('href') url = link.attrib.get('href')
title = extract_text(link) title = extract_text(link)
content = escape(extract_text(result.xpath('.//p'))) content = extract_text(result.xpath('.//p'))
# append result # append result
results.append({'url': url, results.append({'url': url,

@ -11,7 +11,6 @@
""" """
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import quote from urllib import quote
from lxml import html from lxml import html
from operator import itemgetter from operator import itemgetter
@ -51,8 +50,8 @@ def response(resp):
for result in search_res: for result in search_res:
link = result.xpath('.//td[@class="torrent_name"]//a')[0] link = result.xpath('.//td[@class="torrent_name"]//a')[0]
href = urljoin(url, link.attrib.get('href')) href = urljoin(url, link.attrib.get('href'))
title = escape(extract_text(link)) title = extract_text(link)
content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0])) content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])
content = "<br />".join(content.split("\n")) content = "<br />".join(content.split("\n"))
filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0] filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

@ -14,7 +14,6 @@
from urllib import urlencode from urllib import urlencode
from json import loads from json import loads
from cgi import escape
from datetime import datetime from datetime import datetime
# engine dependent config # engine dependent config
@ -57,7 +56,7 @@ def response(resp):
for res in search_res['list']: for res in search_res['list']:
title = res['title'] title = res['title']
url = res['url'] url = res['url']
content = escape(res['description']) content = res['description']
thumbnail = res['thumbnail_360_url'] thumbnail = res['thumbnail_360_url']
publishedDate = datetime.fromtimestamp(res['created_time'], None) publishedDate = datetime.fromtimestamp(res['created_time'], None)
embedded = embedded_url.format(videoid=res['id']) embedded = embedded_url.format(videoid=res['id'])

@ -51,10 +51,11 @@ def response(resp):
if url.startswith('http://'): if url.startswith('http://'):
url = 'https' + url[4:] url = 'https' + url[4:]
content = result['artist']['name'] +\ content = '{} - {} - {}'.format(
" &bull; " +\ result['artist']['name'],
result['album']['title'] +\ result['album']['title'],
" &bull; " + result['title'] result['title'])
embedded = embedded_url.format(audioid=result['id']) embedded = embedded_url.format(audioid=result['id'])
# append result # append result

@ -12,7 +12,6 @@
import re import re
from urlparse import urljoin from urlparse import urljoin
from lxml import html from lxml import html
from cgi import escape
from searx.utils import is_valid_lang from searx.utils import is_valid_lang
categories = ['general'] categories = ['general']
@ -62,8 +61,8 @@ def response(resp):
results.append({ results.append({
'url': urljoin(resp.url, '?%d' % k), 'url': urljoin(resp.url, '?%d' % k),
'title': escape(from_result.text_content()), 'title': from_result.text_content(),
'content': escape('; '.join(to_results)) 'content': '; '.join(to_results)
}) })
return results return results

@ -13,7 +13,6 @@
from urllib import quote_plus from urllib import quote_plus
from json import loads from json import loads
from lxml import html from lxml import html
from cgi import escape
from dateutil import parser from dateutil import parser
# engine dependent config # engine dependent config
@ -56,7 +55,7 @@ def response(resp):
url = result.attrib.get('data-contenturl') url = result.attrib.get('data-contenturl')
thumbnail = result.xpath('.//img')[0].attrib.get('src') thumbnail = result.xpath('.//img')[0].attrib.get('src')
title = ''.join(result.xpath(title_xpath)) title = ''.join(result.xpath(title_xpath))
content = escape(''.join(result.xpath(content_xpath))) content = ''.join(result.xpath(content_xpath))
pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime') pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
publishedDate = parser.parse(pubdate) publishedDate = parser.parse(pubdate)

@ -9,7 +9,6 @@
@parse url, title, content @parse url, title, content
""" """
from cgi import escape
from urllib import urlencode from urllib import urlencode
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
from lxml import html from lxml import html
@ -43,7 +42,7 @@ def response(resp):
img_src = app.xpath('.//img/@src')[0] img_src = app.xpath('.//img/@src')[0]
content = extract_text(app.xpath('./p')[0]) content = extract_text(app.xpath('./p')[0])
content = escape(content.replace(title, '', 1).strip()) content = content.replace(title, '', 1).strip()
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,

@ -77,21 +77,13 @@ def response(resp):
url = build_flickr_url(photo['owner'], photo['id']) url = build_flickr_url(photo['owner'], photo['id'])
title = photo['title']
content = '<span class="photo-author">' +\
photo['ownername'] +\
'</span><br />' +\
'<span class="description">' +\
photo['description']['_content'] +\
'</span>'
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': photo['title'],
'img_src': img_src, 'img_src': img_src,
'thumbnail_src': thumbnail_src, 'thumbnail_src': thumbnail_src,
'content': content, 'content': photo['description']['_content'],
'author': photo['ownername'],
'template': 'images.html'}) 'template': 'images.html'})
# return results # return results

@ -102,16 +102,15 @@ def response(resp):
title = photo.get('title', '') title = photo.get('title', '')
content = '<span class="photo-author">' +\ author = photo['username']
photo['username'] +\
'</span><br />'
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'img_src': img_src, 'img_src': img_src,
'thumbnail_src': thumbnail_src, 'thumbnail_src': thumbnail_src,
'content': content, 'content': '',
'author': author,
'template': 'images.html'}) 'template': 'images.html'})
return results return results

@ -10,7 +10,6 @@
@parse url, title, content @parse url, title, content
""" """
from cgi import escape
from json import loads from json import loads
from random import randint from random import randint
from time import time from time import time
@ -78,8 +77,8 @@ def response(resp):
for result in response_json['results']: for result in response_json['results']:
# append result # append result
results.append({'url': result['url'], results.append({'url': result['url'],
'title': escape(result['title']), 'title': result['title'],
'content': escape(result['sum'])}) 'content': result['sum']})
# return results # return results
return results return results

@ -12,7 +12,6 @@
from urllib import urlencode from urllib import urlencode
from json import loads from json import loads
from cgi import escape
# engine dependent config # engine dependent config
categories = ['it'] categories = ['it']
@ -48,7 +47,7 @@ def response(resp):
url = res['html_url'] url = res['html_url']
if res['description']: if res['description']:
content = escape(res['description'][:500]) content = res['description'][:500]
else: else:
content = '' content = ''

@ -9,7 +9,6 @@
# @parse url, title, content, suggestion # @parse url, title, content, suggestion
import re import re
from cgi import escape
from urllib import urlencode from urllib import urlencode
from urlparse import urlparse, parse_qsl from urlparse import urlparse, parse_qsl
from lxml import html, etree from lxml import html, etree
@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):
def extract_text_from_dom(result, xpath): def extract_text_from_dom(result, xpath):
r = result.xpath(xpath) r = result.xpath(xpath)
if len(r) > 0: if len(r) > 0:
return escape(extract_text(r[0])) return extract_text(r[0])
return None return None
@ -264,7 +263,7 @@ def response(resp):
# parse suggestion # parse suggestion
for suggestion in dom.xpath(suggestion_xpath): for suggestion in dom.xpath(suggestion_xpath):
# append suggestion # append suggestion
results.append({'suggestion': escape(extract_text(suggestion))}) results.append({'suggestion': extract_text(suggestion)})
# return results # return results
return results return results

@ -11,7 +11,6 @@
""" """
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import quote from urllib import quote
from lxml import html from lxml import html
from operator import itemgetter from operator import itemgetter
@ -57,7 +56,7 @@ def response(resp):
link = result.xpath('.//a[@class="cellMainLink"]')[0] link = result.xpath('.//a[@class="cellMainLink"]')[0]
href = urljoin(url, link.attrib['href']) href = urljoin(url, link.attrib['href'])
title = extract_text(link) title = extract_text(link)
content = escape(extract_text(result.xpath(content_xpath))) content = extract_text(result.xpath(content_xpath))
seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]')) filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

@ -9,7 +9,6 @@
@parse url, title, content, seed, leech, torrentfile @parse url, title, content, seed, leech, torrentfile
""" """
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
@ -78,7 +77,7 @@ def response(resp):
# torrent title # torrent title
page_a = result.xpath(xpath_title)[0] page_a = result.xpath(xpath_title)[0]
title = escape(extract_text(page_a)) title = extract_text(page_a)
# link to the page # link to the page
href = page_a.attrib.get('href') href = page_a.attrib.get('href')
@ -90,7 +89,7 @@ def response(resp):
try: try:
file_size, suffix = result.xpath(xpath_filesize)[0].split(' ') file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
file_size = int(float(file_size) * get_filesize_mul(suffix)) file_size = int(float(file_size) * get_filesize_mul(suffix))
except Exception as e: except:
file_size = None file_size = None
# seed count # seed count
@ -105,7 +104,6 @@ def response(resp):
# content string contains all information not included into template # content string contains all information not included into template
content = 'Category: "{category}". Downloaded {downloads} times.' content = 'Category: "{category}". Downloaded {downloads} times.'
content = content.format(category=category, downloads=downloads) content = content.format(category=category, downloads=downloads)
content = escape(content)
results.append({'url': href, results.append({'url': href,
'title': title, 'title': title,

@ -43,7 +43,7 @@ def response(resp):
if 'display_name' not in r: if 'display_name' not in r:
continue continue
title = r['display_name'] title = r['display_name'] or u''
osm_type = r.get('osm_type', r.get('type')) osm_type = r.get('osm_type', r.get('type'))
url = result_base_url.format(osm_type=osm_type, url = result_base_url.format(osm_type=osm_type,
osm_id=r['osm_id']) osm_id=r['osm_id'])

@ -9,7 +9,6 @@
# @parse url, title, content, seed, leech, magnetlink # @parse url, title, content, seed, leech, magnetlink
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import quote from urllib import quote
from lxml import html from lxml import html
from operator import itemgetter from operator import itemgetter
@ -62,7 +61,7 @@ def response(resp):
link = result.xpath('.//div[@class="detName"]//a')[0] link = result.xpath('.//div[@class="detName"]//a')[0]
href = urljoin(url, link.attrib.get('href')) href = urljoin(url, link.attrib.get('href'))
title = extract_text(link) title = extract_text(link)
content = escape(extract_text(result.xpath(content_xpath))) content = extract_text(result.xpath(content_xpath))
seed, leech = result.xpath('.//td[@align="right"]/text()')[:2] seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
# convert seed to int if possible # convert seed to int if possible

@ -11,7 +11,6 @@
""" """
import json import json
from cgi import escape
from urllib import urlencode from urllib import urlencode
from urlparse import urlparse, urljoin from urlparse import urlparse, urljoin
from datetime import datetime from datetime import datetime
@ -68,7 +67,7 @@ def response(resp):
img_results.append(params) img_results.append(params)
else: else:
created = datetime.fromtimestamp(data['created_utc']) created = datetime.fromtimestamp(data['created_utc'])
content = escape(data['selftext']) content = data['selftext']
if len(content) > 500: if len(content) > 500:
content = content[:500] + '...' content = content[:500] + '...'
params['content'] = content params['content'] = content

@ -44,20 +44,12 @@ def response(resp):
# parse results # parse results
for result in search_results.get('results', []): for result in search_results.get('results', []):
href = result['url'] href = result['url']
title = "[" + result['type'] + "] " +\ title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])
result['namespace'] +\
" " + result['name']
content = '<span class="highlight">[' +\
result['type'] + "] " +\
result['name'] + " " +\
result['synopsis'] +\
"</span><br />" +\
result['description']
# append result # append result
results.append({'url': href, results.append({'url': href,
'title': title, 'title': title,
'content': content}) 'content': result['description']})
# return results # return results
return results return results

@ -9,7 +9,6 @@
# @parse url, title, content, seed, leech, magnetlink # @parse url, title, content, seed, leech, magnetlink
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import quote from urllib import quote
from lxml import html from lxml import html
from operator import itemgetter from operator import itemgetter

@ -46,10 +46,11 @@ def response(resp):
if result['type'] == 'track': if result['type'] == 'track':
title = result['name'] title = result['name']
url = result['external_urls']['spotify'] url = result['external_urls']['spotify']
content = result['artists'][0]['name'] +\ content = '{} - {} - {}'.format(
" &bull; " +\ result['artists'][0]['name'],
result['album']['name'] +\ result['album']['name'],
" &bull; " + result['name'] result['name'])
embedded = embedded_url.format(audioid=result['id']) embedded = embedded_url.format(audioid=result['id'])
# append result # append result

@ -11,7 +11,6 @@
""" """
from urlparse import urljoin from urlparse import urljoin
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
@ -48,8 +47,8 @@ def response(resp):
for result in dom.xpath(results_xpath): for result in dom.xpath(results_xpath):
link = result.xpath(link_xpath)[0] link = result.xpath(link_xpath)[0]
href = urljoin(url, link.attrib.get('href')) href = urljoin(url, link.attrib.get('href'))
title = escape(extract_text(link)) title = extract_text(link)
content = escape(extract_text(result.xpath(content_xpath))) content = extract_text(result.xpath(content_xpath))
# append result # append result
results.append({'url': href, results.append({'url': href,

@ -11,7 +11,6 @@
# @todo paging # @todo paging
from lxml import html from lxml import html
from cgi import escape
from dateutil import parser from dateutil import parser
from datetime import datetime, timedelta from datetime import datetime, timedelta
import re import re
@ -79,10 +78,10 @@ def response(resp):
if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url): if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):
continue continue
title = escape(extract_text(link)) title = extract_text(link)
if result.xpath('./p[@class="desc clk"]'): if result.xpath('./p[@class="desc clk"]'):
content = escape(extract_text(result.xpath('./p[@class="desc clk"]'))) content = extract_text(result.xpath('./p[@class="desc clk"]'))
else: else:
content = '' content = ''

@ -10,7 +10,6 @@
@parse url, title, content @parse url, title, content
""" """
from cgi import escape
from urllib import quote_plus from urllib import quote_plus
from lxml import html from lxml import html
from searx.languages import language_codes from searx.languages import language_codes
@ -59,7 +58,7 @@ def response(resp):
elif search_lang: elif search_lang:
href = href + search_lang + '/' href = href + search_lang + '/'
title = escape(extract_text(link)) title = extract_text(link)
content = extract_text(result.xpath('.//div[contains(@class,"red")]')) content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
content = content + " - " content = content + " - "
@ -75,7 +74,7 @@ def response(resp):
# append result # append result
results.append({'url': href, results.append({'url': href,
'title': title, 'title': title,
'content': escape(content)}) 'content': content})
# return results # return results
return results return results

@ -10,7 +10,6 @@
@parse url, title, content @parse url, title, content
""" """
from cgi import escape
from json import loads from json import loads
from urllib import urlencode, unquote from urllib import urlencode, unquote
import re import re
@ -78,7 +77,7 @@ def response(resp):
# append result # append result
results.append({'url': result['SourceUrl'], results.append({'url': result['SourceUrl'],
'title': escape(result['Title']), 'title': result['Title'],
'content': '', 'content': '',
'img_src': img_url, 'img_src': img_url,
'template': 'images.html'}) 'template': 'images.html'})
@ -90,8 +89,8 @@ def response(resp):
# append result # append result
results.append({'url': result_url, results.append({'url': result_url,
'title': escape(result_title), 'title': result_title,
'content': escape(result_content)}) 'content': result_content})
# parse images # parse images
for result in json.get('Images', []): for result in json.get('Images', []):
@ -100,7 +99,7 @@ def response(resp):
# append result # append result
results.append({'url': result['SourceUrl'], results.append({'url': result['SourceUrl'],
'title': escape(result['Title']), 'title': result['Title'],
'content': '', 'content': '',
'img_src': img_url, 'img_src': img_url,
'template': 'images.html'}) 'template': 'images.html'})

@ -11,7 +11,6 @@
""" """
import re import re
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text

@ -12,7 +12,6 @@
""" """
import re import re
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text

@ -9,7 +9,6 @@
@parse url, title, content @parse url, title, content
""" """
import re import re
from cgi import escape
from searx.utils import is_valid_lang from searx.utils import is_valid_lang
categories = ['general'] categories = ['general']
@ -52,14 +51,14 @@ def request(query, params):
def response(resp): def response(resp):
results = [] results = []
results.append({ results.append({
'url': escape(web_url.format( 'url': web_url.format(
from_lang=resp.search_params['from_lang'][2], from_lang=resp.search_params['from_lang'][2],
to_lang=resp.search_params['to_lang'][2], to_lang=resp.search_params['to_lang'][2],
query=resp.search_params['query'])), query=resp.search_params['query']),
'title': escape('[{0}-{1}] {2}'.format( 'title': '[{0}-{1}] {2}'.format(
resp.search_params['from_lang'][1], resp.search_params['from_lang'][1],
resp.search_params['to_lang'][1], resp.search_params['to_lang'][1],
resp.search_params['query'])), resp.search_params['query']),
'content': escape(resp.json()['responseData']['translatedText']) 'content': resp.json()['responseData']['translatedText']
}) })
return results return results

@ -8,7 +8,6 @@
# @stable no # @stable no
# @parse url, infobox # @parse url, infobox
from cgi import escape
from json import loads from json import loads
from time import time from time import time
from urllib import urlencode from urllib import urlencode

@ -9,7 +9,6 @@
@parse url, title, content @parse url, title, content
""" """
from cgi import escape
from urllib import urlencode from urllib import urlencode
from lxml import html from lxml import html
from searx.search import logger from searx.search import logger
@ -52,8 +51,8 @@ def response(resp):
for result in dom.xpath(results_xpath): for result in dom.xpath(results_xpath):
try: try:
res = {'url': result.xpath(url_xpath)[0], res = {'url': result.xpath(url_xpath)[0],
'title': escape(''.join(result.xpath(title_xpath))), 'title': ''.join(result.xpath(title_xpath)),
'content': escape(''.join(result.xpath(content_xpath)))} 'content': ''.join(result.xpath(content_xpath))}
except: except:
logger.exception('yandex parse crash') logger.exception('yandex parse crash')
continue continue

@ -27,5 +27,5 @@ def on_result(request, search, result):
if doi.endswith(suffix): if doi.endswith(suffix):
doi = doi[:-len(suffix)] doi = doi[:-len(suffix)]
result['url'] = 'http://doai.io/' + doi result['url'] = 'http://doai.io/' + doi
result['parsed_url'] = urlparse(ctx['result']['url']) result['parsed_url'] = urlparse(result['url'])
return True return True

@ -49,28 +49,32 @@ class StringSetting(Setting):
class EnumStringSetting(Setting): class EnumStringSetting(Setting):
"""Setting of a value which can only come from the given choices""" """Setting of a value which can only come from the given choices"""
def _validate_selection(self, selection):
if selection not in self.choices:
raise ValidationException('Invalid value: "{0}"'.format(selection))
def _post_init(self): def _post_init(self):
if not hasattr(self, 'choices'): if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices') raise MissingArgumentException('Missing argument: choices')
self._validate_selection(self.value)
if self.value != '' and self.value not in self.choices:
raise ValidationException('Invalid default value: {0}'.format(self.value))
def parse(self, data): def parse(self, data):
if data not in self.choices and data != self.value: self._validate_selection(data)
raise ValidationException('Invalid choice: {0}'.format(data))
self.value = data self.value = data
class MultipleChoiceSetting(EnumStringSetting): class MultipleChoiceSetting(EnumStringSetting):
"""Setting of values which can only come from the given choices""" """Setting of values which can only come from the given choices"""
def _validate_selections(self, selections):
for item in selections:
if item not in self.choices:
raise ValidationException('Invalid value: "{0}"'.format(selections))
def _post_init(self): def _post_init(self):
if not hasattr(self, 'choices'): if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices') raise MissingArgumentException('Missing argument: choices')
for item in self.value: self._validate_selections(self.value)
if item not in self.choices:
raise ValidationException('Invalid default value: {0}'.format(self.value))
def parse(self, data): def parse(self, data):
if data == '': if data == '':
@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):
return return
elements = data.split(',') elements = data.split(',')
for item in elements: self._validate_selections(elements)
if item not in self.choices:
raise ValidationException('Invalid choice: {0}'.format(item))
self.value = elements self.value = elements
def parse_form(self, data): def parse_form(self, data):
@ -214,11 +216,12 @@ class Preferences(object):
super(Preferences, self).__init__() super(Preferences, self).__init__()
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories), self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
'language': EnumStringSetting('all', choices=LANGUAGE_CODES), 'language': EnumStringSetting(settings['search']['language'],
choices=LANGUAGE_CODES),
'locale': EnumStringSetting(settings['ui']['default_locale'], 'locale': EnumStringSetting(settings['ui']['default_locale'],
choices=settings['locales'].keys()), choices=settings['locales'].keys() + ['']),
'autocomplete': EnumStringSetting(settings['search']['autocomplete'], 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
choices=autocomplete.backends.keys()), choices=autocomplete.backends.keys() + ['']),
'image_proxy': MapSetting(settings['server']['image_proxy'], 'image_proxy': MapSetting(settings['server']['image_proxy'],
map={'': settings['server']['image_proxy'], map={'': settings['server']['image_proxy'],
'0': False, '0': False,

@ -146,16 +146,17 @@ class ResultContainer(object):
self._number_of_results.append(result['number_of_results']) self._number_of_results.append(result['number_of_results'])
results.remove(result) results.remove(result)
with RLock(): if engine_name in engines:
engines[engine_name].stats['search_count'] += 1 with RLock():
engines[engine_name].stats['result_count'] += len(results) engines[engine_name].stats['search_count'] += 1
engines[engine_name].stats['result_count'] += len(results)
if not results: if not results:
return return
self.results[engine_name].extend(results) self.results[engine_name].extend(results)
if not self.paging and engines[engine_name].paging: if not self.paging and engine_name in engines and engines[engine_name].paging:
self.paging = True self.paging = True
for i, result in enumerate(results): for i, result in enumerate(results):

@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib
from searx.engines import ( from searx.engines import (
categories, engines categories, engines
) )
from searx.answerers import ask
from searx.utils import gen_useragent from searx.utils import gen_useragent
from searx.query import RawTextQuery, SearchQuery from searx.query import RawTextQuery, SearchQuery
from searx.results import ResultContainer from searx.results import ResultContainer
@ -300,6 +301,14 @@ class Search(object):
# start time # start time
start_time = time() start_time = time()
# answeres ?
answerers_results = ask(self.search_query)
if answerers_results:
for results in answerers_results:
self.result_container.extend('answer', results)
return self.result_container
# init vars # init vars
requests = [] requests = []

@ -5,6 +5,7 @@ general:
search: search:
safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
language : "all"
server: server:
port : 8888 port : 8888

@ -5,6 +5,7 @@ general:
search: search:
safe_search : 0 safe_search : 0
autocomplete : "" autocomplete : ""
language: "all"
server: server:
port : 11111 port : 11111

@ -5,7 +5,7 @@ $(document).ready(function() {
var formData = $('#pagination form:last').serialize(); var formData = $('#pagination form:last').serialize();
if (formData) { if (formData) {
$('#pagination').html('<div class="loading-spinner"></div>'); $('#pagination').html('<div class="loading-spinner"></div>');
$.post('/', formData, function (data) { $.post('./', formData, function (data) {
var body = $(data); var body = $(data);
$('#pagination').remove(); $('#pagination').remove();
$('#main_results').append('<hr/>'); $('#main_results').append('<hr/>');

@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom"> xmlns:atom="http://www.w3.org/2005/Atom">
<channel> <channel>
<title>Searx search: {{ q }}</title> <title>Searx search: {{ q|e }}</title>
<link>{{ base_url }}?q={{ q }}</link> <link>{{ base_url }}?q={{ q|e }}</link>
<description>Search results for "{{ q }}" - searx</description> <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex> <opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
<opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %} {% for r in results %}
<item> <item>
<title>{{ r.title }}</title> <title>{{ r.title }}</title>

@ -1,6 +1,6 @@
{% extends "courgette/base.html" %} {% extends "courgette/base.html" %}
{% block title %}{{ q }} - {% endblock %} {% block title %}{{ q|e }} - {% endblock %}
{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %} {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
{% block content %} {% block content %}
<div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div> <div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div>
<div class="small search center"> <div class="small search center">
@ -17,7 +17,7 @@
{% for output_type in ('csv', 'json', 'rss') %} {% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left"> <div class="left">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="format" value="{{ output_type }}" /> <input type="hidden" name="format" value="{{ output_type }}" />
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
@ -62,7 +62,7 @@
{% if pageno > 1 %} {% if pageno > 1 %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left"> <div class="left">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %} {% endfor %}
@ -76,7 +76,7 @@
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %} {% endfor %}
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="submit" value="{{ _('next page') }} >>" /> <input type="submit" value="{{ _('next page') }} >>" />
</div> </div>

@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom"> xmlns:atom="http://www.w3.org/2005/Atom">
<channel> <channel>
<title>Searx search: {{ q }}</title> <title>Searx search: {{ q|e }}</title>
<link>{{ base_url }}?q={{ q }}</link> <link>{{ base_url }}?q={{ q|e }}</link>
<description>Search results for "{{ q }}" - searx</description> <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex> <opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
<opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %} {% for r in results %}
<item> <item>
<title>{{ r.title }}</title> <title>{{ r.title }}</title>

@ -1,6 +1,6 @@
{% extends "legacy/base.html" %} {% extends "legacy/base.html" %}
{% block title %}{{ q }} - {% endblock %} {% block title %}{{ q|e }} - {% endblock %}
{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %} {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
{% block content %} {% block content %}
<div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div> <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
<div class="small search center"> <div class="small search center">
@ -18,7 +18,7 @@
{% for output_type in ('csv', 'json', 'rss') %} {% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left"> <div class="left">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="format" value="{{ output_type }}" /> <input type="hidden" name="format" value="{{ output_type }}" />
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
@ -73,7 +73,7 @@
{% if pageno > 1 %} {% if pageno > 1 %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="{% if rtl %}right{% else %}left{% endif %}"> <div class="{% if rtl %}right{% else %}left{% endif %}">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %} {% endfor %}
@ -87,7 +87,7 @@
{% for category in selected_categories %} {% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/> <input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %} {% endfor %}
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="submit" value="{{ _('next page') }} >>" /> <input type="submit" value="{{ _('next page') }} >>" />
</div> </div>

@ -1,3 +1,4 @@
{% from 'oscar/macros.html' import icon %}
<!DOCTYPE html> <!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}> <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>
<head> <head>
@ -54,6 +55,20 @@
<body> <body>
{% include 'oscar/navbar.html' %} {% include 'oscar/navbar.html' %}
<div class="container"> <div class="container">
{% if errors %}
<div class="alert alert-danger fade in" role="alert">
<button class="close" data-dismiss="alert" type="button">
<span aria-hidden="true">×</span>
<span class="sr-only">{{ _('Close') }}</span>
</button>
<strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong>
<ul>
{% for message in errors %}
<li>{{ message }}</li>
{% endfor %}
</ul>
</div>
{% endif %}
{% block site_alert_error %} {% block site_alert_error %}
{% endblock %} {% endblock %}

@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom"> xmlns:atom="http://www.w3.org/2005/Atom">
<channel> <channel>
<title>Searx search: {{ q }}</title> <title>Searx search: {{ q|e }}</title>
<link>{{ base_url }}?q={{ q }}</link> <link>{{ base_url }}?q={{ q|e }}</link>
<description>Search results for "{{ q }}" - searx</description> <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex> <opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
<opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %} {% for r in results %}
<item> <item>
<title>{{ r.title }}</title> <title>{{ r.title }}</title>

@ -12,6 +12,7 @@
<li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li> <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
<li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li> <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
<li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li> <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
{% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %}
<li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li> <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>
</ul> </ul>
@ -224,6 +225,34 @@
</fieldset> </fieldset>
</div> </div>
{% if answerers %}
<div class="tab-pane active_if_nojs" id="tab_answerers">
<noscript>
<h3>{{ _('Answerers') }}</h3>
</noscript>
<p class="text-muted" style="margin:20px 0;">
{{ _('This is the list of searx\'s instant answering modules.') }}
</p>
<table class="table table-striped">
<tr>
<th class="text-muted">{{ _('Name') }}</th>
<th class="text-muted">{{ _('Keywords') }}</th>
<th class="text-muted">{{ _('Description') }}</th>
<th class="text-muted">{{ _('Examples') }}</th>
</tr>
{% for answerer in answerers %}
<tr>
<td class="text-muted">{{ answerer.info.name }}</td>
<td class="text-muted">{{ answerer.keywords|join(', ') }}</td>
<td class="text-muted">{{ answerer.info.description }}</td>
<td class="text-muted">{{ answerer.info.examples|join(', ') }}</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
<div class="tab-pane active_if_nojs" id="tab_cookies"> <div class="tab-pane active_if_nojs" id="tab_cookies">
<noscript> <noscript>
<h3>{{ _('Cookies') }}</h3> <h3>{{ _('Cookies') }}</h3>

@ -13,7 +13,12 @@
</div> </div>
<div class="modal-body"> <div class="modal-body">
<img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}"> <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
{% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %} {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
{% if result.content %}
<p class="result-content">
{{ result.content }}
</p>
{% endif %}
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<div class="clearfix"></div> <div class="clearfix"></div>

@ -1,6 +1,6 @@
{% extends "oscar/base.html" %} {% extends "oscar/base.html" %}
{% block title %}{{ q }} - {% endblock %} {% block title %}{{ q|e }} - {% endblock %}
{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}&amp;time_range={{ time_range }}">{% endblock %} {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}&amp;time_range={{ time_range }}">{% endblock %}
{% block content %} {% block content %}
<div class="row"> <div class="row">
<div class="col-sm-8" id="main_results"> <div class="col-sm-8" id="main_results">
@ -37,9 +37,9 @@
<div id="pagination"> <div id="pagination">
<div class="pull-left"> <div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button> <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
@ -59,7 +59,7 @@
<div id="pagination"> <div id="pagination">
<div class="pull-left"> <div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno-1 }}" /> <input type="hidden" name="pageno" value="{{ pageno-1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" />
@ -69,7 +69,7 @@
<div class="pull-right"> <div class="pull-right">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="q" value="{{ q }}" /> <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button> <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
@ -130,7 +130,7 @@
<div class="clearfix"></div> <div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %} {% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
<input type="hidden" name="q" value="{{ q }}"> <input type="hidden" name="q" value="{{ q|e }}">
<input type="hidden" name="format" value="{{ output_type }}"> <input type="hidden" name="format" value="{{ output_type }}">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %} {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno }}"> <input type="hidden" name="pageno" value="{{ pageno }}">

@ -5,7 +5,7 @@
{% endfor %} {% endfor %}
{% else %} {% else %}
{% extends "pix-art/base.html" %} {% extends "pix-art/base.html" %}
{% block title %}{{ q }} - {% endblock %} {% block title %}{{ q|e }} - {% endblock %}
{% block meta %}{% endblock %} {% block meta %}{% endblock %}
{% block content %} {% block content %}
<div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div> <div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div>
@ -25,8 +25,8 @@
</span> </span>
<div id="pagination"> <div id="pagination">
<br /> <br />
<input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" /> <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />
</div> </div>
</div> </div>
{% endblock %} {% endblock %}
{% endif %} {% endif %}

@ -6,7 +6,10 @@ import re
from babel.dates import format_date from babel.dates import format_date
from codecs import getincrementalencoder from codecs import getincrementalencoder
from HTMLParser import HTMLParser from HTMLParser import HTMLParser
from imp import load_source
from os.path import splitext, join
from random import choice from random import choice
import sys
from searx.version import VERSION_STRING from searx.version import VERSION_STRING
from searx.languages import language_codes from searx.languages import language_codes
@ -285,3 +288,13 @@ def is_valid_lang(lang):
if l[1].lower() == lang.lower(): if l[1].lower() == lang.lower():
return (True, l[0][:2], l[1].lower()) return (True, l[0][:2], l[1].lower())
return False return False
def load_module(filename, module_dir):
modname = splitext(filename)[0]
if modname in sys.modules:
del sys.modules[modname]
filepath = join(module_dir, filename)
module = load_source(modname, filepath)
module.name = modname
return module

@ -40,7 +40,7 @@ except:
logger.critical("cannot import dependency: pygments") logger.critical("cannot import dependency: pygments")
from sys import exit from sys import exit
exit(1) exit(1)
from cgi import escape
from datetime import datetime, timedelta from datetime import datetime, timedelta
from urllib import urlencode from urllib import urlencode
from urlparse import urlparse, urljoin from urlparse import urlparse, urljoin
@ -62,11 +62,12 @@ from searx.utils import (
) )
from searx.version import VERSION_STRING from searx.version import VERSION_STRING
from searx.languages import language_codes from searx.languages import language_codes
from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp from searx.search import SearchWithPlugins, get_search_query_from_webapp
from searx.query import RawTextQuery, SearchQuery from searx.query import RawTextQuery
from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins from searx.plugins import plugins
from searx.preferences import Preferences, ValidationException from searx.preferences import Preferences, ValidationException
from searx.answerers import answerers
# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed. # check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
# They are needed for SSL connection without trouble, see #298 # They are needed for SSL connection without trouble, see #298
@ -344,6 +345,8 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['cookies'] = request.cookies kwargs['cookies'] = request.cookies
kwargs['errors'] = request.errors
kwargs['instance_name'] = settings['general']['instance_name'] kwargs['instance_name'] = settings['general']['instance_name']
kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab') kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
@ -364,15 +367,16 @@ def render(template_name, override_theme=None, **kwargs):
@app.before_request @app.before_request
def pre_request(): def pre_request():
# merge GET, POST vars request.errors = []
preferences = Preferences(themes, categories.keys(), engines, plugins) preferences = Preferences(themes, categories.keys(), engines, plugins)
request.preferences = preferences
try: try:
preferences.parse_cookies(request.cookies) preferences.parse_cookies(request.cookies)
except: except:
# TODO throw error message to the user request.errors.append(gettext('Invalid settings, please edit your preferences'))
logger.warning('Invalid config')
request.preferences = preferences
# merge GET, POST vars
# request.form # request.form
request.form = dict(request.form.items()) request.form = dict(request.form.items())
for k, v in request.args.items(): for k, v in request.args.items():
@ -397,7 +401,7 @@ def index():
Supported outputs: html, json, csv, rss. Supported outputs: html, json, csv, rss.
""" """
if not request.args and not request.form: if request.form.get('q') is None:
return render( return render(
'index.html', 'index.html',
) )
@ -411,6 +415,8 @@ def index():
search = SearchWithPlugins(search_query, request) search = SearchWithPlugins(search_query, request)
result_container = search.search() result_container = search.search()
except: except:
request.errors.append(gettext('search error'))
logger.exception('search error')
return render( return render(
'index.html', 'index.html',
) )
@ -427,8 +433,10 @@ def index():
for result in results: for result in results:
if output_format == 'html': if output_format == 'html':
if 'content' in result and result['content']: if 'content' in result and result['content']:
result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8')) result['content'] = highlight_content(escape(result['content'][:1024]),
result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8')) search_query.query.encode('utf-8'))
result['title'] = highlight_content(escape(result['title'] or u''),
search_query.query.encode('utf-8'))
else: else:
if result.get('content'): if result.get('content'):
result['content'] = html_to_text(result['content']).strip() result['content'] = html_to_text(result['content']).strip()
@ -572,7 +580,7 @@ def preferences():
try: try:
request.preferences.parse_form(request.form) request.preferences.parse_form(request.form)
except ValidationException: except ValidationException:
# TODO use flash feature of flask request.errors.append(gettext('Invalid settings, please edit your preferences'))
return resp return resp
return request.preferences.save(resp) return request.preferences.save(resp)
@ -609,6 +617,7 @@ def preferences():
language_codes=language_codes, language_codes=language_codes,
engines_by_category=categories, engines_by_category=categories,
stats=stats, stats=stats,
answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
disabled_engines=disabled_engines, disabled_engines=disabled_engines,
autocomplete_backends=autocomplete_backends, autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()}, shortcuts={y: x for x, y in engine_shortcuts.items()},

@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase):
params = bing.request(query, dicto) params = bing.request(query, dicto)
self.assertTrue('url' in params) self.assertTrue('url' in params)
self.assertTrue(query in params['url']) self.assertTrue(query in params['url'])
self.assertTrue('language%3AFR' in params['url'])
self.assertTrue('bing.com' in params['url']) self.assertTrue('bing.com' in params['url'])
self.assertTrue('SRCHHPGUSR' in params['cookies'])
self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])
dicto['language'] = 'all' dicto['language'] = 'all'
params = bing.request(query, dicto) params = bing.request(query, dicto)
self.assertTrue('SRCHHPGUSR' in params['cookies']) self.assertTrue('language' not in params['url'])
self.assertTrue('en' in params['cookies']['SRCHHPGUSR'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, bing.response, None) self.assertRaises(AttributeError, bing.response, None)

@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase):
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['title'], 'Title of track')
self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042') self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')
self.assertEqual(results[0]['content'], 'Artist Name &bull; Album Title &bull; Title of track') self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
self.assertTrue('100' in results[0]['embedded']) self.assertTrue('100' in results[0]['embedded'])
json = r""" json = r"""

@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('n.jpg' in results[0]['thumbnail_src']) self.assertTrue('n.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = r""" json = r"""
@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('z.jpg' in results[0]['img_src']) self.assertTrue('z.jpg' in results[0]['img_src'])
self.assertTrue('z.jpg' in results[0]['thumbnail_src']) self.assertTrue('z.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = r""" json = r"""
@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('o.jpg' in results[0]['thumbnail_src']) self.assertTrue('o.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = r""" json = r"""

@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('k.jpg', results[0]['img_src'])
self.assertIn('n.jpg', results[0]['thumbnail_src']) self.assertIn('n.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['author'])
# no n size, only the z size # no n size, only the z size
json = """ json = """
@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['img_src'])
self.assertIn('z.jpg', results[0]['thumbnail_src']) self.assertIn('z.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['author'])
# no z or n size # no z or n size
json = """ json = """
@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('o.jpg', results[0]['img_src']) self.assertIn('o.jpg', results[0]['img_src'])
self.assertIn('o.jpg', results[0]['thumbnail_src']) self.assertIn('o.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['author'])
# no image test # no image test
json = """ json = """

@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['leech'], 1)
self.assertEqual(results[0]['filesize'], 449) self.assertEqual(results[0]['filesize'], 449)
@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(len(results), 5) self.assertEqual(len(results), 5)
self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['leech'], 1)
self.assertEqual(results[0]['files'], 4) self.assertEqual(results[0]['files'], 4)

@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase):
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], '[Type] Namespace test') self.assertEqual(results[0]['title'], '[Type] Namespace test')
self.assertEqual(results[0]['url'], 'http://url') self.assertEqual(results[0]['url'], 'http://url')
self.assertIn('Synopsis', results[0]['content'])
self.assertIn('Type', results[0]['content'])
self.assertIn('test', results[0]['content'])
self.assertIn('Description', results[0]['content']) self.assertIn('Description', results[0]['content'])
json = r""" json = r"""

@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase):
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['title'], 'Title of track')
self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa') self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa')
self.assertEqual(results[0]['content'], 'Artist Name &bull; Album Title &bull; Title of track') self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
self.assertIn('1000', results[0]['embedded']) self.assertIn('1000', results[0]['embedded'])
json = """ json = """

@ -0,0 +1,16 @@
# -*- coding: utf-8 -*-
from mock import Mock
from searx.answerers import answerers
from searx.testing import SearxTestCase
class AnswererTest(SearxTestCase):
def test_unicode_input(self):
query = Mock()
unicode_payload = u'árvíztűrő tükörfúrógép'
for answerer in answerers:
query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)
self.assertTrue(isinstance(answerer.answer(query), list))

@ -5,6 +5,7 @@ from mock import Mock
from urlparse import ParseResult from urlparse import ParseResult
from searx import webapp from searx import webapp
from searx.testing import SearxTestCase from searx.testing import SearxTestCase
from searx.search import Search
class ViewsTestCase(SearxTestCase): class ViewsTestCase(SearxTestCase):
@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase):
results_number=lambda: 3, results_number=lambda: 3,
results_length=lambda: len(self.test_results)) results_length=lambda: len(self.test_results))
webapp.Search.search = search_mock Search.search = search_mock
def get_current_theme_name_mock(override=None): def get_current_theme_name_mock(override=None):
return 'legacy' return 'legacy'

Loading…
Cancel
Save