mirror of https://github.com/searxng/searxng
Merge remote-tracking branch 'remotes/origin/result-handling-refactor'
commit
d8ef98371b
@ -0,0 +1,239 @@
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from operator import itemgetter
|
||||
from threading import RLock
|
||||
from urlparse import urlparse, unquote
|
||||
from searx.engines import engines
|
||||
|
||||
CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile('[,;:!?\./\\\\ ()-_]', re.M | re.U)
|
||||
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
|
||||
|
||||
|
||||
# return the meaningful length of the content for a result
|
||||
def result_content_len(content):
|
||||
if isinstance(content, basestring):
|
||||
return len(CONTENT_LEN_IGNORED_CHARS_REGEX.sub('', content))
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def compare_urls(url_a, url_b):
|
||||
if url_a.netloc != url_b.netloc or url_a.query != url_b.query:
|
||||
return False
|
||||
|
||||
# remove / from the end of the url if required
|
||||
path_a = url_a.path[:-1]\
|
||||
if url_a.path.endswith('/')\
|
||||
else url_a.path
|
||||
path_b = url_b.path[:-1]\
|
||||
if url_b.path.endswith('/')\
|
||||
else url_b.path
|
||||
|
||||
return unquote(path_a) == unquote(path_b)
|
||||
|
||||
|
||||
def merge_two_infoboxes(infobox1, infobox2):
|
||||
if 'urls' in infobox2:
|
||||
urls1 = infobox1.get('urls', None)
|
||||
if urls1 is None:
|
||||
urls1 = []
|
||||
infobox1.set('urls', urls1)
|
||||
|
||||
urlSet = set()
|
||||
for url in infobox1.get('urls', []):
|
||||
urlSet.add(url.get('url', None))
|
||||
|
||||
for url in infobox2.get('urls', []):
|
||||
if url.get('url', None) not in urlSet:
|
||||
urls1.append(url)
|
||||
|
||||
if 'attributes' in infobox2:
|
||||
attributes1 = infobox1.get('attributes', None)
|
||||
if attributes1 is None:
|
||||
attributes1 = []
|
||||
infobox1.set('attributes', attributes1)
|
||||
|
||||
attributeSet = set()
|
||||
for attribute in infobox1.get('attributes', []):
|
||||
if attribute.get('label', None) not in attributeSet:
|
||||
attributeSet.add(attribute.get('label', None))
|
||||
|
||||
for attribute in infobox2.get('attributes', []):
|
||||
attributes1.append(attribute)
|
||||
|
||||
if 'content' in infobox2:
|
||||
content1 = infobox1.get('content', None)
|
||||
content2 = infobox2.get('content', '')
|
||||
if content1 is not None:
|
||||
if result_content_len(content2) > result_content_len(content1):
|
||||
infobox1['content'] = content2
|
||||
else:
|
||||
infobox1.set('content', content2)
|
||||
|
||||
|
||||
def result_score(result):
|
||||
weight = 1.0
|
||||
|
||||
for result_engine in result['engines']:
|
||||
if hasattr(engines[result_engine], 'weight'):
|
||||
weight *= float(engines[result_engine].weight)
|
||||
|
||||
occurences = len(result['positions'])
|
||||
|
||||
return sum((occurences * weight) / position for position in result['positions'])
|
||||
|
||||
|
||||
class ResultContainer(object):
|
||||
"""docstring for ResultContainer"""
|
||||
def __init__(self):
|
||||
super(ResultContainer, self).__init__()
|
||||
self.results = defaultdict(list)
|
||||
self._merged_results = []
|
||||
self.infoboxes = []
|
||||
self._infobox_ids = {}
|
||||
self.suggestions = set()
|
||||
self.answers = set()
|
||||
|
||||
def extend(self, engine_name, results):
|
||||
for result in list(results):
|
||||
if 'suggestion' in result:
|
||||
self.suggestions.add(result['suggestion'])
|
||||
results.remove(result)
|
||||
elif 'answer' in result:
|
||||
self.answers.add(result['suggestion'])
|
||||
results.remove(result)
|
||||
elif 'infobox' in result:
|
||||
self._merge_infobox(result)
|
||||
results.remove(result)
|
||||
|
||||
with RLock():
|
||||
engines[engine_name].stats['search_count'] += 1
|
||||
engines[engine_name].stats['result_count'] += len(results)
|
||||
|
||||
if not results:
|
||||
return
|
||||
|
||||
self.results[engine_name].extend(results)
|
||||
|
||||
for i, result in enumerate(results):
|
||||
position = i + 1
|
||||
self._merge_result(result, position)
|
||||
|
||||
def _merge_infobox(self, infobox):
|
||||
add_infobox = True
|
||||
infobox_id = infobox.get('id', None)
|
||||
if infobox_id is not None:
|
||||
existingIndex = self._infobox_ids.get(infobox_id, None)
|
||||
if existingIndex is not None:
|
||||
merge_two_infoboxes(self.infoboxes[existingIndex], infobox)
|
||||
add_infobox = False
|
||||
|
||||
if add_infobox:
|
||||
self.infoboxes.append(infobox)
|
||||
self._infobox_ids[infobox_id] = len(self.infoboxes) - 1
|
||||
|
||||
def _merge_result(self, result, position):
|
||||
result['parsed_url'] = urlparse(result['url'])
|
||||
|
||||
# if the result has no scheme, use http as default
|
||||
if not result['parsed_url'].scheme:
|
||||
result['parsed_url'] = result['parsed_url']._replace(scheme="http")
|
||||
|
||||
result['host'] = result['parsed_url'].netloc
|
||||
|
||||
if result['host'].startswith('www.'):
|
||||
result['host'] = result['host'].replace('www.', '', 1)
|
||||
|
||||
result['engines'] = [result['engine']]
|
||||
|
||||
# strip multiple spaces and cariage returns from content
|
||||
if result.get('content'):
|
||||
result['content'] = WHITESPACE_REGEX.sub(' ', result['content'])
|
||||
|
||||
# check for duplicates
|
||||
duplicated = False
|
||||
for merged_result in self._merged_results:
|
||||
if compare_urls(result['parsed_url'], merged_result['parsed_url'])\
|
||||
and result.get('template') == merged_result.get('template'):
|
||||
duplicated = merged_result
|
||||
break
|
||||
|
||||
# merge duplicates together
|
||||
if duplicated:
|
||||
# using content with more text
|
||||
if result_content_len(result.get('content', '')) >\
|
||||
result_content_len(duplicated.get('content', '')):
|
||||
duplicated['content'] = result['content']
|
||||
|
||||
# add the new position
|
||||
duplicated['positions'].append(position)
|
||||
|
||||
# add engine to list of result-engines
|
||||
duplicated['engines'].append(result['engine'])
|
||||
|
||||
# using https if possible
|
||||
if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
|
||||
duplicated['url'] = result['parsed_url'].geturl()
|
||||
duplicated['parsed_url'] = result['parsed_url']
|
||||
|
||||
# if there is no duplicate found, append result
|
||||
else:
|
||||
result['positions'] = [position]
|
||||
with RLock():
|
||||
self._merged_results.append(result)
|
||||
|
||||
def get_ordered_results(self):
|
||||
for result in self._merged_results:
|
||||
score = result_score(result)
|
||||
result['score'] = score
|
||||
with RLock():
|
||||
for result_engine in result['engines']:
|
||||
engines[result_engine].stats['score_count'] += score
|
||||
|
||||
results = sorted(self._merged_results, key=itemgetter('score'), reverse=True)
|
||||
|
||||
# pass 2 : group results by category and template
|
||||
gresults = []
|
||||
categoryPositions = {}
|
||||
|
||||
for i, res in enumerate(results):
|
||||
# FIXME : handle more than one category per engine
|
||||
category = engines[res['engine']].categories[0] + ':' + ''\
|
||||
if 'template' not in res\
|
||||
else res['template']
|
||||
|
||||
current = None if category not in categoryPositions\
|
||||
else categoryPositions[category]
|
||||
|
||||
# group with previous results using the same category
|
||||
# if the group can accept more result and is not too far
|
||||
# from the current position
|
||||
if current is not None and (current['count'] > 0)\
|
||||
and (len(gresults) - current['index'] < 20):
|
||||
# group with the previous results using
|
||||
# the same category with this one
|
||||
index = current['index']
|
||||
gresults.insert(index, res)
|
||||
|
||||
# update every index after the current one
|
||||
# (including the current one)
|
||||
for k in categoryPositions:
|
||||
v = categoryPositions[k]['index']
|
||||
if v >= index:
|
||||
categoryPositions[k]['index'] = v + 1
|
||||
|
||||
# update this category
|
||||
current['count'] -= 1
|
||||
|
||||
else:
|
||||
# same category
|
||||
gresults.append(res)
|
||||
|
||||
# update categoryIndex
|
||||
categoryPositions[category] = {'index': len(gresults), 'count': 8}
|
||||
|
||||
# return gresults
|
||||
return gresults
|
||||
|
||||
def results_length(self):
|
||||
return len(self._merged_results)
|
@ -1 +1,88 @@
|
||||
html{position:relative;min-height:100%}body{margin-bottom:80px}.footer{position:absolute;bottom:0;width:100%;height:60px}input[type=checkbox]:checked+.label_hide_if_checked,input[type=checkbox]:checked+.label_hide_if_not_checked+.label_hide_if_checked{display:none}input[type=checkbox]:not(:checked)+.label_hide_if_not_checked,input[type=checkbox]:not(:checked)+.label_hide_if_checked+.label_hide_if_not_checked{display:none}.result_header{margin-bottom:5px;margin-top:20px}.result_header .favicon{margin-bottom:-3px}.result_header a{vertical-align:bottom}.result_header a .highlight{font-weight:bold}.result-content{margin-top:5px;word-wrap:break-word}.result-content .highlight{font-weight:bold}.result-default{clear:both}.result-images{float:left !important}.img-thumbnail{margin:5px;max-height:128px;min-height:128px}.result-videos{clear:both}.result-torrents{clear:both}.result-map{clear:both}.result-code{clear:both}.suggestion_item{margin:2px 5px}.result_download{margin-right:5px}#pagination{margin-top:30px;padding-bottom:50px}.infobox .infobox_part{margin-bottom:20px;word-wrap:break-word}.infobox .infobox_part:last-child{margin-bottom:0}.search_categories{margin:10px 0;text-transform:capitalize}.cursor-text{cursor:text !important}.cursor-pointer{cursor:pointer !important}.highlight .hll{background-color:#ffc}.highlight{background:#f8f8f8}.highlight .c{color:#408080;font-style:italic}.highlight .err{border:1px solid #f00}.highlight .k{color:#008000;font-weight:bold}.highlight .o{color:#666}.highlight .cm{color:#408080;font-style:italic}.highlight .cp{color:#bc7a00}.highlight .c1{color:#408080;font-style:italic}.highlight .cs{color:#408080;font-style:italic}.highlight .gd{color:#a00000}.highlight .ge{font-style:italic}.highlight .gr{color:#f00}.highlight .gh{color:#000080;font-weight:bold}.highlight .gi{color:#00a000}.highlight .go{color:#888}.highlight .gp{color:#000080;font-weight:bold}.highlight .gs{font-weight:bold}.highlight .gu{color:#800080;font-weight:bold}.highlight .gt{color:#04d}.highlight .kc{color:#008000;font-weight:bold}.highlight .kd{color:#008000;font-weight:bold}.highlight .kn{color:#008000;font-weight:bold}.highlight .kp{color:#008000}.highlight .kr{color:#008000;font-weight:bold}.highlight .kt{color:#b00040}.highlight .m{color:#666}.highlight .s{color:#ba2121}.highlight .na{color:#7d9029}.highlight .nb{color:#008000}.highlight .nc{color:#00f;font-weight:bold}.highlight .no{color:#800}.highlight .nd{color:#a2f}.highlight .ni{color:#999;font-weight:bold}.highlight .ne{color:#d2413a;font-weight:bold}.highlight .nf{color:#00f}.highlight .nl{color:#a0a000}.highlight .nn{color:#00f;font-weight:bold}.highlight .nt{color:#008000;font-weight:bold}.highlight .nv{color:#19177c}.highlight .ow{color:#a2f;font-weight:bold}.highlight .w{color:#bbb}.highlight .mf{color:#666}.highlight .mh{color:#666}.highlight .mi{color:#666}.highlight .mo{color:#666}.highlight .sb{color:#ba2121}.highlight .sc{color:#ba2121}.highlight .sd{color:#ba2121;font-style:italic}.highlight .s2{color:#ba2121}.highlight .se{color:#b62;font-weight:bold}.highlight .sh{color:#ba2121}.highlight .si{color:#b68;font-weight:bold}.highlight .sx{color:#008000}.highlight .sr{color:#b68}.highlight .s1{color:#ba2121}.highlight .ss{color:#19177c}.highlight .bp{color:#008000}.highlight .vc{color:#19177c}.highlight .vg{color:#19177c}.highlight .vi{color:#19177c}.highlight .il{color:#666}.highlight .lineno{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;cursor:default}.highlight .lineno::selection{background:transparent}.highlight .lineno::-moz-selection{background:transparent}
|
||||
html{position:relative;min-height:100%}
|
||||
body{margin-bottom:80px}
|
||||
.footer{position:absolute;bottom:0;width:100%;height:60px}
|
||||
input[type=checkbox]:checked+.label_hide_if_checked,input[type=checkbox]:checked+.label_hide_if_not_checked+.label_hide_if_checked{display:none}
|
||||
input[type=checkbox]:not(:checked)+.label_hide_if_not_checked,input[type=checkbox]:not(:checked)+.label_hide_if_checked+.label_hide_if_not_checked{display:none}
|
||||
.result_header{margin-bottom:5px;margin-top:20px}.result_header .favicon{margin-bottom:-3px}
|
||||
.result_header a{vertical-align:bottom}.result_header a .highlight{font-weight:bold}
|
||||
.result-content{margin-top:5px;word-wrap:break-word}.result-content .highlight{font-weight:bold}
|
||||
.result-default{clear:both}
|
||||
.result-images{float:left !important}
|
||||
.img-thumbnail{margin:5px;max-height:128px;min-height:128px}
|
||||
.result-videos{clear:both}
|
||||
.result-torrents{clear:both}
|
||||
.result-map{clear:both}
|
||||
.result-code{clear:both}
|
||||
.suggestion_item{margin:2px 5px}
|
||||
.result_download{margin-right:5px}
|
||||
#pagination{margin-top:30px;padding-bottom:50px}
|
||||
.label-default{color:#aaa;background:#fff}
|
||||
.infobox .infobox_part{margin-bottom:20px;word-wrap:break-word}
|
||||
.infobox .infobox_part:last-child{margin-bottom:0}
|
||||
.search_categories{margin:10px 0;text-transform:capitalize}
|
||||
.cursor-text{cursor:text !important}
|
||||
.cursor-pointer{cursor:pointer !important}
|
||||
.highlight .hll{background-color:#ffc}
|
||||
.highlight{background:#f8f8f8}
|
||||
.highlight .c{color:#408080;font-style:italic}
|
||||
.highlight .err{border:1px solid #f00}
|
||||
.highlight .k{color:#008000;font-weight:bold}
|
||||
.highlight .o{color:#666}
|
||||
.highlight .cm{color:#408080;font-style:italic}
|
||||
.highlight .cp{color:#bc7a00}
|
||||
.highlight .c1{color:#408080;font-style:italic}
|
||||
.highlight .cs{color:#408080;font-style:italic}
|
||||
.highlight .gd{color:#a00000}
|
||||
.highlight .ge{font-style:italic}
|
||||
.highlight .gr{color:#f00}
|
||||
.highlight .gh{color:#000080;font-weight:bold}
|
||||
.highlight .gi{color:#00a000}
|
||||
.highlight .go{color:#888}
|
||||
.highlight .gp{color:#000080;font-weight:bold}
|
||||
.highlight .gs{font-weight:bold}
|
||||
.highlight .gu{color:#800080;font-weight:bold}
|
||||
.highlight .gt{color:#04d}
|
||||
.highlight .kc{color:#008000;font-weight:bold}
|
||||
.highlight .kd{color:#008000;font-weight:bold}
|
||||
.highlight .kn{color:#008000;font-weight:bold}
|
||||
.highlight .kp{color:#008000}
|
||||
.highlight .kr{color:#008000;font-weight:bold}
|
||||
.highlight .kt{color:#b00040}
|
||||
.highlight .m{color:#666}
|
||||
.highlight .s{color:#ba2121}
|
||||
.highlight .na{color:#7d9029}
|
||||
.highlight .nb{color:#008000}
|
||||
.highlight .nc{color:#00f;font-weight:bold}
|
||||
.highlight .no{color:#800}
|
||||
.highlight .nd{color:#a2f}
|
||||
.highlight .ni{color:#999;font-weight:bold}
|
||||
.highlight .ne{color:#d2413a;font-weight:bold}
|
||||
.highlight .nf{color:#00f}
|
||||
.highlight .nl{color:#a0a000}
|
||||
.highlight .nn{color:#00f;font-weight:bold}
|
||||
.highlight .nt{color:#008000;font-weight:bold}
|
||||
.highlight .nv{color:#19177c}
|
||||
.highlight .ow{color:#a2f;font-weight:bold}
|
||||
.highlight .w{color:#bbb}
|
||||
.highlight .mf{color:#666}
|
||||
.highlight .mh{color:#666}
|
||||
.highlight .mi{color:#666}
|
||||
.highlight .mo{color:#666}
|
||||
.highlight .sb{color:#ba2121}
|
||||
.highlight .sc{color:#ba2121}
|
||||
.highlight .sd{color:#ba2121;font-style:italic}
|
||||
.highlight .s2{color:#ba2121}
|
||||
.highlight .se{color:#b62;font-weight:bold}
|
||||
.highlight .sh{color:#ba2121}
|
||||
.highlight .si{color:#b68;font-weight:bold}
|
||||
.highlight .sx{color:#008000}
|
||||
.highlight .sr{color:#b68}
|
||||
.highlight .s1{color:#ba2121}
|
||||
.highlight .ss{color:#19177c}
|
||||
.highlight .bp{color:#008000}
|
||||
.highlight .vc{color:#19177c}
|
||||
.highlight .vg{color:#19177c}
|
||||
.highlight .vi{color:#19177c}
|
||||
.highlight .il{color:#666}
|
||||
.highlight .lineno{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;cursor:default}.highlight .lineno::selection{background:transparent;}
|
||||
.highlight .lineno::-moz-selection{background:transparent;}
|
||||
|
@ -0,0 +1,41 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from searx.results import ResultContainer
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
def fake_result(url='https://aa.bb/cc?dd=ee#ff',
|
||||
title='aaa',
|
||||
content='bbb',
|
||||
engine='wikipedia', **kwargs):
|
||||
result = {'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'engine': engine}
|
||||
result.update(kwargs)
|
||||
return result
|
||||
|
||||
|
||||
# TODO
|
||||
class ResultContainerTestCase(SearxTestCase):
|
||||
|
||||
def test_empty(self):
|
||||
c = ResultContainer()
|
||||
self.assertEqual(c.get_ordered_results(), [])
|
||||
|
||||
def test_one_result(self):
|
||||
c = ResultContainer()
|
||||
c.extend('wikipedia', [fake_result()])
|
||||
self.assertEqual(c.results_length(), 1)
|
||||
|
||||
def test_one_suggestion(self):
|
||||
c = ResultContainer()
|
||||
c.extend('wikipedia', [fake_result(suggestion=True)])
|
||||
self.assertEqual(len(c.suggestions), 1)
|
||||
self.assertEqual(c.results_length(), 0)
|
||||
|
||||
def test_result_merge(self):
|
||||
c = ResultContainer()
|
||||
c.extend('wikipedia', [fake_result()])
|
||||
c.extend('wikidata', [fake_result(), fake_result(url='https://example.com/')])
|
||||
self.assertEqual(c.results_length(), 2)
|
@ -1,25 +1,10 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from searx.search import score_results
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
def fake_result(url='https://aa.bb/cc?dd=ee#ff',
|
||||
title='aaa',
|
||||
content='bbb',
|
||||
engine='wikipedia'):
|
||||
return {'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'engine': engine}
|
||||
# TODO
|
||||
class SearchTestCase(SearxTestCase):
|
||||
|
||||
|
||||
class ScoreResultsTestCase(SearxTestCase):
|
||||
|
||||
def test_empty(self):
|
||||
self.assertEqual(score_results(dict()), [])
|
||||
|
||||
def test_urlparse(self):
|
||||
results = score_results(dict(a=[fake_result(url='https://aa.bb/cc?dd=ee#ff')]))
|
||||
parsed_url = results[0]['parsed_url']
|
||||
self.assertEqual(parsed_url.query, 'dd=ee')
|
||||
def test_(self):
|
||||
pass
|
||||
|
Loading…
Reference in New Issue