forked from Archives/searxng
[enh] checker: background check
See settings.yml for the options SIGUSR1 signal starts the checker. The result is available at /stats/checkerdependabot/pip/master/sphinx-6.1.3
parent
6e2872f436
commit
3a9f513521
@ -1 +1,4 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
from .impl import Checker
|
from .impl import Checker
|
||||||
|
from .background import initialize, get_result
|
||||||
|
@ -0,0 +1,106 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
|
||||||
|
from searx import logger, settings, searx_debug
|
||||||
|
from searx.exceptions import SearxSettingsException
|
||||||
|
from searx.search.processors import processors
|
||||||
|
from searx.search.checker import Checker
|
||||||
|
from searx.shared import schedule, storage
|
||||||
|
|
||||||
|
|
||||||
|
CHECKER_RESULT = 'CHECKER_RESULT'
|
||||||
|
running = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_interval(every, error_msg):
|
||||||
|
if isinstance(every, int):
|
||||||
|
every = (every, every)
|
||||||
|
if not isinstance(every, (tuple, list))\
|
||||||
|
or len(every) != 2\
|
||||||
|
or not isinstance(every[0], int)\
|
||||||
|
or not isinstance(every[1], int):
|
||||||
|
raise SearxSettingsException(error_msg, None)
|
||||||
|
return every
|
||||||
|
|
||||||
|
|
||||||
|
def _get_every():
|
||||||
|
every = settings.get('checker', {}).get('scheduling', {}).get('every', (300, 1800))
|
||||||
|
return _get_interval(every, 'checker.scheduling.every is not a int or list')
|
||||||
|
|
||||||
|
|
||||||
|
def get_result():
|
||||||
|
serialized_result = storage.get_str('CHECKER_RESULT')
|
||||||
|
if serialized_result is not None:
|
||||||
|
return json.loads(serialized_result)
|
||||||
|
|
||||||
|
|
||||||
|
def run():
|
||||||
|
if not running.acquire(blocking=False):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
logger.info('Starting checker')
|
||||||
|
result = {}
|
||||||
|
for name, processor in processors.items():
|
||||||
|
logger.debug('Checking %s engine', name)
|
||||||
|
checker = Checker(processor)
|
||||||
|
checker.run()
|
||||||
|
if checker.test_results.succesfull:
|
||||||
|
result[name] = {'status': True}
|
||||||
|
else:
|
||||||
|
result[name] = {'status': False, 'errors': checker.test_results.errors}
|
||||||
|
|
||||||
|
storage.set_str('CHECKER_RESULT', json.dumps(result))
|
||||||
|
logger.info('Check done')
|
||||||
|
finally:
|
||||||
|
running.release()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_with_delay():
|
||||||
|
every = _get_every()
|
||||||
|
delay = random.randint(0, every[1] - every[0])
|
||||||
|
logger.debug('Start checker in %i seconds', delay)
|
||||||
|
time.sleep(delay)
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
||||||
|
def _start_scheduling():
|
||||||
|
every = _get_every()
|
||||||
|
schedule(every[0], _run_with_delay)
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
||||||
|
def _signal_handler(signum, frame):
|
||||||
|
t = threading.Thread(target=run)
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
|
||||||
|
def initialize():
|
||||||
|
logger.info('Send SIGUSR1 signal to pid %i to start the checker', os.getpid())
|
||||||
|
signal.signal(signal.SIGUSR1, _signal_handler)
|
||||||
|
|
||||||
|
# special case when debug is activate
|
||||||
|
if searx_debug and settings.get('checker', {}).get('off_when_debug', True):
|
||||||
|
logger.info('debug mode: checker is disabled')
|
||||||
|
return
|
||||||
|
|
||||||
|
# check value of checker.scheduling.every now
|
||||||
|
scheduling = settings.get('checker', {}).get('scheduling', None)
|
||||||
|
if scheduling is None or not scheduling:
|
||||||
|
logger.info('Checker scheduler is disabled')
|
||||||
|
return
|
||||||
|
|
||||||
|
#
|
||||||
|
start_after = scheduling.get('start_after', (300, 1800))
|
||||||
|
start_after = _get_interval(start_after, 'checker.scheduling.start_after is not a int or list')
|
||||||
|
delay = random.randint(start_after[0], start_after[1])
|
||||||
|
logger.info('Start checker in %i seconds', delay)
|
||||||
|
t = threading.Timer(delay, _start_scheduling)
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
@ -0,0 +1,69 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import typing
|
||||||
|
|
||||||
|
|
||||||
|
class EngineRef:
|
||||||
|
|
||||||
|
__slots__ = 'name', 'category'
|
||||||
|
|
||||||
|
def __init__(self, name: str, category: str):
|
||||||
|
self.name = name
|
||||||
|
self.category = category
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "EngineRef({!r}, {!r})".format(self.name, self.category)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.name == other.name and self.category == other.category
|
||||||
|
|
||||||
|
def __hash__(self):
|
||||||
|
return hash((self.name, self.category))
|
||||||
|
|
||||||
|
|
||||||
|
class SearchQuery:
|
||||||
|
"""container for all the search parameters (query, language, etc...)"""
|
||||||
|
|
||||||
|
__slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\
|
||||||
|
'timeout_limit', 'external_bang'
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
query: str,
|
||||||
|
engineref_list: typing.List[EngineRef],
|
||||||
|
lang: str='all',
|
||||||
|
safesearch: int=0,
|
||||||
|
pageno: int=1,
|
||||||
|
time_range: typing.Optional[str]=None,
|
||||||
|
timeout_limit: typing.Optional[float]=None,
|
||||||
|
external_bang: typing.Optional[str]=None):
|
||||||
|
self.query = query
|
||||||
|
self.engineref_list = engineref_list
|
||||||
|
self.lang = lang
|
||||||
|
self.safesearch = safesearch
|
||||||
|
self.pageno = pageno
|
||||||
|
self.time_range = time_range
|
||||||
|
self.timeout_limit = timeout_limit
|
||||||
|
self.external_bang = external_bang
|
||||||
|
|
||||||
|
@property
|
||||||
|
def categories(self):
|
||||||
|
return list(set(map(lambda engineref: engineref.category, self.engineref_list)))
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "SearchQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
|
||||||
|
format(self.query, self.engineref_list, self.lang, self.safesearch,
|
||||||
|
self.pageno, self.time_range, self.timeout_limit, self.external_bang)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.query == other.query\
|
||||||
|
and self.engineref_list == other.engineref_list\
|
||||||
|
and self.lang == other.lang\
|
||||||
|
and self.safesearch == other.safesearch\
|
||||||
|
and self.pageno == other.pageno\
|
||||||
|
and self.time_range == other.time_range\
|
||||||
|
and self.timeout_limit == other.timeout_limit\
|
||||||
|
and self.external_bang == other.external_bang
|
||||||
|
|
||||||
|
def __hash__(self):
|
||||||
|
return hash((self.query, tuple(self.engineref_list), self.lang, self.safesearch, self.pageno, self.time_range,
|
||||||
|
self.timeout_limit, self.external_bang))
|
Loading…
Reference in New Issue