From 09e7ecdce22733be82bb141355c646ddec538714 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 22 Apr 2021 17:47:53 +0200 Subject: [PATCH] [mod] /stats : add reliability column and sort by column links --- searx/metrics/__init__.py | 79 ++++++++++++------ searx/static/themes/oscar/css/pointhi.css | 19 +++++ searx/static/themes/oscar/css/pointhi.min.css | Bin 10308 -> 10666 bytes .../themes/oscar/css/pointhi.min.css.map | Bin 5380 -> 5618 bytes .../oscar/src/less/pointhi/preferences.less | 15 ++++ searx/templates/oscar/stats.html | 28 +++++-- searx/webapp.py | 40 ++++++++- 7 files changed, 146 insertions(+), 35 deletions(-) diff --git a/searx/metrics/__init__.py b/searx/metrics/__init__.py index 9648e6215..5665ca63c 100644 --- a/searx/metrics/__init__.py +++ b/searx/metrics/__init__.py @@ -97,12 +97,12 @@ def initialize(engine_names=None): histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'total') -def get_engine_errors(engline_list): +def get_engine_errors(engline_name_list): result = {} engine_names = list(errors_per_engines.keys()) engine_names.sort() for engine_name in engine_names: - if engine_name not in engline_list: + if engine_name not in engline_name_list: continue error_stats = errors_per_engines[engine_name] @@ -126,61 +126,86 @@ def get_engine_errors(engline_list): return result -def to_percentage(stats, maxvalue): - for engine_stat in stats: - if maxvalue: - engine_stat['percentage'] = int(engine_stat['avg'] / maxvalue * 100) +def get_reliabilities(engline_name_list, checker_results): + reliabilities = {} + + engine_errors = get_engine_errors(engline_name_list) + + for engine_name in engline_name_list: + checker_result = checker_results.get(engine_name, {}) + checker_success = checker_result.get('success', True) + errors = engine_errors.get(engine_name) or [] + if counter('engine', engine_name, 'search', 'count', 'sent') == 0: + # no request + reliablity = None + elif checker_success and not errors: + reliablity = 100 + elif 'simple' in checker_result.get('errors', {}): + # the basic (simple) test doesn't work: the engine is broken accoding to the checker + # even if there is no exception + reliablity = 0 else: - engine_stat['percentage'] = 0 - return stats + reliablity = 100 - sum([error['percentage'] for error in errors if not error.get('secondary')]) + + reliabilities[engine_name] = { + 'reliablity': reliablity, + 'errors': errors, + 'checker': checker_results.get(engine_name, {}).get('errors', {}).keys(), + } + return reliabilities -def get_engines_stats(engine_list): +def round_or_none(number, digits): + return round(number, digits) if number else number + + +def get_engines_stats(engine_name_list): assert counter_storage is not None assert histogram_storage is not None list_time = [] max_time_total = max_result_count = None # noqa - for engine_name in engine_list: - successful_count = counter('engine', engine_name, 'search', 'count', 'successful') - if successful_count == 0: + for engine_name in engine_name_list: + sent_count = counter('engine', engine_name, 'search', 'count', 'sent') + if sent_count == 0: continue - result_count_sum = histogram('engine', engine_name, 'result', 'count').sum + successful_count = counter('engine', engine_name, 'search', 'count', 'successful') + time_total = histogram('engine', engine_name, 'time', 'total').percentage(50) time_http = histogram('engine', engine_name, 'time', 'http').percentage(50) time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80) time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80) time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95) time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95) - result_count = result_count_sum / float(successful_count) - if result_count: + result_count = histogram('engine', engine_name, 'result', 'count').percentage(50) + result_count_sum = histogram('engine', engine_name, 'result', 'count').sum + if successful_count and result_count_sum: score = counter('engine', engine_name, 'score') # noqa score_per_result = score / float(result_count_sum) else: score = score_per_result = 0.0 - max_time_total = max(time_total, max_time_total or 0) - max_result_count = max(result_count, max_result_count or 0) + max_time_total = max(time_total or 0, max_time_total or 0) + max_result_count = max(result_count or 0, max_result_count or 0) list_time.append({ - 'total': round(time_total, 1), - 'total_p80': round(time_total_p80, 1), - 'total_p95': round(time_total_p95, 1), - 'http': round(time_http, 1), - 'http_p80': round(time_http_p80, 1), - 'http_p95': round(time_http_p95, 1), 'name': engine_name, - 'processing': round(time_total - time_http, 1), - 'processing_p80': round(time_total_p80 - time_http_p80, 1), - 'processing_p95': round(time_total_p95 - time_http_p95, 1), + 'total': round_or_none(time_total, 1), + 'total_p80': round_or_none(time_total_p80, 1), + 'total_p95': round_or_none(time_total_p95, 1), + 'http': round_or_none(time_http, 1), + 'http_p80': round_or_none(time_http_p80, 1), + 'http_p95': round_or_none(time_http_p95, 1), + 'processing': round(time_total - time_http, 1) if time_total else None, + 'processing_p80': round(time_total_p80 - time_http_p80, 1) if time_total else None, + 'processing_p95': round(time_total_p95 - time_http_p95, 1) if time_total else None, 'score': score, 'score_per_result': score_per_result, 'result_count': result_count, }) - return { 'time': list_time, 'max_time': math.ceil(max_time_total or 0), diff --git a/searx/static/themes/oscar/css/pointhi.css b/searx/static/themes/oscar/css/pointhi.css index 64f612d79..99ed7b576 100644 --- a/searx/static/themes/oscar/css/pointhi.css +++ b/searx/static/themes/oscar/css/pointhi.css @@ -682,6 +682,7 @@ input[type=checkbox]:not(:checked) + .label_hide_if_checked + .label_hide_if_not padding: 0.5rem 1rem; margin: 0rem 0 0 2rem; border: 1px solid #ddd; + box-shadow: 2px 2px 2px 0px rgba(0, 0, 0, 0.1); background: white; font-size: 14px; font-weight: normal; @@ -756,3 +757,21 @@ td:hover .engine-tooltip, padding: 0.4rem 0; width: 1px; } +.stacked-bar-chart-serie1 { + display: flex; + flex-shrink: 0; + flex-grow: 0; + flex-basis: unset; + background: #5bc0de; + box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); + padding: 0.4rem 0; +} +.stacked-bar-chart-serie2 { + display: flex; + flex-shrink: 0; + flex-grow: 0; + flex-basis: unset; + background: #deb15b; + box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); + padding: 0.4rem 0; +} diff --git a/searx/static/themes/oscar/css/pointhi.min.css b/searx/static/themes/oscar/css/pointhi.min.css index 4332e4767e124fc129392541da678aab9e505dc5..80027ff8ee4a93e8596c635086be171af84ddbd0 100644 GIT binary patch delta 212 zcmX>Suqt@NbY->t3f5*K2A607DK2{Qv*} diff --git a/searx/static/themes/oscar/css/pointhi.min.css.map b/searx/static/themes/oscar/css/pointhi.min.css.map index abb30817f90796bb7f1ca29ce819551aff376df9..708e2f63a1ef87f924e997018ccd7804fc485d0b 100644 GIT binary patch delta 177 zcmZqC`lP)fNl4Vw(a}){LAZjrshcB(Rx)t|h26b$44fSEoG1H {% endblock %} {% block title %}{{ _('stats') }} - {% endblock %} + +{%- macro th_sort(column_order, column_name) -%} + {% if column_order==sort_order %} + {{ column_name }} {{ icon('chevron-down') }} + {% else %} + {{ column_name }} + {% endif %} +{%- endmacro -%} + {% block content %}

{{ _('Engine stats') }}

@@ -25,27 +33,33 @@ {% else %} - - - - + + + + + {% for engine_stat in engine_stats.get('time', []) %}
{{ _("Engine name") }}{{ _('Scores') }}{{ _('Number of results') }}{{ _('Response time') }}{{ th_sort('name', _("Engine name")) }}{{ th_sort('score', _('Scores')) }}{{ th_sort('result_count', _('Result count')) }}{{ th_sort('time', _('Response time')) }}{{ th_sort('reliability', _('Reliability')) }}
{{ engine_stat.name }} + {% if engine_stat.score %} {{ engine_stat.score|round(1) }} + {% endif %} + {%- if engine_stat.result_count -%} {{- engine_stat.result_count | int -}}{{- "" -}} {{- "" -}} + + {%- endif -%} + {%- if engine_stat.total -%} {{- engine_stat.total | round(1) -}}{{- "" -}}
+ {%- endif -%} + {{ engine_reliabilities.get(engine_stat.name, {}).get('reliablity') }} {% endfor %} diff --git a/searx/webapp.py b/searx/webapp.py index d2a1ad5f5..d917c16d4 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -93,7 +93,7 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES from searx.answerers import answerers from searx.network import stream as http_stream from searx.answerers import ask -from searx.metrics import get_engines_stats, get_engine_errors, histogram, counter +from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter # serve pages with HTTP/1.1 from werkzeug.serving import WSGIRequestHandler @@ -1073,11 +1073,47 @@ def image_proxy(): @app.route('/stats', methods=['GET']) def stats(): """Render engine statistics page.""" + checker_results = checker_get_result() + checker_results = checker_results['engines'] \ + if checker_results['status'] == 'ok' and 'engines' in checker_results else {} + filtered_engines = dict(filter(lambda kv: (kv[0], request.preferences.validate_token(kv[1])), engines.items())) engine_stats = get_engines_stats(filtered_engines) + engine_reliabilities = get_reliabilities(filtered_engines, checker_results) + + sort_order = request.args.get('sort', default='name', type=str) + + SORT_PARAMETERS = { + 'name': (False, 'name', ''), + 'score': (True, 'score', 0), + 'result_count': (True, 'result_count', 0), + 'time': (False, 'total', 0), + 'reliability': (False, 'reliability', 100), + } + + if sort_order not in SORT_PARAMETERS: + sort_order = 'name' + + reverse, key_name, default_value = SORT_PARAMETERS[sort_order] + + def get_key(engine_stat): + reliability = engine_reliabilities.get(engine_stat['name']).get('reliablity', 0) + reliability_order = 0 if reliability else 1 + if key_name == 'reliability': + key = reliability + reliability_order = 0 + else: + key = engine_stat.get(key_name) or default_value + if reverse: + reliability_order = 1 - reliability_order + return (reliability_order, key, engine_stat['name']) + + engine_stats['time'] = sorted(engine_stats['time'], reverse=reverse, key=get_key) return render( 'stats.html', - engine_stats=engine_stats + sort_order=sort_order, + engine_stats=engine_stats, + engine_reliabilities=engine_reliabilities, )