From 1c7d8815fb54dd37ed6765560c223cb93976b6f4 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 3 Oct 2020 15:02:44 +0200 Subject: [PATCH 1/2] [mod] URL for the static file contains the sha1 * allow to cache the static file forever * avoid bugs when the static files are updated but not reloaded --- searx/webapp.py | 14 ++++++++------ searx/webutils.py | 33 +++++++++++++++++++++++++-------- tests/unit/test_webapp.py | 5 +++++ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/searx/webapp.py b/searx/webapp.py index df717a7b1..999b4f64b 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -16,7 +16,7 @@ from timeit import default_timer from html import escape from io import StringIO import typing -from typing import List, Dict, Iterable +from typing import List, Dict, Iterable, Optional import urllib import urllib.parse @@ -348,7 +348,7 @@ def code_highlighter(codelines, language=None): return html_code -def get_current_theme_name(override: str = None) -> str: +def get_current_theme_name(override: Optional[str] = None) -> str: """Returns theme name. Checks in this order: @@ -373,14 +373,16 @@ def get_result_template(theme_name: str, template_name: str): return 'result_templates/' + template_name -def url_for_theme(endpoint: str, override_theme: str = None, **values): +def url_for_theme(endpoint: str, override_theme: Optional[str] = None, **values): + suffix = "" if endpoint == 'static' and values.get('filename'): theme_name = get_current_theme_name(override=override_theme) filename_with_theme = "themes/{}/{}".format(theme_name, values['filename']) - if filename_with_theme in static_files: + file_hash = static_files.get(filename_with_theme) + if file_hash: values['filename'] = filename_with_theme - url = url_for(endpoint, **values) - return url + suffix = "?" + file_hash + return url_for(endpoint, **values) + suffix def proxify(url: str): diff --git a/searx/webutils.py b/searx/webutils.py index 0c0854dfc..5be721eec 100644 --- a/searx/webutils.py +++ b/searx/webutils.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import os +import pathlib import csv import hashlib import hmac import re import inspect import itertools -from typing import Iterable, List, Tuple +from typing import Iterable, List, Tuple, Dict from io import StringIO from codecs import getincrementalencoder @@ -58,13 +59,29 @@ def get_themes(templates_path): return themes -def get_static_files(static_path): - static_files = set() - static_path_length = len(static_path) + 1 - for directory, _, files in os.walk(static_path): - for filename in files: - f = os.path.join(directory[static_path_length:], filename) - static_files.add(f) +def get_hash_for_file(file: pathlib.Path) -> str: + m = hashlib.sha1() + with file.open('rb') as f: + m.update(f.read()) + return m.hexdigest() + + +def get_static_files(static_path: str) -> Dict[str, str]: + static_files: Dict[str, str] = {} + static_path_path = pathlib.Path(static_path) + + def walk(path: pathlib.Path): + for file in path.iterdir(): + if file.name.startswith('.'): + # ignore hidden file + continue + if file.is_file(): + static_files[str(file.relative_to(static_path_path))] = get_hash_for_file(file) + if file.is_dir() and file.name not in ('node_modules', 'src'): + # ignore "src" and "node_modules" directories + walk(file) + + walk(static_path_path) return static_files diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index aed6f2dd4..872b8d014 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -23,6 +23,11 @@ class ViewsTestCase(SearxTestCase): webapp.app.config['TESTING'] = True # to get better error messages self.app = webapp.app.test_client() + # remove sha for the static file + # so the tests don't have to care about the changing URLs + for k in webapp.static_files: + webapp.static_files[k] = None + # set some defaults test_results = [ { From 55833364401ed72c30b3e85c16824a75de6b547a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 28 Feb 2022 21:41:33 +0100 Subject: [PATCH 2/2] [docker] change static-expires to one year --- dockerfiles/uwsgi.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/uwsgi.ini b/dockerfiles/uwsgi.ini index fe518e887..f35e2308b 100644 --- a/dockerfiles/uwsgi.ini +++ b/dockerfiles/uwsgi.ini @@ -44,9 +44,9 @@ logto = /var/log/uwsgi/uwsgi.log add-header = Connection: close # uwsgi serves the static files -# expires set to one day as Flask does +# expires set to one year since there are hashes static-map = /static=/usr/local/searxng/searx/static -static-expires = /* 864000 +static-expires = /* 31557600 static-gzip-all = True offload-threads = %k