From f9c5727ddc74b9ee3bb95225c30f57c7aeb14806 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 7 Jan 2024 18:33:34 +0100 Subject: [PATCH] [mod] get rid of ./utils/brand.env and its workflow All the environments defined in ./utils/brand.env are generated on the fly, so there is no longer a need to define the brand environment in this file and all the workflows to handle this file. Signed-off-by: Markus Heiser --- .github/workflows/integration.yml | 1 - Makefile | 3 +- docs/admin/settings/settings_server.rst | 10 +--- docs/dev/makefile.rst | 73 +------------------------ manage | 26 --------- searx/settings.yml | 4 +- utils/brand.env | 5 -- utils/brand.sh | 32 +++++++++++ utils/build_env.py | 61 --------------------- utils/filtron.sh | 2 - utils/get_setting.py | 11 ---- utils/lxc.sh | 2 - utils/morty.sh | 2 - utils/searx.sh | 2 - utils/searxng.sh | 11 ++-- utils/searxng_check.py | 9 +++ 16 files changed, 53 insertions(+), 201 deletions(-) delete mode 100644 utils/brand.env create mode 100755 utils/brand.sh delete mode 100644 utils/build_env.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8836aeefd..27b698cb6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -163,7 +163,6 @@ jobs: - name: Update transations id: update run: | - git restore utils/brand.env make V=1 weblate.push.translations dockers: diff --git a/Makefile b/Makefile index fbed8a512..fa0753fff 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ test.shell: $(Q)shellcheck -x -s dash \ dockerfiles/docker-entrypoint.sh $(Q)shellcheck -x -s bash \ - utils/brand.env \ + utils/brand.sh \ $(MTOOLS) \ utils/lib.sh \ utils/lib_sxng*.sh \ @@ -74,7 +74,6 @@ test.shell: # wrap ./manage script -MANAGE += buildenv MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean diff --git a/docs/admin/settings/settings_server.rst b/docs/admin/settings/settings_server.rst index e4e66ee2f..b1b3a14f7 100644 --- a/docs/admin/settings/settings_server.rst +++ b/docs/admin/settings/settings_server.rst @@ -21,17 +21,13 @@ X-Robots-Tag : noindex, nofollow Referrer-Policy : no-referrer -``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv ` +``base_url`` : ``$SEARXNG_URL`` The base URL where SearXNG is deployed. Used to create correct inbound links. - If you change the value, don't forget to rebuild instance's environment - (:ref:`utils/brand.env `) -``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS`` :ref:`buildenv ` +``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS`` Port number and *bind address* of the SearXNG web application if you run it directly using ``python searx/webapp.py``. Doesn't apply to a SearXNG - services running behind a proxy and using socket communications. If you - change the value, don't forget to rebuild instance's environment - (:ref:`utils/brand.env `) + services running behind a proxy and using socket communications. ``secret_key`` : ``$SEARXNG_SECRET`` Used for cryptography purpose. diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index 8f29dd252..3c3b2bf3b 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -61,13 +61,9 @@ working tree and release a ``make install`` to get a virtualenv with a $ make install PYENV [virtualenv] installing ./requirements*.txt into local/py3 ... - PYENV OK PYENV [install] pip install -e 'searx[test]' ... - Successfully installed argparse-1.4.0 searx - BUILDENV INFO:searx:load the default settings from ./searx/settings.yml - BUILDENV INFO:searx:Initialisation done - BUILDENV build utils/brand.env + Successfully installed searxng-2023.7.19+a446dea1b If you release ``make install`` multiple times the installation will only rebuild if the sha256 sum of the *requirement files* fails. With other words: @@ -82,13 +78,9 @@ the check fails if you edit the requirements listed in ... PYENV [virtualenv] installing ./requirements*.txt into local/py3 ... - PYENV OK PYENV [install] pip install -e 'searx[test]' ... - Successfully installed argparse-1.4.0 searx - BUILDENV INFO:searx:load the default settings from ./searx/settings.yml - BUILDENV INFO:searx:Initialisation done - BUILDENV build utils/brand.env + Successfully installed searxng-2023.7.19+a446dea1b .. sidebar:: drop environment @@ -98,67 +90,6 @@ the check fails if you edit the requirements listed in If you think, something goes wrong with your ./local environment or you change the :origin:`setup.py` file, you have to call :ref:`make clean`. -.. _make buildenv: - -``make buildenv`` -================= - -Rebuild instance's environment with the modified settings from the -:ref:`settings brand` and :ref:`settings server` section of your -:ref:`settings.yml `. - - What is the :origin:`utils/brand.env` needed for and why do you need to rebuild - it if necessary? - - Short answer: :ref:`installation and maintenance ` - scripts are running outside of instance's runtime environment and need some - values defined in the runtime environment. - -All the SearXNG setups are centralized in the :ref:`settings.yml` file. This -setup is available as long we are in a *installed instance*. E.g. the -*installed instance* on the server or the *installed developer instance* at -``./local`` (the later one is created by a :ref:`make install ` or -:ref:`make run `). - -Tasks running outside of an *installed instance*, especially :ref:`installation -and maintenance ` tasks running at (pre-) installation time -do not have access to the SearXNG setup (from a *installed instance*). Those -tasks need a *build environment*. - -The ``make buildenv`` target will update the *build environment* in: - -- :origin:`utils/brand.env` - -Tasks running outside of an *installed instance*, need the following settings -from the YAML configuration: - -- ``SEARXNG_URL`` from :ref:`server.base_url ` (aka - ``PUBLIC_URL``) -- ``SEARXNG_BIND_ADDRESS`` from :ref:`server.bind_address ` -- ``SEARXNG_PORT`` from :ref:`server.port ` - -The ``GIT_URL`` and ``GIT_BRANCH`` in the origin:`utils/brand.env` file, are -read from the git VCS and the branch that is checked out when ``make -buildenv`` command runs. - -.. _brand: - -**I would like to create my own brand, how should I proceed?** - -Create a remote branch (``example.org``), checkout the remote branch (on your -local developer desktop) and in the :origin:`searx/settings.yml` file in the -:ref:`settings server` section set ``base_url``. Run ``make buildenv`` and -create a commit for your brand. - -On your server you clone the branch (``example.org``) into your HOME folder -``~`` from where you run the :ref:`installation ` and -:ref:`maintenance ` task. - -To upgrade you brand, rebase on SearXNG's master branch (on your local -developer desktop), force push it to your remote branch. Go to your server, do -a force pull and run :ref:`sudo -H ./utils/searxng.sh instance update `. - .. _make node.env: Node.js environment (``make node.env``) diff --git a/manage b/manage index 49bfca4b0..7e60cea3f 100755 --- a/manage +++ b/manage @@ -83,8 +83,6 @@ PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc" help() { nvm.help cat <&1 \ - | prefix_stdout "${_Blue}BUILDENV${_creset} " - ) - return "${PIPESTATUS[0]}" -} - docker.push() { docker.build push } @@ -345,7 +320,6 @@ pyenv.install() { pyenv build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}" - buildenv ) local exit_val=$? if [ ! $exit_val -eq 0 ]; then diff --git a/searx/settings.yml b/searx/settings.yml index c00ab10fa..5e70b7803 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -66,9 +66,7 @@ search: - html server: - # If you change port, bind_address or base_url don't forget to rebuild - # instance's environment (make buildenv). Is overwritten by ${SEARXNG_PORT} - # and ${SEARXNG_BIND_ADDRESS} + # Is overwritten by ${SEARXNG_PORT} and ${SEARXNG_BIND_ADDRESS} port: 8888 bind_address: "127.0.0.1" # public URL of the instance, to ensure correct inbound links. Is overwritten diff --git a/utils/brand.env b/utils/brand.env deleted file mode 100644 index 31afce53c..000000000 --- a/utils/brand.env +++ /dev/null @@ -1,5 +0,0 @@ -export SEARXNG_URL='' -export SEARXNG_PORT='8888' -export SEARXNG_BIND_ADDRESS='127.0.0.1' -export GIT_URL='https://github.com/searxng/searxng' -export GIT_BRANCH='master' diff --git a/utils/brand.sh b/utils/brand.sh new file mode 100755 index 000000000..1749efb31 --- /dev/null +++ b/utils/brand.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +. /dev/null + +build.env.export() { + GIT_BRANCH="$(git branch | grep '\*' | cut -d' ' -f2-)" + GIT_REMOTE="$(git config "branch.${GIT_BRANCH}.remote")" + GIT_URL="$(git config --get "remote.${GIT_REMOTE}.url")" + if [[ "${GIT_URL}" == git@* ]]; then + GIT_URL="${GIT_URL/://}" + GIT_URL="${GIT_URL/git@/https://}" + fi + if [[ "${GIT_URL}" == *.git ]]; then + GIT_URL="${GIT_URL%.git}" + fi + + SEARXNG_URL="$(python "${REPO_ROOT}/utils/get_setting.py" server.base_url)" + SEARXNG_PORT="$(python "${REPO_ROOT}/utils/get_setting.py" server.port)" + SEARXNG_BIND_ADDRESS="$(python "${REPO_ROOT}/utils/get_setting.py" server.bind_address)" + export GIT_URL + export GIT_BRANCH + export SEARXNG_URL + export SEARXNG_PORT + export SEARXNG_BIND_ADDRESS + +} + +pushd "${REPO_ROOT}" &> /dev/null +build.env.export +popd &> /dev/null diff --git a/utils/build_env.py b/utils/build_env.py deleted file mode 100644 index 1ef8ab700..000000000 --- a/utils/build_env.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -"""build environment used by shell scripts -""" - -# set path -import sys -import os -from os.path import realpath, dirname, join, sep, abspath - -repo_root = realpath(dirname(realpath(__file__)) + sep + '..') -sys.path.insert(0, repo_root) - -# Assure that the settings file from repository's working tree is used to -# generate the build_env, not from /etc/searxng/settings.yml. -os.environ['SEARXNG_SETTINGS_PATH'] = join(repo_root, 'etc', 'settings.yml') - -def _env(*arg, **kwargs): - val = get_setting(*arg, **kwargs) - if val is True: - val = '1' - elif val is False: - val = '' - return val - -# If you add or remove variables here, do not forget to update: -# - ./docs/admin/engines/settings.rst -# - ./docs/dev/makefile.rst (section make buildenv) - -name_val = [ - - ('SEARXNG_URL' , 'server.base_url'), - ('SEARXNG_PORT' , 'server.port'), - ('SEARXNG_BIND_ADDRESS' , 'server.bind_address'), - -] - -brand_env = 'utils' + sep + 'brand.env' - -# Some defaults in the settings.yml are taken from the environment, -# e.g. SEARXNG_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). When the -# 'brand.env' file is created these environment variables should be unset first:: - -_unset = object() -for name, option in name_val: - if not os.environ.get(name, _unset) is _unset: - del os.environ[name] - -# After the variables are unset in the environ, we can import from the searx -# package (what will read the values from the settings.yml). - -from searx.version import GIT_URL, GIT_BRANCH -from searx import get_setting - -print('build %s (settings from: %s)' % (brand_env, os.environ['SEARXNG_SETTINGS_PATH'])) -sys.path.insert(0, repo_root) - -with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f: - for name, option in name_val: - print("export %s='%s'" % (name, _env(option)), file=f) - print(f"export GIT_URL='{GIT_URL}'", file=f) - print(f"export GIT_BRANCH='{GIT_BRANCH}'", file=f) diff --git a/utils/filtron.sh b/utils/filtron.sh index 7607bf6aa..2ac3b6dfd 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -4,8 +4,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" # ---------------------------------------------------------------------------- # config diff --git a/utils/get_setting.py b/utils/get_setting.py index fa8d9cf8f..94c894e3e 100644 --- a/utils/get_setting.py +++ b/utils/get_setting.py @@ -12,17 +12,6 @@ from pathlib import Path repo_root = Path(__file__).resolve().parent.parent -# If you add or remove variables here, do not forget to update: -# - ./docs/admin/engines/settings.rst -# - ./docs/dev/makefile.rst (section make buildenv) - -name_val = [ - ("SEARXNG_URL", "server.base_url"), - ("SEARXNG_PORT", "server.port"), - ("SEARXNG_BIND_ADDRESS", "server.bind_address"), -] - - def main(setting_name): settings_path = repo_root / "searx" / "settings.yml" diff --git a/utils/lxc.sh b/utils/lxc.sh index 28cc30265..723451115 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -4,8 +4,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" # load environment of the LXC suite LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}" diff --git a/utils/morty.sh b/utils/morty.sh index 33c5c0e73..52f0fec31 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -3,8 +3,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" # ---------------------------------------------------------------------------- # config diff --git a/utils/searx.sh b/utils/searx.sh index 719cd2031..1d339cd68 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -4,8 +4,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" # ---------------------------------------------------------------------------- # config diff --git a/utils/searxng.sh b/utils/searxng.sh index 197188a4a..8cabbb722 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -9,8 +9,8 @@ SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}" source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" # shellcheck source=utils/lib_redis.sh source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" +# shellcheck source=utils/brand.sh +source "${REPO_ROOT}/utils/brand.sh" SERVICE_NAME="searxng" SERVICE_USER="searxng" @@ -159,7 +159,7 @@ searxng.instance.env() { echo " SEARXNG_INTERNAL_HTTP: ${SEARXNG_INTERNAL_HTTP}" fi cat <