From 414a6105e7793e9499c64dd649eda43250971c50 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Jun 2021 19:01:07 +0200 Subject: [PATCH 01/10] [mod] load .config.sh from installation (utils/lib_install.sh) **new** utils/lib_install.sh: Used to initialize installation procedures - Modified source_dot_config function that - loads .config.sh from an existing installation (at SEARX_SRC). - initialize **SEARX_SRC_INIT_FILES** - functions like: - install_log_searx_instance() - install_searx_get_state() **modified** utils/searx.sh - obsolete environment SEARX_INSTANCE_NAME has been replaced by string 'SearXNG'. **modified** utils/filtron.sh, utils/morty.sh, utils/searx.sh - source utils/lib_install.sh - normalize logging of environment variables using new function install_log_searx_instance() **modified** utils/lib.sh - fix marginal typos **Installation scripts** The utils/lib_install.sh is sourced by the installations scripts: - utils/searx.sh - utils/morty.sh - utils/filtron.sh If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function loads this configuration (instead of './.config.sh'). **SEARX_SRC_INIT_FILES** Array of file names to sync into a installation at $SEARX_SRC. The file names are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). Most often theses are files like: - .config.sh - searx/settings.yml - utils/brand.env - ... Signed-off-by: Markus Heiser --- Makefile | 1 + utils/filtron.sh | 52 +++++------- utils/lib.sh | 6 +- utils/lib_install.sh | 184 +++++++++++++++++++++++++++++++++++++++++++ utils/morty.sh | 33 ++++---- utils/searx.sh | 67 +++++----------- 6 files changed, 244 insertions(+), 99 deletions(-) create mode 100755 utils/lib_install.sh diff --git a/Makefile b/Makefile index 2b9dfacc..6c22d5e2 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,7 @@ test.shell: utils/brand.env \ $(MTOOLS) \ utils/lib.sh \ + utils/lib_install.sh \ utils/lib_static.sh \ utils/filtron.sh \ utils/searx.sh \ diff --git a/utils/filtron.sh b/utils/filtron.sh index 5680811e..7469cd80 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck disable=SC2119,SC2001 +# shellcheck disable=SC2001 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -94,28 +90,17 @@ apache (${PUBLIC_URL}) nginx (${PUBLIC_URL}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_FILTRON_SITE} - filtron rules: ${FILTRON_RULES_TEMPLATE} - -If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL : ${PUBLIC_URL} - PUBLIC_HOST : ${PUBLIC_HOST} - SERVICE_USER : ${SERVICE_USER} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} +---- sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} EOF - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:4005/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:4005/" - fi - done - fi + + install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } @@ -349,16 +334,15 @@ inspect_service() { cat < [ []] # # First form uses $CACHE/ as destination folder, second form clones - # into . If repository is allready cloned, pull from and + # into . If repository is already cloned, pull from and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/searxng/searxng searx-src origin/master searxlogin diff --git a/utils/lib_install.sh b/utils/lib_install.sh new file mode 100755 index 00000000..19d438c4 --- /dev/null +++ b/utils/lib_install.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 +# shellcheck source=utils/lib.sh +. /dev/null + +# Initialize installation procedures: +# +# - Modified source_dot_config function that +# - loads .config.sh from an existing installation (at SEARX_SRC). +# - initialize **SEARX_SRC_INIT_FILES** +# - functions like: +# - install_log_searx_instance() +# - install_searx_get_state() +# +# usage: +# source lib_install.sh +# +# **Installation scripts** +# +# The utils/lib_install.sh is sourced by the installations scripts: +# +# - utils/searx.sh +# - utils/morty.sh +# - utils/filtron.sh +# +# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function +# loads this configuration (instead of './.config.sh'). + +# **SEARX_SRC_INIT_FILES** +# +# Array of file names to sync into a installation at $SEARX_SRC. The file names +# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). +# Most often theses are files like: +# - .config.sh +# - searx/settings.yml +# - utils/brand.env +# - ... + + +SEARX_SRC_INIT_FILES=() + +eval orig_"$(declare -f source_dot_config)" + +source_dot_config() { + + # Modified source_dot_config function that + # - loads .config.sh from an existing installation (at SEARX_SRC). + # - initialize SEARX_SRC_INIT_FILES + + if [ -z "$eval_SEARX_SRC" ]; then + export eval_SEARX_SRC='true' + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + if [ ! -r "${SEARX_SRC}" ]; then + build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" + orig_source_dot_config + return 0 + fi + build_msg INSTANCE "using instance at: ${SEARX_SRC}" + + # set and log DOT_CONFIG + if [ -r "${SEARX_SRC}/.config.sh" ]; then + build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh" + DOT_CONFIG="${SEARX_SRC}/.config.sh" + else + build_msg INSTANCE "using local config: ${DOT_CONFIG}" + fi + init_SEARX_SRC_INIT_FILES + fi +} + +init_SEARX_SRC_INIT_FILES(){ + # init environment SEARX_SRC_INIT_FILES + + # Monitor modified files in the working-tree from the local repository, only + # if the local file differs to the corresponding file in the instance. Most + # often theses are files like: + # + # - .config.sh + # - searx/settings.yml + # - utils/brand.env + # - ... + + # keep list empty if there is no installation + SEARX_SRC_INIT_FILES=() + if [ ! -r "$SEARX_SRC" ]; then + return 0 + fi + + local fname + local msg="" + + # Monitor local modified files from the repository, only if the local file + # differs to the corresponding file in the instance + + while IFS= read -r fname; do + if [ -z "$fname" ]; then + continue + fi + if [ -r "${SEARX_SRC}/${fname}" ]; then + # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then + SEARX_SRC_INIT_FILES+=("${fname}") + build_msg INSTANCE "local clone (workingtree), modified file: ./$fname" + msg="to update use: sudo -H ./utils/searx.sh install init-src" + fi + fi + done <<< "$(git diff --name-only)" + [ -n "$msg" ] && build_msg INSTANCE "$msg" +} + +install_log_searx_instance() { + + echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" + echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" + echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" + + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "---- container setup" + echo -e " ${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi +} + +install_searx_get_state(){ + + # usage: install_searx_get_state + # + # Prompts a string indicating the status of the installation procedure + # + # missing-searx-clone: + # There is no clone at ${SEARX_SRC} + # missing-searx-pyenv: + # There is no pyenv in ${SEARX_PYENV} + # installer-modified: + # There are files modified locally in the installer (clone), + # see ${SEARX_SRC_INIT_FILES} description. + # python-installed: + # Scripts can be executed in instance's environment + # - user: ${SERVICE_USER} + # - pyenv: ${SEARX_PYENV} + + if ! [ -r "${SEARX_SRC}" ]; then + echo "missing-searx-clone" + return + fi + if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then + echo "missing-searx-pyenv" + return + fi + if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then + echo "missing-settings" + return + fi + if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + echo "installer-modified" + return + fi + echo "python-installed" +} + +# Initialization of the installation procedure +# -------------------------------------------- + +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +source_dot_config + +# shellcheck source=utils/lxc-searx.env +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env diff --git a/utils/morty.sh b/utils/morty.sh index 0a44ef57..afda41d2 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" -source_dot_config -SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env +# shellcheck source=utils/lib_install.sh +source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -86,12 +81,15 @@ apache : ${PUBLIC_URL_MORTY} nginx (${PUBLIC_URL_MORTY}) :install: nginx site with a reverse proxy (ProxyPass) :remove: nginx site ${NGINX_MORTY_SITE} - -If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: - PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} - MORTY_LISTEN: ${MORTY_LISTEN} - SERVICE_USER: ${SERVICE_USER} +---- +sourced ${DOT_CONFIG} : + SERVICE_USER : ${SERVICE_USER} + SERVICE_HOME : ${SERVICE_HOME} + PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} + MORTY_LISTEN: : ${MORTY_LISTEN} EOF + + install_log_searx_instance if in_container; then # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) for ip in $(global_IPs) ; do @@ -335,11 +333,14 @@ inspect_service() { cat <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" -sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" EOF } @@ -577,14 +555,14 @@ remove_searx_uwsgi() { } activate_service() { - rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "Activate SearXNG (service)" section echo uWSGI_enable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" } deactivate_service() { - rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + rst_title "De-Activate SearXNG (service)" section echo uWSGI_disable_app "$SEARX_UWSGI_APP" uWSGI_restart "$SEARX_UWSGI_APP" @@ -609,7 +587,7 @@ EOF } enable_debug() { - warn_msg "Do not enable debug in production enviroments!!" + warn_msg "Do not enable debug in production environments!!" info_msg "try to enable debug mode ..." tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -680,14 +658,11 @@ inspect_service() { rst_title "service status & log" cat < Date: Tue, 29 Jun 2021 19:46:06 +0200 Subject: [PATCH 02/10] [fix] utils/build_env.py and add documentation modified docs/admin/engines/settings.rst - Fix documentation and add section 'brand'. - Add remarks about **buildenv** variables. - Add remarks about settings from environment variables $SEARX_DEBUG, $SEARX_PORT, $SEARX_BIND_ADDRESS and $SEARX_SECRET modified docs/admin/installation-searx.rst & docs/build-templates/searx.rst Fix template location /templates/etc/searx/settings.yml modified docs/dev/makefile.rst Add description of the 'make buildenv' target and describe - we have all SearXNG setups are centralized in the settings.yml file - why some tasks need a utils/brand.env (aka instance's buildenv) modified manage Settings file from repository's working tree are used by default and ask user if a /etc/searx/settings.yml file exists. modified searx/settings.yml Add comments about when it is needed to run 'make buildenv' modified searx/settings_defaults.py Default for server:port is taken from enviroment variable SEARX_PORT. modified utils/build_env.py - Some defaults in the settings.yml are taken from the environment, e.g. SEARX_BIND_ADDRESS (searx.settings_defaults.SHEMA). When the 'brand.env' file is created these enviroment variables should be unset first. - The CONTACT_URL enviroment is not needed in the utils/brand.env Signed-off-by: Markus Heiser --- docs/admin/engines/settings.rst | 58 +++++++++++++++++++++---------- docs/admin/installation-searx.rst | 2 +- docs/build-templates/searx.rst | 3 +- docs/dev/makefile.rst | 24 +++++++++++++ manage | 10 ++++++ searx/settings.yml | 12 ++++--- searx/settings_defaults.py | 2 +- utils/brand.env | 1 - utils/build_env.py | 41 ++++++++++++++-------- 9 files changed, 111 insertions(+), 42 deletions(-) diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 7335b9a2..7325ea3b 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -37,15 +37,13 @@ see how you can simplify your *user defined* ``settings.yml``. Global Settings =============== -``general:`` ------------- +.. _settings global brand: -.. code:: yaml +``brand:`` +------------ - general: - debug: false # Debug mode, only for development - instance_name: "searxng" # displayed name - contact_url: false # mailto:contact@example.com +If you change a value in this section, don't forget to rebuild instance's +environment (:ref:`utils/brand.env `) .. code:: yaml @@ -57,22 +55,42 @@ Global Settings public_instances: https://searx.space wiki_url: https://github.com/searxng/searxng/wiki -``debug`` : +``git_url`` & ``git_branch`` : :ref:`buildenv GIT_URL & GIT_BRANCH` + Changes this, to point to your searx fork (branch). + +``issue_url`` : :ref:`buildenv ISSUE_URL` + If you host your own issue tracker change this URL. + +``docs_url`` : :ref:`buildenv DOCS_URL` + If you host your own documentation change this URL. + +``public_instances`` : :ref:`buildenv PUBLIC_INSTANCES` + If you host your own https://searx.space change this URL. + +``wiki_url`` : :ref:`buildenv WIKI_URL` + Link to your wiki (or ``false``) + +.. _settings global general: + +``general:`` +------------ + +.. code:: yaml + + general: + debug: false # Debug mode, only for development + instance_name: "searxng" # displayed name + contact_url: false # mailto:contact@example.com + +``debug`` : ``$SEARX_DEBUG`` Allow a more detailed log if you run searx directly. Display *detailed* error messages in the browser too, so this must be deactivated in production. ``contact_url``: Contact ``mailto:`` address or WEB form. -``git_url`` and ``git_branch``: - Changes this, to point to your searx fork (branch). - -``docs_url`` - If you host your own documentation, change this URL. - -``wiki_url``: - Link to your wiki (or ``false``) +.. _settings global server: ``server:`` ----------- @@ -94,16 +112,18 @@ Global Settings X-Robots-Tag : noindex, nofollow Referrer-Policy : no-referrer -``port`` & ``bind_address``: +``port`` & ``bind_address``: ``$SEARX_PORT`` & ``$SEARX_BIND_ADDRESS`` Port number and *bind address* of the searx web application if you run it directly using ``python searx/webapp.py``. Doesn't apply to searx running on Apache or Nginx. -``secret_key`` : +``secret_key`` : ``$SEARX_SECRET`` Used for cryptography purpose. -``base_url`` : +``base_url`` : :ref:`buildenv SEARX_URL ` The base URL where searx is deployed. Used to create correct inbound links. + If you change the value, don't forget to rebuild instance's environment + (:ref:`utils/brand.env `) ``image_proxy`` : Allow your instance of searx of being able to proxy images. Uses memory space. diff --git a/docs/admin/installation-searx.rst b/docs/admin/installation-searx.rst index 5c9985e8..c5e1a05e 100644 --- a/docs/admin/installation-searx.rst +++ b/docs/admin/installation-searx.rst @@ -94,7 +94,7 @@ For a *minimal setup*, configure like shown below – replace ``searx@$(uname .. group-tab:: Use default settings - .. literalinclude:: ../../utils/templates/etc/searx/use_default_settings.yml + .. literalinclude:: ../../utils/templates/etc/searx/settings.yml :language: yaml .. group-tab:: searx/settings.yml diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searx.rst index 33ed9601..e4d5f06b 100644 --- a/docs/build-templates/searx.rst +++ b/docs/build-templates/searx.rst @@ -134,7 +134,7 @@ ${fedora_build} .. code-block:: sh $ sudo -H mkdir -p \"$(dirname ${SEARX_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searx/use_default_settings.yml\" \\ + $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searx/settings.yml\" \\ \"${SEARX_SETTINGS_PATH}\" .. group-tab:: searx/settings.yml @@ -152,7 +152,6 @@ ${fedora_build} .. code-block:: sh $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARX_SETTINGS_PATH\" - $ sudo -H sed -i -e \"s/{instance_name}/searx@\$(uname -n)/g\" \"$SEARX_SETTINGS_PATH\" .. END searx config diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index b7472dad..2c10181b 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -81,6 +81,30 @@ the check fails if you edit the requirements listed in If you think, something goes wrong with your ./local environment or you change the :origin:`setup.py` file, you have to call :ref:`make clean`. +.. _make buildenv: + +``make buildenv`` +================= + +Rebuild instance's environment with the modified settings from the +:ref:`settings global brand` and :ref:`settings global server` section of your +:ref:`settings.yml `. + +We have all SearXNG setups are centralized in the :ref:`settings.yml` file. +This setup is available as long we are in a *installed instance*. E.g. the +*installed instance* on the server or the *installed developer instance* at +``./local`` (the later one is created by a :ref:`make install ` or :ref:`make run `). + +Tasks running outside of an *installed instance*, especially those tasks and +scripts running at (pre-) installation time do not have access to the SearXNG +setup (from a *installed instance*). Those tasks need a *build environment*. + +The ``make buildenv`` target will update the *build environment* in: + +- :origin:`utils/brand.env` + + .. _make run: ``make run`` diff --git a/manage b/manage index f0334383..a67bdc74 100755 --- a/manage +++ b/manage @@ -105,6 +105,16 @@ fi export DOCS_BUILD buildenv() { + + # settings file from repository's working tree are used by default + SEARX_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml" + + if [ -r '/etc/searx/settings.yml' ]; then + if ask_yn "should settings read from: /etc/searx/settings.yml"; then + SEARX_SETTINGS_PATH='/etc/searx/settings.yml' + fi + fi + export SEARX_SETTINGS_PATH SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \ | prefix_stdout "${_Blue}BUILDENV${_creset} " return "${PIPESTATUS[0]}" diff --git a/searx/settings.yml b/searx/settings.yml index faadb36d..8b642d66 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -4,6 +4,8 @@ general: contact_url: false # mailto:contact@example.com brand: + # If you change a value in this section don't forget to rebuild instance's + # enviroment (make buildenv) git_url: https://github.com/searxng/searxng git_branch: master issue_url: https://github.com/searxng/searxng/issues @@ -31,12 +33,14 @@ search: server: port: 8888 - # address to listen on bind_address: "127.0.0.1" - secret_key: "ultrasecretkey" # change this! - # Set custom base_url. Possible values: - # false or "https://your.custom.host/location/" + + # Set custom base_url. Possible values: false or "https://example.org/location" + # If you change base_url don't forget to rebuild instance's enviroment (make + # buildenv) port & address to listen on base_url: false + + secret_key: "ultrasecretkey" # change this! # Proxying image results through searx image_proxy: false # 1.0 and 1.1 are supported diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index d034cb1c..ccf4df5c 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -147,7 +147,7 @@ SCHEMA = { 'formats': SettingsValue(list, OUTPUT_FORMATS), }, 'server': { - 'port': SettingsValue(int, 8888), + 'port': SettingsValue((int,str), 8888, 'SEARX_PORT'), 'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'), 'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'), 'base_url': SettingsValue((False, str), False), diff --git a/utils/brand.env b/utils/brand.env index e0c670a0..a8491d15 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -4,5 +4,4 @@ export GIT_BRANCH='master' export ISSUE_URL='https://github.com/searxng/searxng/issues' export DOCS_URL='https://searxng.github.io/searxng' export PUBLIC_INSTANCES='https://searx.space' -export CONTACT_URL='' export WIKI_URL='https://github.com/searxng/searxng/wiki' diff --git a/utils/build_env.py b/utils/build_env.py index c52111e8..52ca9461 100644 --- a/utils/build_env.py +++ b/utils/build_env.py @@ -9,15 +9,12 @@ from os.path import realpath, dirname, join, sep, abspath repo_root = realpath(dirname(realpath(__file__)) + sep + '..') sys.path.insert(0, repo_root) -os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + '/settings.yml') # Under the assumption that a brand is always a fork assure that the settings # file from reposetorie's working tree is used to generate the build_env, not # from /etc/searx/settings.yml. os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + sep + 'settings.yml') -from searx import get_setting - def _env(*arg, **kwargs): val = get_setting(*arg, **kwargs) if val is True: @@ -27,19 +24,35 @@ def _env(*arg, **kwargs): return val name_val = [ - ('SEARX_URL' , _env('server.base_url','')), - ('GIT_URL' , _env('brand.git_url', '')), - ('GIT_BRANCH' , _env('brand.git_branch', '')), - ('ISSUE_URL' , _env('brand.issue_url', '')), - ('DOCS_URL' , _env('brand.docs_url', '')), - ('PUBLIC_INSTANCES' , _env('brand.public_instances', '')), - ('CONTACT_URL' , _env('general.contact_url', '')), - ('WIKI_URL' , _env('brand.wiki_url', '')), + ('SEARX_URL' , 'server.base_url'), + ('GIT_URL' , 'brand.git_url'), + ('GIT_BRANCH' , 'brand.git_branch'), + ('ISSUE_URL' , 'brand.issue_url'), + ('DOCS_URL' , 'brand.docs_url'), + ('PUBLIC_INSTANCES' , 'brand.public_instances'), + ('WIKI_URL' , 'brand.wiki_url'), ] brand_env = 'utils' + sep + 'brand.env' -print('build %s' % brand_env) +# Some defaults in the settings.yml are taken from the environment, +# e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). When the +# 'brand.env' file is created these enviroment variables should be unset first:: + +_unset = object() +for name, option in name_val: + if not os.environ.get(name, _unset) is _unset: + del os.environ[name] + +# After the variables are unset in the environ, we can import settings +# (get_setting) from searx module. + +from searx import get_setting + +print('build %s (settings from: %s)' % (brand_env, os.environ['SEARX_SETTINGS_PATH'])) +sys.path.insert(0, repo_root) +from searx import settings + with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f: - for name, val in name_val: - print("export %s='%s'" % (name, val), file=f) + for name, option in name_val: + print("export %s='%s'" % (name, _env(option)), file=f) From f61c918dd4ea211f32874f1072ecfca0f39fcca4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Jun 2021 20:08:10 +0200 Subject: [PATCH 03/10] [mod] normalize .config.sh with settings.yml In commit 94851790 we have centralized all SearXNG setups in the settings.yml file: 94851790 [mod] move brand options from Makefile to settings.yml This step has not yet been completed for the installation procedures! Since all SearXNG setups are done in the settings.yml these environment variables needs to be removed from the ./conf.sh file. Scripts and other tasks running outside of an instance got the needed values from the ./utils/brand.env file. By example: ATM the environment variables of the ./config.sh file are in conflict with them from settings.yml: - PUBLIC_URL --> {server:base_url} - SEARX_INTERNAL_HTTP --> {server:bind_address}.{server:port} - GIT_BRANCH --> {brand:GIT_URL} These environment variable of a SearXNG instance and additional - SEARX_SETTINGS_TEMPLATE has been remove from the '.config.sh' file. With this patch, the main focus of ./conf.sh resists on environment variables needed for the installation of morty, filtron software. modified .config.sh: - removed no longer supported variables (see above) - add comment about: SearXNG setup in settings.yml modified utils/searx.sh: - SEARX_INTERNAL_HTTP no longer take from .config.sh - SEARX_SETTINGS_PATH /etc/searx/settings.yml - SEARX_SETTINGS_TEMPLATE obsolete modified utils/lib_install.sh: Initialize environment variables SEARX_PYENV, SEARX_SETTINGS_PATH and PUBLIC_URL. modified: utils/morty.sh Add missing hint about SEARX_SETTINGS_PATH and move PUBLIC_URL to utils/lib_install.sh modified: utils/morty.sh Move PUBLIC_URL to utils/lib_install.sh Renamed utils/templates/etc/searx/use_default_settings.yml -> settings.yml - removed option which can't be modified after installation - add some comments with examples Signed-off-by: Markus Heiser --- .config.sh | 57 ++++++++----------- docs/admin/api.rst | 2 +- utils/filtron.sh | 1 - utils/lib_install.sh | 15 +++++ utils/morty.sh | 5 +- utils/searx.sh | 4 +- ...{use_default_settings.yml => settings.yml} | 40 +++++++++---- 7 files changed, 74 insertions(+), 50 deletions(-) rename utils/templates/etc/searx/{use_default_settings.yml => settings.yml} (55%) diff --git a/.config.sh b/.config.sh index 3a17b8b2..4d86aca4 100644 --- a/.config.sh +++ b/.config.sh @@ -2,48 +2,41 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck shell=bash disable=SC2034 # -# This environment is used by ./utils scripts like filtron.sh or searx.sh. The -# default values are *most flexible* and *best maintained*, you normally not -# need to change the defaults (except PUBLIC_URL). +# This file should be edited only ones just before the installation of any +# service is done. After the installation of the searx service a copy of this +# file is placed into the $SEARX_SRC of the instance, e.g.:: # -# Before you change any value here you have to uninstall any previous -# installation. Further is it recommended to backup your changes simply by -# adding them to you local brand (git branch):: +# /usr/local/searx/searx-src/.config.sh # -# git add .config - -# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx" -# The default is taken from ./utils/brand.env. - -PUBLIC_URL="${SEARX_URL}" - -if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then - # hint: Linux containers do not have DNS entries, lets use IPs - PUBLIC_URL="http://$(primary_ip)/searx" -fi - -# searx.sh -# --------- +# .. hint:: +# +# Before you change a value here, You have to fully uninstall any previous +# installation of searx, morty and filtron services! -# SEARX_INTERNAL_HTTP="127.0.0.1:8888" -# SEARX_SETTINGS_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" +# utils/searx.sh +# -------------- -# Only change, if you maintain a searx brand in your searx fork (GIT_URL) which -# is not hold by branch 'master'. The branch has to be a local branch, in the -# repository from which you install (which is most often the case). If you want -# to install branch 'foo', don't forget to run 'git branch foo origin/foo' once. -# GIT_BRANCH="${GIT_BRANCH:-master}" +# The setup of the SearXNG instance is done in the settings.yml +# (SEARX_SETTINGS_PATH). Read the remarks in [1] carefully and don't forget to +# rebuild instance's environment (make buildenv) if needed. The settings.yml +# file of an already installed instance is shown by:: +# +# $ ./utils/searx.sh --help +# ---- SearXNG instance setup (already installed) +# SEARX_SETTINGS_PATH : /etc/searx/settings.yml +# SEARX_SRC : /usr/local/searx/searx-src +# +# [1] https://searxng.github.io/searxng/admin/engines/settings.html -# filtron.sh -# ---------- +# utils/filtron.sh +# ---------------- # FILTRON_API="127.0.0.1:4005" # FILTRON_LISTEN="127.0.0.1:4004" # FILTRON_TARGET="127.0.0.1:8888" -# FILTRON_RULES_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" -# morty.sh -# -------- +# utils/morty.sh +# -------------- # morty listen address # MORTY_LISTEN="127.0.0.1:3000" diff --git a/docs/admin/api.rst b/docs/admin/api.rst index 7804a866..8c782ea9 100644 --- a/docs/admin/api.rst +++ b/docs/admin/api.rst @@ -87,7 +87,7 @@ HTML of the site. URL of the searx instance and values are customizable. .. code:: html -
+ diff --git a/utils/filtron.sh b/utils/filtron.sh index 7469cd80..dfd89909 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -11,7 +11,6 @@ source "${REPO_ROOT}/utils/lib_install.sh" # config # ---------------------------------------------------------------------------- -PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ diff --git a/utils/lib_install.sh b/utils/lib_install.sh index 19d438c4..a8cb23ab 100755 --- a/utils/lib_install.sh +++ b/utils/lib_install.sh @@ -52,6 +52,8 @@ source_dot_config() { if [ -z "$eval_SEARX_SRC" ]; then export eval_SEARX_SRC='true' SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) + SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) if [ ! -r "${SEARX_SRC}" ]; then build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" orig_source_dot_config @@ -114,6 +116,7 @@ install_log_searx_instance() { echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" echo -e " SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}" + echo -e " SSEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" echo -e " SEARX_URL : ${_BBlue}${SEARX_URL:-none}${_creset}" @@ -177,6 +180,18 @@ install_searx_get_state(){ # shellcheck source=utils/brand.env source "${REPO_ROOT}/utils/brand.env" +# SEARX_URL aka PUBLIC_URL: the public URL of the instance (e.g. +# "https://example.org/searx"). The value is taken from environment $SEARX_URL +# in ./utils/brand.env. This variable is a empty string if server.base_url in +# the settings.yml is set to 'false'. + +SEARX_URL="${SEARX_URL:-http://$(uname -n)}" +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARX_URL="http://$(primary_ip)" +fi +PUBLIC_URL="${SEARX_URL}" + source_dot_config # shellcheck source=utils/lxc-searx.env diff --git a/utils/morty.sh b/utils/morty.sh index afda41d2..b473eeef 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -110,8 +110,9 @@ EOF info_searx() { # shellcheck disable=SC1117 cat < Date: Tue, 29 Jun 2021 21:36:32 +0200 Subject: [PATCH 04/10] [mod] utils/.searx.sh: add commands: install [dot-config|init-src] This patch implements two new commands: install dot-config : - copy ./config.sh to ${SEARX_SRC} - implemented in new function install_DOT_CONFIG install init-src: - sync files (SEARX_SRC_INIT_FILES) with ${SEARX_SRC} - implemented in new function init_SEARX_SRC() new functions: verify_continue_install() and prompt_installation_status() These functions are used in installation procedures to get the status of the installation procedure. The status is based on the existing function: usage: install_searx_get_state Prompts a string indicating the status of the installation procedure missing-searx-clone: There is no clone at ${SEARX_SRC} missing-searx-pyenv: There is no pyenv in ${SEARX_PYENV} installer-modified: There are files modified locally in the installer (clone), see ${SEARX_SRC_INIT_FILES} description. python-installed: Scripts can be executed in instance's environment - user: ${SERVICE_USER} - pyenv: ${SEARX_PYENV} Signed-off-by: Markus Heiser --- utils/searx.sh | 183 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 43 deletions(-) diff --git a/utils/searx.sh b/utils/searx.sh index e8c2720e..cfe6cbcd 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -141,7 +141,7 @@ usage() { cat < /dev/null } -install_settings() { - rst_title "${SEARX_SETTINGS_PATH}" section +prompt_installation_status(){ + local _state + _state="$(install_searx_get_state)" + case $_state in + missing-searx-clone) + info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" + return 0 + ;; + *) + warn_msg "SearXNG instance already installed at: $SEARX_SRC" + warn_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " + return 42 + ;; + esac +} + +verify_continue_install(){ + if ! prompt_installation_status; then + MSG="[${_BCyan}KEY${_creset}] to continue installation / [${_BCyan}CTRL-C${_creset}] to exit" \ + wait_key + fi +} + +init_SEARX_SRC(){ + rst_title "Update instance: ${SEARX_SRC}/" section + if ! clone_is_available; then err_msg "you have to install searx first" - exit 42 + return 1 fi - mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" - if [[ ! -f "${SEARX_SETTINGS_PATH}" ]]; then - info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}" - info_msg " --> ${SEARX_SETTINGS_PATH}" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - configure_searx - return + init_SEARX_SRC_INIT_FILES + + if [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then + info_msg "no files registered in SEARX_SRC_INIT_FILES" + return 2 + fi + + echo + echo "Manipulating files like settings.yml can break existing installation!" + echo "Update instance with file(s) from: ${REPO_ROOT}" + echo + for i in "${SEARX_SRC_INIT_FILES[@]}"; do + echo "- $i" + done + if ! ask_yn "Do you really want to update these files in the instance?" Yn; then + return 42 fi + for fname in "${SEARX_SRC_INIT_FILES[@]}"; do + while true; do + choose_one _reply "choose next step with file ${fname}" \ + "leave file unchanged" \ + "replace file" \ + "diff files" \ + "interactive shell" + + case $_reply in + "leave file unchanged") + break + ;; + "replace file") + info_msg "copy: ${REPO_ROOT}/${fname} --> ${SEARX_SRC}/${fname}" + cp "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" + break + ;; + "diff files") + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + ;; + "interactive shell") + backup_file "${SEARX_SRC}/${fname}" + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${SERVICE_USER}" -i + $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + esac + done + done +} - rst_para "Diff between origin's setting file (+) and current (-):" - echo "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" - $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_TEMPLATE}" +install_DOT_CONFIG(){ + rst_title "Update instance: ${SEARX_SRC}/.config.sh" section - local action - choose_one action "What should happen to the settings file? " \ - "keep configuration unchanged" \ - "use origin settings" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave settings file unchanged" - ;; - "use origin settings") - backup_file "${SEARX_SETTINGS_PATH}" - info_msg "install origin settings" - cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - ;; - "start interactive shell") - backup_file "${SEARX_SETTINGS_PATH}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new setting file (-) and current (+):' - echo - $DIFF_CMD "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" - wait_key - ;; - esac + if cmp --silent "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh"; then + info_msg "${SEARX_SRC}/.config.sh is up to date" + return 0 + fi + + diff "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" + if ! ask_yn "Do you want to copy file .config.sh into instance?" Yn; then + return 42 + fi + backup_file "${SEARX_SRC}/.config.sh" + cp "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" +} + +install_settings() { + rst_title "${SEARX_SETTINGS_PATH}" section + + if ! clone_is_available; then + err_msg "you have to install searx first" + exit 42 + fi + + mkdir -p "$(dirname "${SEARX_SETTINGS_PATH}")" + install_template \ + "${SEARX_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + configure_searx } remove_settings() { From 28c874bf3f95be7b34da4cc7957771d8bd964178 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 Jul 2021 15:31:42 +0200 Subject: [PATCH 05/10] [brand] searx.sh & morty.sh - renamed 'searx' into 'SearXNG' Signed-off-by: Markus Heiser --- utils/morty.sh | 2 +- utils/searx.sh | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/utils/morty.sh b/utils/morty.sh index b473eeef..ccf92027 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -236,7 +236,7 @@ install_all() { fi fi info_searx - if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then + if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" "${REPO_ROOT}/utils/searx.sh" option image-proxy-on fi diff --git a/utils/searx.sh b/utils/searx.sh index cfe6cbcd..bc57f7b4 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -164,7 +164,7 @@ install / remove :packages: install needed packages from OS package manager :buildhost: install packages from OS package manager needed by buildhosts update searx - Update searx installation ($SERVICE_HOME) + Update SearXNG installation ($SERVICE_HOME) activate service activate and start service daemon (systemd unit) deactivate service @@ -174,7 +174,7 @@ inspect service option set one of the available options apache - :install: apache site with the searx uwsgi app + :install: apache site with the SearXNG uwsgi app :remove: apache site ${APACHE_FILTRON_SITE} ---- sourced ${DOT_CONFIG} SERVICE_USER : ${SERVICE_USER} @@ -239,7 +239,7 @@ main() { verify_continue_install install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi ;; packages) @@ -325,7 +325,7 @@ install_all() { wait_key install_searx_uwsgi if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" fi if ask_yn "Do you want to inspect the installation?" Ny; then inspect_service @@ -333,7 +333,7 @@ install_all() { } update_searx() { - rst_title "Update searx instance" + rst_title "Update SearXNG instance" echo tee_stderr 0.3 <&1 | prefix_stdout "$_service_prefix" @@ -391,14 +391,14 @@ clone_is_available() { # shellcheck disable=SC2164 clone_searx() { - rst_title "Clone searx sources" section + rst_title "Clone SearXNG sources" section echo if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" fi SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" if [[ ! "${SERVICE_HOME}" ]]; then - err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + err_msg "to clone SearXNG sources, user $SERVICE_USER hast to be created first" return 42 fi if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then @@ -451,7 +451,7 @@ init_SEARX_SRC(){ rst_title "Update instance: ${SEARX_SRC}/" section if ! clone_is_available; then - err_msg "you have to install searx first" + err_msg "you have to install SearXNG first" return 1 fi @@ -530,7 +530,7 @@ install_settings() { rst_title "${SEARX_SETTINGS_PATH}" section if ! clone_is_available; then - err_msg "you have to install searx first" + err_msg "you have to install SearXNG first" exit 42 fi @@ -542,7 +542,7 @@ install_settings() { } remove_settings() { - rst_title "remove searx settings" section + rst_title "remove SearXNG settings" section echo info_msg "delete ${SEARX_SETTINGS_PATH}" rm -f "${SEARX_SETTINGS_PATH}" @@ -550,10 +550,10 @@ remove_settings() { remove_searx() { rst_title "Drop searx sources" section - if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + if ask_yn "Do you really want to drop SearXNG sources ($SEARX_SRC)?"; then rm -rf "$SEARX_SRC" else - rst_para "Leave searx sources unchanged." + rst_para "Leave SearXNG sources unchanged." fi } @@ -565,7 +565,7 @@ create_pyenv() { rst_title "Create virtualenv (python)" section echo if [[ ! -f "${SEARX_SRC}/manage" ]]; then - err_msg "to create pyenv for searx, searx has to be cloned first" + err_msg "to create pyenv for SearXNG, SearXNG has to be cloned first" return 42 fi info_msg "create pyenv in ${SEARX_PYENV}" @@ -605,8 +605,8 @@ EOF } configure_searx() { - rst_title "Configure searx" section - rst_para "Setup searx config located at $SEARX_SETTINGS_PATH" + rst_title "Configure SearXNG" section + rst_para "Setup SearXNG config located at $SEARX_SETTINGS_PATH" echo tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -615,7 +615,7 @@ EOF } test_local_searx() { - rst_title "Testing searx instance localy" section + rst_title "Testing SearXNG instance localy" section echo if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then @@ -637,14 +637,14 @@ EOF } install_searx_uwsgi() { - rst_title "Install searx's uWSGI app (searx.ini)" section + rst_title "Install SearXNG's uWSGI app (searx.ini)" section echo install_uwsgi uWSGI_install_app "$SEARX_UWSGI_APP" } remove_searx_uwsgi() { - rst_title "Remove searx's uWSGI app (searx.ini)" section + rst_title "Remove SearXNG's uWSGI app (searx.ini)" section echo uWSGI_remove_app "$SEARX_UWSGI_APP" } @@ -772,9 +772,9 @@ EOF fi if clone_is_available; then - info_msg "~$SERVICE_USER: Searx software is installed." + info_msg "~$SERVICE_USER: SearXNG software is installed." else - err_msg "~$SERVICE_USER: Missing searx software!" + err_msg "~$SERVICE_USER: Missing SearXNG software!" fi if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then @@ -807,7 +807,7 @@ EOF fi local _debug_on - if ask_yn "Enable searx debug mode?"; then + if ask_yn "Enable SearXNG debug mode?"; then enable_debug _debug_on=1 fi @@ -849,7 +849,7 @@ install_apache_site() { rst_title "Install Apache site $APACHE_SEARX_SITE" rst_para "\ -This installs the searx uwsgi app as apache site. If your server is public to +This installs the SearXNG uwsgi app as apache site. If your server is public to the internet, you should instead use a reverse proxy (filtron) to block excessively bot queries." @@ -863,7 +863,7 @@ excessively bot queries." apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" - rst_title "Install searx's uWSGI app (searx.ini)" section + rst_title "Install SearXNG's uWSGI app (searx.ini)" section echo uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP" @@ -887,7 +887,7 @@ This removes apache site ${APACHE_SEARX_SITE}." apache_remove_site "${APACHE_SEARX_SITE}" - rst_title "Remove searx's uWSGI app (searx.ini)" section + rst_title "Remove SearXNG's uWSGI app (searx.ini)" section echo uWSGI_remove_app "$SEARX_UWSGI_APP" } From 57b8f340a6689fafd0c8d92aebd2b263b090af18 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 Jul 2021 16:51:36 +0200 Subject: [PATCH 06/10] [mod] utils/searx.sh - add command 'inspect settings' Inspect YAML setting from SearXNG instance (${SEARX_SRC}):: utils/searx.sh inspect settings server.base_url utils/lib_install.sh should not log on stdout which is used for the output of function prompt_installation_setting(). Turned build_msg (stdout) into info_msg (stderr). Signed-off-by: Markus Heiser --- utils/lib_install.sh | 12 ++++----- utils/searx.sh | 61 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/utils/lib_install.sh b/utils/lib_install.sh index a8cb23ab..4ad11d63 100755 --- a/utils/lib_install.sh +++ b/utils/lib_install.sh @@ -55,18 +55,18 @@ source_dot_config() { SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) SEARX_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SETTINGS_PATH) if [ ! -r "${SEARX_SRC}" ]; then - build_msg INSTANCE "not yet cloned: ${SEARX_SRC}" + info_msg "not yet cloned: ${SEARX_SRC}" orig_source_dot_config return 0 fi - build_msg INSTANCE "using instance at: ${SEARX_SRC}" + info_msg "using instance at: ${SEARX_SRC}" # set and log DOT_CONFIG if [ -r "${SEARX_SRC}/.config.sh" ]; then - build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh" + info_msg "switching to ${SEARX_SRC}/.config.sh" DOT_CONFIG="${SEARX_SRC}/.config.sh" else - build_msg INSTANCE "using local config: ${DOT_CONFIG}" + info_msg "using local config: ${DOT_CONFIG}" fi init_SEARX_SRC_INIT_FILES fi @@ -104,12 +104,12 @@ init_SEARX_SRC_INIT_FILES(){ # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then SEARX_SRC_INIT_FILES+=("${fname}") - build_msg INSTANCE "local clone (workingtree), modified file: ./$fname" + info_msg "local clone (workingtree), modified file: ./$fname" msg="to update use: sudo -H ./utils/searx.sh install init-src" fi fi done <<< "$(git diff --name-only)" - [ -n "$msg" ] && build_msg INSTANCE "$msg" + [ -n "$msg" ] && info_msg "$msg" } install_log_searx_instance() { diff --git a/utils/searx.sh b/utils/searx.sh index bc57f7b4..f0261105 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -146,7 +146,7 @@ usage:: $(basename "$0") remove [all|user|pyenv|searx-src] $(basename "$0") activate [service] $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] + $(basename "$0") inspect [service|settings ] $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy ] $(basename "$0") apache [install|remove] @@ -169,8 +169,9 @@ activate service activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -inspect service - run some small tests and inspect service's status and log +inspect + :service: run some small tests and inspect service's status and log + :settings: inspect YAML setting from SearXNG instance (${SEARX_SRC}) option set one of the available options apache @@ -205,6 +206,10 @@ main() { sudo_or_exit inspect_service ;; + settings) + prompt_installation_setting "$3" + dump_return $? + ;; *) usage "$_usage"; exit 42;; esac ;; install) @@ -425,9 +430,14 @@ EOF } prompt_installation_status(){ - local _state - _state="$(install_searx_get_state)" - case $_state in + + local state branch remote remote_url instance_setting + state="$(install_searx_get_state)" + branch="$(git name-rev --name-only HEAD)" + remote="$(git config branch."${branch}".remote)" + remote_url="$(git config remote."${remote}".url)" + + case $state in missing-searx-clone) info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" return 0 @@ -435,6 +445,16 @@ prompt_installation_status(){ *) warn_msg "SearXNG instance already installed at: $SEARX_SRC" warn_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " + instance_setting="$(prompt_installation_setting brand.git_url)" + if ! [ "$instance_setting" = "$remote_url" ]; then + warn_msg "instance's brand.git_url: '${instance_setting}'" \ + "differs from local clone's remote URL: ${remote_url}" + fi + instance_setting="$(prompt_installation_setting brand.git_branch)" + if ! [ "$instance_setting" = "$branch" ]; then + warn_msg "instance brand.git_branch: ${instance_setting}" \ + "differs from local clone's branch: ${branch}" + fi return 42 ;; esac @@ -447,6 +467,35 @@ verify_continue_install(){ fi } +prompt_installation_setting(){ + + # usage: prompt_installation_setting brand.git_url + # + # Prompts the value of the (YAML) setting in the SearXNG instance. + + local _state + _state="$(install_searx_get_state)" + case $_state in + python-installed|installer-modified) + sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" < Date: Tue, 20 Jul 2021 13:16:21 +0200 Subject: [PATCH 07/10] [mod] reduce enviroment variables in shell scripts to what is needed Not all settings from the 'brand:' section of the YAML files are needed in the shell scripts. This patch reduce the variables in ./utils/brand.env to what is needed. The following ('brand:' settings) can be removed from this file: - ISSUE_URL - DOCS_URL - PUBLIC_INSTANCES - WIKI_URL Tasks running outside of an *installed instance*, need the following settings from the YAML configuration: - GIT_URL <--> brand.git_url - GIT_BRANCH <--> brand.git_branch - SEARX_URL <--> server.base_url (aka PUBLIC_URL) - SEARX_PORT <--> server.port - SEARX_BIND_ADDRESS <--> server.bind_address Signed-off-by: Markus Heiser --- docs/admin/engines/settings.rst | 37 +++++++++++++++----------- docs/dev/makefile.rst | 10 +++++++ searx/settings.yml | 20 +++++++------- utils/brand.env | 8 +++--- utils/build_env.py | 15 +++++++---- utils/lxc.sh | 2 +- utils/morty.sh | 3 +-- utils/templates/etc/searx/settings.yml | 2 +- 8 files changed, 59 insertions(+), 38 deletions(-) diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 7325ea3b..cab4ee49 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -42,9 +42,6 @@ Global Settings ``brand:`` ------------ -If you change a value in this section, don't forget to rebuild instance's -environment (:ref:`utils/brand.env `) - .. code:: yaml brand: @@ -55,19 +52,24 @@ environment (:ref:`utils/brand.env `) public_instances: https://searx.space wiki_url: https://github.com/searxng/searxng/wiki -``git_url`` & ``git_branch`` : :ref:`buildenv GIT_URL & GIT_BRANCH` +.. sidebar:: buildenv + + Changing a value tagged by :ref:`buildenv `, needs to + rebuild instance's environment :ref:`utils/brand.env `. + +``git_url`` & ``git_branch`` : :ref:`buildenv GIT_URL & GIT_BRANCH ` Changes this, to point to your searx fork (branch). -``issue_url`` : :ref:`buildenv ISSUE_URL` +``issue_url`` : If you host your own issue tracker change this URL. -``docs_url`` : :ref:`buildenv DOCS_URL` +``docs_url`` : If you host your own documentation change this URL. -``public_instances`` : :ref:`buildenv PUBLIC_INSTANCES` +``public_instances`` : If you host your own https://searx.space change this URL. -``wiki_url`` : :ref:`buildenv WIKI_URL` +``wiki_url`` : Link to your wiki (or ``false``) .. _settings global general: @@ -98,10 +100,10 @@ environment (:ref:`utils/brand.env `) .. code:: yaml server: + base_url: false # set custom base_url (or false) port: 8888 bind_address: "127.0.0.1" # address to listen on secret_key: "ultrasecretkey" # change this! - base_url: false # set custom base_url (or false) image_proxy: false # proxying image results through searx default_locale: "" # default interface locale default_theme: oscar # ui theme @@ -112,19 +114,24 @@ environment (:ref:`utils/brand.env `) X-Robots-Tag : noindex, nofollow Referrer-Policy : no-referrer -``port`` & ``bind_address``: ``$SEARX_PORT`` & ``$SEARX_BIND_ADDRESS`` - Port number and *bind address* of the searx web application if you run it - directly using ``python searx/webapp.py``. Doesn't apply to searx running on - Apache or Nginx. +.. sidebar:: buildenv -``secret_key`` : ``$SEARX_SECRET`` - Used for cryptography purpose. + Changing a value tagged by :ref:`buildenv `, needs to + rebuild instance's environment :ref:`utils/brand.env `. ``base_url`` : :ref:`buildenv SEARX_URL ` The base URL where searx is deployed. Used to create correct inbound links. If you change the value, don't forget to rebuild instance's environment (:ref:`utils/brand.env `) +``port`` & ``bind_address``: :ref:`buildenv SEARX_PORT & SEARX_BIND_ADDRESS ` + Port number and *bind address* of the searx web application if you run it + directly using ``python searx/webapp.py``. Doesn't apply to searx running on + Apache or Nginx. + +``secret_key`` : ``$SEARX_SECRET`` + Used for cryptography purpose. + ``image_proxy`` : Allow your instance of searx of being able to proxy images. Uses memory space. diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index 2c10181b..5bf3c13b 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -104,6 +104,16 @@ The ``make buildenv`` target will update the *build environment* in: - :origin:`utils/brand.env` +Tasks running outside of an *installed instance*, need the following settings +from the YAML configuration: + +- ``GIT_URL`` from :ref:`brand.git_url ` +- ``GIT_BRANCH`` from :ref:`brand.git_branch ` + +- ``SEARX_URL`` from :ref:`server.base_url ` (aka + ``PUBLIC_URL``) +- ``SEARX_BIND_ADDRESS`` from :ref:`server.bind_address ` +- ``SEARX_PORT`` from :ref:`server.port ` .. _make run: diff --git a/searx/settings.yml b/searx/settings.yml index 8b642d66..a951b862 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -4,15 +4,16 @@ general: contact_url: false # mailto:contact@example.com brand: - # If you change a value in this section don't forget to rebuild instance's - # enviroment (make buildenv) - git_url: https://github.com/searxng/searxng - git_branch: master - issue_url: https://github.com/searxng/searxng/issues new_issue_url: https://github.com/searxng/searxng/issues/new docs_url: https://searxng.github.io/searxng public_instances: https://searx.space wiki_url: https://github.com/searxng/searxng/wiki + issue_url: https://github.com/searxng/searxng/issues + + # If you change a value below don't forget to rebuild instance's enviroment + # (make buildenv) + git_url: https://github.com/searxng/searxng + git_branch: master search: # Filter results. 0: None, 1: Moderate, 2: Strict @@ -32,13 +33,14 @@ search: formats: [html, csv, json, rss] server: + # If you change port, bind_address or base_url don't forget to rebuild + # instance's enviroment (make buildenv) port: 8888 bind_address: "127.0.0.1" + base_url: false # Possible values: false or "https://example.org/location". - # Set custom base_url. Possible values: false or "https://example.org/location" - # If you change base_url don't forget to rebuild instance's enviroment (make - # buildenv) port & address to listen on - base_url: false + # If your instance owns a /etc/searx/settings.yml file, then set the following + # values there. secret_key: "ultrasecretkey" # change this! # Proxying image results through searx diff --git a/utils/brand.env b/utils/brand.env index a8491d15..2075a2bf 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,7 +1,5 @@ -export SEARX_URL='' export GIT_URL='https://github.com/searxng/searxng' export GIT_BRANCH='master' -export ISSUE_URL='https://github.com/searxng/searxng/issues' -export DOCS_URL='https://searxng.github.io/searxng' -export PUBLIC_INSTANCES='https://searx.space' -export WIKI_URL='https://github.com/searxng/searxng/wiki' +export SEARX_URL='' +export SEARX_PORT='8888' +export SEARX_BIND_ADDRESS='127.0.0.1' diff --git a/utils/build_env.py b/utils/build_env.py index 52ca9461..28674f92 100644 --- a/utils/build_env.py +++ b/utils/build_env.py @@ -23,14 +23,19 @@ def _env(*arg, **kwargs): val = '' return val +# If you add or remove variables here, do not forgett to update: +# - docs/admin/engines/settings.rst +# - /docs/dev/makefile.rst (section make buildenv) + name_val = [ - ('SEARX_URL' , 'server.base_url'), + ('GIT_URL' , 'brand.git_url'), ('GIT_BRANCH' , 'brand.git_branch'), - ('ISSUE_URL' , 'brand.issue_url'), - ('DOCS_URL' , 'brand.docs_url'), - ('PUBLIC_INSTANCES' , 'brand.public_instances'), - ('WIKI_URL' , 'brand.wiki_url'), + + ('SEARX_URL' , 'server.base_url'), + ('SEARX_PORT' , 'server.port'), + ('SEARX_BIND_ADDRESS' , 'server.bind_address'), + ] brand_env = 'utils' + sep + 'brand.env' diff --git a/utils/lxc.sh b/utils/lxc.sh index f94e9841..9a79606c 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -568,7 +568,7 @@ check_connectivity() { info_msg "Most often the connectivity is blocked by a docker installation:" info_msg "Whenever docker is started (reboot) it sets the iptables policy " info_msg "for the FORWARD chain to DROP, see:" - info_msg " ${DOCS_URL}/utils/lxc.sh.html#internet-connectivity-docker" + info_msg " https://searxng.github.io/searxng/utils/lxc.sh.html#internet-connectivity-docker" iptables-save | grep ":FORWARD" fi return $ret_val diff --git a/utils/morty.sh b/utils/morty.sh index ccf92027..8be0e6ba 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -12,7 +12,6 @@ source "${REPO_ROOT}/utils/lib_install.sh" MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" - PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" # shellcheck disable=SC2034 @@ -111,7 +110,7 @@ info_searx() { # shellcheck disable=SC1117 cat < Date: Wed, 21 Jul 2021 09:41:48 +0200 Subject: [PATCH 08/10] [fix] test.robot - do not use environment from ./utils/brand.env Some defaults in the settings.yml are taken from the environment. By example; The manage scripts sources the ./utils/brand.env and sets SEARX_PORT environment. This enviroment *wins over* any settings in a YAML file. Whe we run a:: make test.robot The searx/settings_robot.yml is used, in this file the server settings are:: server: port: 11111 bind_address: 127.0.0.1 To get in use of the 'port: 11111' we have to unset the SEARX_PORT environment which was sourced before. The function buildenv.unset_env() can be called in all use cases where the enviroment from ./utils/brand.env is not wanted. ATM it unset the enviroment variables:: unset GIT_URL unset GIT_BRANCH unset SEARX_URL unset SEARX_PORT unset SEARX_BIND_ADDRESS Signed-off-by: Markus Heiser --- manage | 12 ++++++++++++ utils/build_env.py | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/manage b/manage index a67bdc74..b974c302 100755 --- a/manage +++ b/manage @@ -120,6 +120,17 @@ buildenv() { return "${PIPESTATUS[0]}" } +buildenv.unset_env(){ + # Some defaults in the settings.yml are taken from the environment, + # e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). In + # some tasks (e.g. test.robot) we do not want these envorionment applied. + unset GIT_URL + unset GIT_BRANCH + unset SEARX_URL + unset SEARX_PORT + unset SEARX_BIND_ADDRESS +} + babel.compile() { build_msg BABEL compile pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations" @@ -481,6 +492,7 @@ test.coverage() { test.robot() { build_msg TEST 'robot' + buildenv.unset_env gecko.driver PYTHONPATH=. pyenv.cmd python searx/testing.py robot dump_return $? diff --git a/utils/build_env.py b/utils/build_env.py index 28674f92..dc55bd0a 100644 --- a/utils/build_env.py +++ b/utils/build_env.py @@ -24,8 +24,9 @@ def _env(*arg, **kwargs): return val # If you add or remove variables here, do not forgett to update: -# - docs/admin/engines/settings.rst -# - /docs/dev/makefile.rst (section make buildenv) +# - ./docs/admin/engines/settings.rst +# - ./docs/dev/makefile.rst (section make buildenv) +# - ./manage function buildenv.unset_env() name_val = [ From 7196a9b5dd2525003bb7d69e2b58ee982df0c30b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 20 Jul 2021 15:58:58 +0200 Subject: [PATCH 09/10] [fix] ./utils/filtron.sh - FILTRON_TARGET from YAML settings The filtron target is the SearXNG installation and the default of FILTRON_TARGET is taken from the YAML configuration ('server.port' & 'server.bind_address'). Signed-off-by: Markus Heiser --- .config.sh | 1 - utils/filtron.sh | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.config.sh b/.config.sh index 4d86aca4..0e41a73b 100644 --- a/.config.sh +++ b/.config.sh @@ -33,7 +33,6 @@ # FILTRON_API="127.0.0.1:4005" # FILTRON_LISTEN="127.0.0.1:4004" -# FILTRON_TARGET="127.0.0.1:8888" # utils/morty.sh # -------------- diff --git a/utils/filtron.sh b/utils/filtron.sh index dfd89909..3bd6f016 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -23,7 +23,12 @@ FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/e FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" -FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +# The filtron target is the SearXNG installation, listenning on server.port at +# server.bind_address. The default of FILTRON_TARGET is taken from the YAML +# configuration, do not change this value without reinstalling the entire +# SearXNG suite including filtron & morty. +FILTRON_TARGET="${SEARX_BIND_ADDRESS}:${SEARX_PORT}" SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" From e946c3462c2b44fb154d0b6595f4f42605a326e5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 20 Jul 2021 13:50:01 +0200 Subject: [PATCH 10/10] [brand] !!DO NOT MERGE!! this patch uses branch fix-searx.sh @return42 Do not merge this patch in master branch of SearXNG! This branch exists only for testing the feature branch fix-searx.sh @return42. This patch changes the buildenv to:: GIT_URL='https://github.com/return42/searxng' GIT_BRANCH='fix-searx.sh' SEARX_PORT='7777' SEARX_BIND_ADDRESS='127.0.0.12' To test installation procedure, clone feature branch (fix-searx.sh):: $ cd ~/Downloads $ git clone --branch fix-searx.sh https://github.com/return42/searxng searxng $ cd searxng $ ./utils/searx.sh install all ... Signed-off-by: Markus Heiser --- searx/settings.yml | 8 ++++---- utils/brand.env | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/searx/settings.yml b/searx/settings.yml index a951b862..5307631d 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -12,8 +12,8 @@ brand: # If you change a value below don't forget to rebuild instance's enviroment # (make buildenv) - git_url: https://github.com/searxng/searxng - git_branch: master + git_url: https://github.com/return42/searxng + git_branch: fix-searx.sh search: # Filter results. 0: None, 1: Moderate, 2: Strict @@ -35,8 +35,8 @@ search: server: # If you change port, bind_address or base_url don't forget to rebuild # instance's enviroment (make buildenv) - port: 8888 - bind_address: "127.0.0.1" + port: 7777 + bind_address: "127.0.0.12" base_url: false # Possible values: false or "https://example.org/location". # If your instance owns a /etc/searx/settings.yml file, then set the following diff --git a/utils/brand.env b/utils/brand.env index 2075a2bf..c09ea9d2 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,5 +1,5 @@ -export GIT_URL='https://github.com/searxng/searxng' -export GIT_BRANCH='master' +export GIT_URL='https://github.com/return42/searxng' +export GIT_BRANCH='fix-searx.sh' export SEARX_URL='' -export SEARX_PORT='8888' -export SEARX_BIND_ADDRESS='127.0.0.1' +export SEARX_PORT='7777' +export SEARX_BIND_ADDRESS='127.0.0.12'