From fe8b88c8a4385a3454604143bc7ed4162f713251 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 28 Dec 2021 16:53:26 +0100 Subject: [PATCH 1/4] [mod] script to build & install a redis instance A script to build & install a simple & isolated redis service, dedicated to SearXNG and connected via Unix socket. $ ./manage redis.help redis.: devpkg : install essential packages to compile redis build : build redis binaries at /800GBPCIex4/share/SearXNG/dist/redis/6.2.6/amd64 install : create user (searxng-redis) and install systemd service (searxng-redis) remove : delete user (searxng-redis) and remove service (searxng-redis) shell : start bash interpreter from user searxng-redis src : clone redis source code to and checkput 6.2.6 useradd : create user (searxng-redis) at /usr/local/searxng-redis userdel : delete user (searxng-redis) addgrp : add to group (searxng-redis) rmgrp : remove from group (searxng-redis) Signed-off-by: Markus Heiser --- manage | 10 +- utils/lib_redis.sh | 348 ++++++++++++++++++ .../lib/systemd/system/searxng-redis.service | 42 +++ 3 files changed, 397 insertions(+), 3 deletions(-) create mode 100755 utils/lib_redis.sh create mode 100644 utils/templates/lib/systemd/system/searxng-redis.service diff --git a/manage b/manage index bf202cb6..4323c942 100755 --- a/manage +++ b/manage @@ -17,6 +17,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh" # shellcheck source=utils/lib_go.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" +# shellcheck source=utils/lib_redis.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh" + # config PYOBJECTS="searx" @@ -74,9 +77,10 @@ docker.: gecko.driver: download & install geckodriver if not already installed (required for robot_tests) -EOF - nvm.help - cat < set foo bar +# OK +# redis /usr/local/searxng-redis/redis.sock> get foo +# "bar" +# [CTRL-D] + + +# shellcheck disable=SC2091 +# shellcheck source=utils/lib.sh +. /dev/null + +REDIS_GIT_URL="https://github.com/redis/redis.git" +REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" + +REDIS_USER="searxng-redis" +REDIS_HOME="/usr/local/${REDIS_USER}" +REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" +REDIS_ENV="${REDIS_HOME}/.redis_env" + +REDIS_SERVICE_NAME="searxng-redis" +REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service" + +# binaries to compile & install +REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli) +# link names of redis-server binary +REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof) + +REDIS_CONF="${REDIS_HOME}/redis.conf" +REDIS_CONF_TEMPLATE=$(cat < and checkput ${REDIS_GIT_TAG} + useradd : create user (${REDIS_USER}) at ${REDIS_HOME} + userdel : delete user (${REDIS_USER}) + addgrp : add to group (${REDIS_USER}) + rmgrp : remove from group (${REDIS_USER}) +EOF +} + +redis.devpkg() { + + # Uses OS package manager to install the essential packages to build and + # compile sources + + sudo_or_exit + + case ${DIST_ID} in + ubuntu|debian) + pkg_install git build-essential + ;; + arch) + pkg_install git base-devel + ;; + fedora) + pkg_install git @development-tools + ;; + centos) + pkg_install git + yum groupinstall "Development Tools" -y + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS." + return 42 + ;; + esac +} + +redis.build() { + + # usage: redis.build + + rst_title "get redis sources" section + redis.src "${CACHE}/redis" + + if ! required_commands gcc nm make gawk; then + sudo -H "$0" redis.devpkg + fi + + rst_title "compile redis sources" section + + pushd "${CACHE}/redis" &>/dev/null + + if ask_yn "Do you run 'make distclean' first'?" Ny; then + $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout + fi + + $(bash.cmd) -c "make" 2>&1 | prefix_stdout + if ask_yn "Do you run 'make test'?" Ny; then + $(bash.cmd) -c "make test" | prefix_stdout + fi + + popd &>/dev/null + + tee_stderr 0.1 <&1 | prefix_stdout +mkdir -p "$(redis._get_dist)" +cd "${CACHE}/redis/src" +cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" +EOF + info_msg "redis binaries available at $(redis._get_dist)" +} + + +redis.install() { + sudo_or_exit + ( + set -e + redis.useradd + redis._install_bin + redis._install_conf + redis._install_service + ) + dump_return $? +} + +redis.remove() { + sudo_or_exit + ( + set -e + redis._remove_service + redis.userdel + ) + dump_return $? +} + +redis.shell() { + interactive_shell "${REDIS_USER}" +} + +redis.src() { + + # usage: redis.src "${CACHE}/redis" + + local dest="${1:-${CACHE}/redis}" + + if [ -d "${dest}" ] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <&1 | prefix_stdout +cd "${dest}" +git fetch --all +git reset --hard tags/${REDIS_GIT_TAG} +EOF + else + tee_stderr 0.1 <&1 | prefix_stdout +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone "${REDIS_GIT_URL}" "${dest}" +EOF + tee_stderr 0.1 <&1 | prefix_stdout +cd "${dest}" +git checkout tags/${REDIS_GIT_TAG} -b "build-branch" +EOF + fi +} + +redis.useradd(){ + + # usage: redis.useradd + + rst_title "add user ${REDIS_USER}" section + echo + sudo_or_exit + + # create user account + tee_stderr 0.5 < "${REDIS_ENV}" +grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile +EOF +} + +redis.userdel() { + sudo_or_exit + drop_service_account "${REDIS_USER}" + groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true +} + +redis.addgrp() { + + # usage: redis.addgrp + + [[ -z $1 ]] && die_caller 42 "missing argument " + sudo -H gpasswd -a "$1" "${REDIS_USER}" +} + +redis.rmgrp() { + + # usage: redis.rmgrp + + [[ -z $1 ]] && die_caller 42 "missing argument " + sudo -H gpasswd -d "$1" "${REDIS_USER}" + +} + + +# private redis. functions +# ------------------------ + +redis._install_bin() { + local src + src="$(redis._get_dist)" + ( + set -e + for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do + install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \ + "${src}/${redis_exe}" "${REDIS_HOME_BIN}" + done + + pushd "${REDIS_HOME_BIN}" &> /dev/null + for redis_exe in "${REDIS_LINK_EXE[@]}"; do + info_msg "link redis-server --> ${redis_exe}" + sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}" + done + popd &> /dev/null + + ) +} + +redis._install_conf() { + sudo -H -u "${REDIS_USER}" bash < "${REDIS_CONF}" +EOF +} + +redis._install_service() { + systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" +} + +redis._remove_service() { + systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" +} + +redis._get_dist() { + if [ -z "${REDIS_DIST}" ]; then + echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)" + else + echo "${REDIS_DIST}" + fi +} + +redis._arch() { + local ARCH + case "$(command uname -m)" in + "x86_64") ARCH=amd64 ;; + "aarch64") ARCH=arm64 ;; + "armv6" | "armv7l") ARCH=armv6l ;; + "armv8") ARCH=arm64 ;; + .*386.*) ARCH=386 ;; + ppc64*) ARCH=ppc64le ;; + *) die 42 "ARCH is unknown: $(command uname -m)" ;; + esac + echo "${ARCH}" +} + +# TODO: move this to the right place .. + +bash.cmd(){ + + # print cmd to get a bash in a non-root mode, even if we are in a sudo + # context. + + local user="${USER}" + local bash_cmd="bash" + + if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then + user="${SUDO_USER}" + bash_cmd="sudo -H -u ${SUDO_USER} bash" + fi + + printf "%s" "${bash_cmd}" +} diff --git a/utils/templates/lib/systemd/system/searxng-redis.service b/utils/templates/lib/systemd/system/searxng-redis.service new file mode 100644 index 00000000..d1d163f0 --- /dev/null +++ b/utils/templates/lib/systemd/system/searxng-redis.service @@ -0,0 +1,42 @@ +[Unit] + +Description=SearXNG redis service +After=syslog.target +After=network.target +Documentation=https://redis.io/documentation + +[Service] + +Type=simple +User=${REDIS_USER} +Group=${REDIS_USER} +WorkingDirectory=${REDIS_HOME} +Restart=always +TimeoutStopSec=0 + +Environment=USER=${REDIS_USER} HOME=${REDIS_HOME} +ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF} +ExecPaths=${REDIS_HOME_BIN} + +LimitNOFILE=65535 +NoNewPrivileges=true +PrivateDevices=yes + +# ProtectSystem=full +ProtectHome=yes +ReadOnlyDirectories=/ +ReadWritePaths=-${REDIS_HOME}/run + +UMask=007 +PrivateTmp=yes + +MemoryDenyWriteExecute=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictRealtime=true +RestrictNamespaces=true + +[Install] + +WantedBy=multi-user.target From 4f1130d6639ca93cd827f8c2bdaed1c3864f1af3 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 28 Dec 2021 16:56:52 +0100 Subject: [PATCH 2/4] [mod] test.shell: add utils/lib_redis.sh to shellcheck procedure Signed-off-by: Markus Heiser --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e91d81e7..c15a2345 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,7 @@ test.shell: utils/lib_nvm.sh \ utils/lib_static.sh \ utils/lib_go.sh \ + utils/lib_redis.sh \ utils/filtron.sh \ utils/searx.sh \ utils/morty.sh \ From a6cfab93fa7c1b1f0a04c073df42b50f2834d845 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Jan 2022 17:43:20 +0100 Subject: [PATCH 3/4] [enh] add redis connector searx/shared/redisdb.py Add a redis connector, the default DB connector is a socket at:: unix:///usr/local/searxng-redis/run/redis.sock?db=0 To set up a redis instance simply use:: $ ./manage redis.build $ sudo -H ./manage redis.install A hint for developers: To get access rights to this instance, your developer account needs to be added to the *searxng-redis* group:: $ sudo -H ./manage redis.addgrp "${USER}" # don't forget to logout & login to get member of group Signed-off-by: Markus Heiser --- docs/admin/engines/settings.rst | 30 ++++++++++++++++++++++ docs/src/searx.shared.redisdb.rst | 8 ++++++ requirements.txt | 1 + searx/settings.yml | 4 +++ searx/settings_defaults.py | 3 +++ searx/shared/redisdb.py | 41 +++++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+) create mode 100644 docs/src/searx.shared.redisdb.rst create mode 100644 searx/shared/redisdb.py diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index b04de7cb..9f96a2b6 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -139,6 +139,36 @@ Global Settings ``default_http_headers``: Set additional HTTP headers, see `#755 `__ + +.. _settings redis: + +``redis:`` +---------- + +.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url + +``url`` + URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`:: + + redis://[[username]:[password]]@localhost:6379/0 + rediss://[[username]:[password]]@localhost:6379/0 + unix://[[username]:[password]]@/path/to/socket.sock?db=0 + +.. admonition:: Tip for developers + + To set up a redis instance simply use:: + + $ ./manage redis.build + $ sudo -H ./manage redis.install + + To get access rights to this instance, your developer account needs to be + added to the *searxng-redis* group:: + + $ sudo -H ./manage redis.addgrp "${USER}" + # don't forget to logout & login to get member of group + +.. _settings outgoing: + ``outgoing:`` ------------- diff --git a/docs/src/searx.shared.redisdb.rst b/docs/src/searx.shared.redisdb.rst new file mode 100644 index 00000000..265d8761 --- /dev/null +++ b/docs/src/searx.shared.redisdb.rst @@ -0,0 +1,8 @@ +.. _redis db: + +======== +Redis DB +======== + +.. automodule:: searx.shared.redisdb + :members: diff --git a/requirements.txt b/requirements.txt index e42c1fb7..e7645bd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ uvloop==0.16.0 httpx-socks[asyncio]==0.4.1 langdetect==1.0.9 setproctitle==1.2.2 +redis==4.1.0 diff --git a/searx/settings.yml b/searx/settings.yml index 3227a5a5..0105c793 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -61,6 +61,10 @@ server: X-Robots-Tag: noindex, nofollow Referrer-Policy: no-referrer +redis: + # https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + ui: # Custom static path - leave it blank if you didn't change static_path: "" diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 9c4711bf..669f2fa8 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -169,6 +169,9 @@ SCHEMA = { 'method': SettingsValue(('POST', 'GET'), 'POST'), 'default_http_headers': SettingsValue(dict, {}), }, + 'redis': { + 'url': SettingsValue(str, 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'), + }, 'ui': { 'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')), 'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')), diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py new file mode 100644 index 00000000..613b82a3 --- /dev/null +++ b/searx/shared/redisdb.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Implementation of the redis client (redis-py_). + +.. _redis-py: https://github.com/redis/redis-py + +This implementation uses the :ref:`settings redis` setup from ``settings.yml``. +A redis DB connect can be tested by:: + + >>> from searx.shared import redisdb + >>> redisdb.init() + True + >>> db = redisdb.client() + >>> db.set("foo", "bar") + True + >>> db.get("foo") + b'bar' + >>> + +""" + +import logging +import redis +from searx import get_setting + +logger = logging.getLogger('searx.shared.redis') + + +def client(): + return redis.Redis.from_url(get_setting('redis.url')) + + +def init(): + try: + c = client() + logger.info("connected redis DB --> %s", c.acl_whoami()) + return True + except redis.exceptions.ConnectionError as exc: + logger.error("can't connet redis DB ...") + logger.error(" %s", exc) + return False From dca83944b588be3ec9e49486daea6cf15ef58f78 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 7 Jan 2022 17:29:32 +0100 Subject: [PATCH 4/4] [fix] redis: don't create a new connection at each client() call Suggested-by: @dalf https://github.com/searxng/searxng/pull/686#pullrequestreview-844942973 Signed-off-by: Markus Heiser --- searx/shared/redisdb.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py index 613b82a3..da71d169 100644 --- a/searx/shared/redisdb.py +++ b/searx/shared/redisdb.py @@ -24,10 +24,16 @@ import redis from searx import get_setting logger = logging.getLogger('searx.shared.redis') +_client = None def client(): - return redis.Redis.from_url(get_setting('redis.url')) + global _client # pylint: disable=global-statement + if _client is None: + # not thread safe: in the worst case scenario, two or more clients are + # initialized only one is kept, the others are garbage collected. + _client = redis.Redis.from_url(get_setting('redis.url')) + return _client def init():