diff --git a/Makefile b/Makefile index 747083d0..20f89a79 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,7 @@ test.shell: utils/lib_nvm.sh \ utils/lib_static.sh \ utils/lib_go.sh \ + utils/lib_redis.sh \ utils/filtron.sh \ utils/searx.sh \ utils/morty.sh \ diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index a829b870..c96dd332 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -142,6 +142,36 @@ Global Settings ``default_http_headers``: Set additional HTTP headers, see `#755 `__ + +.. _settings redis: + +``redis:`` +---------- + +.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url + +``url`` + URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`:: + + redis://[[username]:[password]]@localhost:6379/0 + rediss://[[username]:[password]]@localhost:6379/0 + unix://[[username]:[password]]@/path/to/socket.sock?db=0 + +.. admonition:: Tip for developers + + To set up a redis instance simply use:: + + $ ./manage redis.build + $ sudo -H ./manage redis.install + + To get access rights to this instance, your developer account needs to be + added to the *searxng-redis* group:: + + $ sudo -H ./manage redis.addgrp "${USER}" + # don't forget to logout & login to get member of group + +.. _settings outgoing: + ``outgoing:`` ------------- diff --git a/docs/src/searx.shared.redisdb.rst b/docs/src/searx.shared.redisdb.rst new file mode 100644 index 00000000..265d8761 --- /dev/null +++ b/docs/src/searx.shared.redisdb.rst @@ -0,0 +1,8 @@ +.. _redis db: + +======== +Redis DB +======== + +.. automodule:: searx.shared.redisdb + :members: diff --git a/manage b/manage index 3f367268..1643802f 100755 --- a/manage +++ b/manage @@ -17,6 +17,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh" # shellcheck source=utils/lib_go.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" +# shellcheck source=utils/lib_redis.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh" + # config PYOBJECTS="searx" @@ -74,9 +77,10 @@ docker.: gecko.driver: download & install geckodriver if not already installed (required for robot_tests) -EOF - nvm.help - cat <>> from searx.shared import redisdb + >>> redisdb.init() + True + >>> db = redisdb.client() + >>> db.set("foo", "bar") + True + >>> db.get("foo") + b'bar' + >>> + +""" + +import logging +import redis +from searx import get_setting + +logger = logging.getLogger('searx.shared.redis') +_client = None + + +def client(): + global _client # pylint: disable=global-statement + if _client is None: + # not thread safe: in the worst case scenario, two or more clients are + # initialized only one is kept, the others are garbage collected. + _client = redis.Redis.from_url(get_setting('redis.url')) + return _client + + +def init(): + try: + c = client() + logger.info("connected redis DB --> %s", c.acl_whoami()) + return True + except redis.exceptions.ConnectionError as exc: + logger.error("can't connet redis DB ...") + logger.error(" %s", exc) + return False diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh new file mode 100755 index 00000000..5eaa1770 --- /dev/null +++ b/utils/lib_redis.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Tools to build and install redis [1] binaries & packages. +# +# [1] https://redis.io/download#installation +# +# 1. redis.devpkg (sudo) +# 2. redis.build +# 3. redis.install (sudo) +# +# systemd commands:: +# +# sudo -H systemctl status searxng-redis +# sudo -H journalctl -u searxng-redis +# sudo -H journalctl --vacuum-size=1M +# +# Test socket connection from client (local user):: +# +# $ sudo -H ./manage redis.addgrp "${USER}" +# # logout & login to get member of group +# $ groups +# ... searxng-redis ... +# $ source /usr/local/searxng-redis/.redis_env +# $ which redis-cli +# /usr/local/searxng-redis/.local/bin/redis-cli +# +# $ redis-cli -s /usr/local/searxng-redis/redis.sock +# redis /usr/local/searxng-redis/redis.sock> set foo bar +# OK +# redis /usr/local/searxng-redis/redis.sock> get foo +# "bar" +# [CTRL-D] + + +# shellcheck disable=SC2091 +# shellcheck source=utils/lib.sh +. /dev/null + +REDIS_GIT_URL="https://github.com/redis/redis.git" +REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" + +REDIS_USER="searxng-redis" +REDIS_HOME="/usr/local/${REDIS_USER}" +REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" +REDIS_ENV="${REDIS_HOME}/.redis_env" + +REDIS_SERVICE_NAME="searxng-redis" +REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service" + +# binaries to compile & install +REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli) +# link names of redis-server binary +REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof) + +REDIS_CONF="${REDIS_HOME}/redis.conf" +REDIS_CONF_TEMPLATE=$(cat < and checkput ${REDIS_GIT_TAG} + useradd : create user (${REDIS_USER}) at ${REDIS_HOME} + userdel : delete user (${REDIS_USER}) + addgrp : add to group (${REDIS_USER}) + rmgrp : remove from group (${REDIS_USER}) +EOF +} + +redis.devpkg() { + + # Uses OS package manager to install the essential packages to build and + # compile sources + + sudo_or_exit + + case ${DIST_ID} in + ubuntu|debian) + pkg_install git build-essential + ;; + arch) + pkg_install git base-devel + ;; + fedora) + pkg_install git @development-tools + ;; + centos) + pkg_install git + yum groupinstall "Development Tools" -y + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS." + return 42 + ;; + esac +} + +redis.build() { + + # usage: redis.build + + rst_title "get redis sources" section + redis.src "${CACHE}/redis" + + if ! required_commands gcc nm make gawk; then + sudo -H "$0" redis.devpkg + fi + + rst_title "compile redis sources" section + + pushd "${CACHE}/redis" &>/dev/null + + if ask_yn "Do you run 'make distclean' first'?" Ny; then + $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout + fi + + $(bash.cmd) -c "make" 2>&1 | prefix_stdout + if ask_yn "Do you run 'make test'?" Ny; then + $(bash.cmd) -c "make test" | prefix_stdout + fi + + popd &>/dev/null + + tee_stderr 0.1 <&1 | prefix_stdout +mkdir -p "$(redis._get_dist)" +cd "${CACHE}/redis/src" +cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" +EOF + info_msg "redis binaries available at $(redis._get_dist)" +} + + +redis.install() { + sudo_or_exit + ( + set -e + redis.useradd + redis._install_bin + redis._install_conf + redis._install_service + ) + dump_return $? +} + +redis.remove() { + sudo_or_exit + ( + set -e + redis._remove_service + redis.userdel + ) + dump_return $? +} + +redis.shell() { + interactive_shell "${REDIS_USER}" +} + +redis.src() { + + # usage: redis.src "${CACHE}/redis" + + local dest="${1:-${CACHE}/redis}" + + if [ -d "${dest}" ] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <&1 | prefix_stdout +cd "${dest}" +git fetch --all +git reset --hard tags/${REDIS_GIT_TAG} +EOF + else + tee_stderr 0.1 <&1 | prefix_stdout +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone "${REDIS_GIT_URL}" "${dest}" +EOF + tee_stderr 0.1 <&1 | prefix_stdout +cd "${dest}" +git checkout tags/${REDIS_GIT_TAG} -b "build-branch" +EOF + fi +} + +redis.useradd(){ + + # usage: redis.useradd + + rst_title "add user ${REDIS_USER}" section + echo + sudo_or_exit + + # create user account + tee_stderr 0.5 < "${REDIS_ENV}" +grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile +EOF +} + +redis.userdel() { + sudo_or_exit + drop_service_account "${REDIS_USER}" + groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true +} + +redis.addgrp() { + + # usage: redis.addgrp + + [[ -z $1 ]] && die_caller 42 "missing argument " + sudo -H gpasswd -a "$1" "${REDIS_USER}" +} + +redis.rmgrp() { + + # usage: redis.rmgrp + + [[ -z $1 ]] && die_caller 42 "missing argument " + sudo -H gpasswd -d "$1" "${REDIS_USER}" + +} + + +# private redis. functions +# ------------------------ + +redis._install_bin() { + local src + src="$(redis._get_dist)" + ( + set -e + for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do + install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \ + "${src}/${redis_exe}" "${REDIS_HOME_BIN}" + done + + pushd "${REDIS_HOME_BIN}" &> /dev/null + for redis_exe in "${REDIS_LINK_EXE[@]}"; do + info_msg "link redis-server --> ${redis_exe}" + sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}" + done + popd &> /dev/null + + ) +} + +redis._install_conf() { + sudo -H -u "${REDIS_USER}" bash < "${REDIS_CONF}" +EOF +} + +redis._install_service() { + systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" +} + +redis._remove_service() { + systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}" +} + +redis._get_dist() { + if [ -z "${REDIS_DIST}" ]; then + echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)" + else + echo "${REDIS_DIST}" + fi +} + +redis._arch() { + local ARCH + case "$(command uname -m)" in + "x86_64") ARCH=amd64 ;; + "aarch64") ARCH=arm64 ;; + "armv6" | "armv7l") ARCH=armv6l ;; + "armv8") ARCH=arm64 ;; + .*386.*) ARCH=386 ;; + ppc64*) ARCH=ppc64le ;; + *) die 42 "ARCH is unknown: $(command uname -m)" ;; + esac + echo "${ARCH}" +} + +# TODO: move this to the right place .. + +bash.cmd(){ + + # print cmd to get a bash in a non-root mode, even if we are in a sudo + # context. + + local user="${USER}" + local bash_cmd="bash" + + if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then + user="${SUDO_USER}" + bash_cmd="sudo -H -u ${SUDO_USER} bash" + fi + + printf "%s" "${bash_cmd}" +} diff --git a/utils/templates/lib/systemd/system/searxng-redis.service b/utils/templates/lib/systemd/system/searxng-redis.service new file mode 100644 index 00000000..d1d163f0 --- /dev/null +++ b/utils/templates/lib/systemd/system/searxng-redis.service @@ -0,0 +1,42 @@ +[Unit] + +Description=SearXNG redis service +After=syslog.target +After=network.target +Documentation=https://redis.io/documentation + +[Service] + +Type=simple +User=${REDIS_USER} +Group=${REDIS_USER} +WorkingDirectory=${REDIS_HOME} +Restart=always +TimeoutStopSec=0 + +Environment=USER=${REDIS_USER} HOME=${REDIS_HOME} +ExecStart=${REDIS_HOME_BIN}/redis-server ${REDIS_CONF} +ExecPaths=${REDIS_HOME_BIN} + +LimitNOFILE=65535 +NoNewPrivileges=true +PrivateDevices=yes + +# ProtectSystem=full +ProtectHome=yes +ReadOnlyDirectories=/ +ReadWritePaths=-${REDIS_HOME}/run + +UMask=007 +PrivateTmp=yes + +MemoryDenyWriteExecute=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictRealtime=true +RestrictNamespaces=true + +[Install] + +WantedBy=multi-user.target