forked from Archives/searxng
4b43775c91
This commit remove the need to update the brand for GIT_URL and GIT_BRANCH: there are read from the git repository. It is possible to call python -m searx.version freeze to freeze the current version. Useful when the code is installed outside git (distro package, docker, etc...)
549 lines
15 KiB
Bash
Executable File
549 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# shellcheck disable=SC2031
|
|
|
|
# shellcheck source=utils/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
|
|
# shellcheck source=utils/brand.env
|
|
source "${REPO_ROOT}/utils/brand.env"
|
|
source_dot_config
|
|
|
|
# shellcheck source=utils/lib_static.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
|
|
|
|
# config
|
|
|
|
PYOBJECTS="searx"
|
|
PY_SETUP_EXTRAS='[test]'
|
|
GECKODRIVER_VERSION="v0.28.0"
|
|
# SPHINXOPTS=
|
|
|
|
pylint.FILES() {
|
|
|
|
# List files tagged by comment:
|
|
#
|
|
# # lint: pylint
|
|
#
|
|
# These py files are linted by test.pylint(), all other files are linted by
|
|
# test.pep8()
|
|
|
|
grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searx_extra tests
|
|
}
|
|
|
|
YAMLLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
YAMLLINT_FILES+=("$line")
|
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searx/*.yml')"
|
|
|
|
PYLINT_SEARX_DISABLE_OPTION="\
|
|
I,C,R,\
|
|
W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
|
|
E1136"
|
|
PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
|
|
PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
|
|
|
|
help() {
|
|
cat <<EOF
|
|
buildenv:
|
|
rebuild ./utils/brand.env
|
|
babel.compile:
|
|
pybabel compile ./searx/translations
|
|
data.:
|
|
all : update searx/languages.py and ./data/*
|
|
languages : update searx/data/engines_languages.json & searx/languages.py
|
|
useragents: update searx/data/useragents.json with the most recent versions of Firefox.
|
|
docs.:
|
|
html : build HTML documentation
|
|
live : autobuild HTML documentation while editing
|
|
gh-pages : deploy on gh-pages branch
|
|
prebuild : build reST include files (./${DOCS_BUILD}/includes)
|
|
clean : clean documentation build
|
|
docker.:
|
|
build : build docker image
|
|
push : build and push docker image
|
|
gecko.driver:
|
|
download & install geckodriver if not already installed (required for
|
|
robot_tests)
|
|
node.:
|
|
env : download & install npm dependencies locally
|
|
clean : drop npm installations
|
|
py.:
|
|
build : Build python packages at ./${PYDIST}
|
|
clean : delete virtualenv and intermediate py files
|
|
pyenv.:
|
|
install : developer install of searx into virtualenv
|
|
uninstall : uninstall developer installation
|
|
cmd ... : run command ... in virtualenv
|
|
OK : test if virtualenv is OK
|
|
pypi.upload:
|
|
Upload python packages to PyPi (to test use pypi.upload.test)
|
|
test.:
|
|
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
|
pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
|
|
unit : run unit tests
|
|
coverage : run unit tests with coverage
|
|
robot : run robot test
|
|
clean : clean intermediate test stuff
|
|
themes.:
|
|
all : build all themes
|
|
oscar : build oscar theme
|
|
simple : build simple theme
|
|
pygments.:
|
|
less : build LESS files for pygments
|
|
EOF
|
|
static_help
|
|
}
|
|
|
|
|
|
if [ "$VERBOSE" = "1" ]; then
|
|
SPHINX_VERBOSE="-v"
|
|
PYLINT_VERBOSE="-v"
|
|
fi
|
|
|
|
# needed by sphinx-docs
|
|
export DOCS_BUILD
|
|
|
|
buildenv() {
|
|
|
|
# settings file from repository's working tree are used by default
|
|
SEARX_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"
|
|
|
|
if [ -r '/etc/searx/settings.yml' ]; then
|
|
if ask_yn "should settings read from: /etc/searx/settings.yml"; then
|
|
SEARX_SETTINGS_PATH='/etc/searx/settings.yml'
|
|
fi
|
|
fi
|
|
export SEARX_SETTINGS_PATH
|
|
(
|
|
set -e
|
|
SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
|
|
| prefix_stdout "${_Blue}BUILDENV${_creset} "
|
|
)
|
|
return "${PIPESTATUS[0]}"
|
|
}
|
|
|
|
buildenv.unset_env(){
|
|
# Some defaults in the settings.yml are taken from the environment,
|
|
# e.g. SEARX_BIND_ADDRESS (:py:obj:`searx.settings_defaults.SHEMA`). In
|
|
# some tasks (e.g. test.robot) we do not want these envorionment applied.
|
|
unset GIT_URL
|
|
unset GIT_BRANCH
|
|
unset SEARX_URL
|
|
unset SEARX_PORT
|
|
unset SEARX_BIND_ADDRESS
|
|
}
|
|
|
|
babel.compile() {
|
|
build_msg BABEL compile
|
|
pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
|
|
dump_return $?
|
|
}
|
|
|
|
|
|
data.all() {
|
|
data.languages
|
|
data.useragents
|
|
data.osm_keys_tags
|
|
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
|
pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
|
|
build_msg DATA "update searx/data/wikidata_units.json"
|
|
pyenv.cmd python searx_extra/update/update_wikidata_units.py
|
|
build_msg DATA "update searx/data/currencies.json"
|
|
pyenv.cmd python searx_extra/update/update_currencies.py
|
|
}
|
|
|
|
|
|
data.languages() {
|
|
( set -e
|
|
build_msg ENGINES "fetch languages .."
|
|
pyenv.cmd python searx_extra/update/update_languages.py
|
|
build_msg ENGINES "update update searx/languages.py"
|
|
build_msg DATA "update searx/data/engines_languages.json"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
data.useragents() {
|
|
build_msg DATA "update searx/data/useragents.json"
|
|
pyenv.cmd python searx_extra/update/update_firefox_version.py
|
|
dump_return $?
|
|
}
|
|
|
|
data.osm_keys_tags() {
|
|
build_msg DATA "update searx/data/osm_keys_tags.json"
|
|
pyenv.cmd python searx_extra/update/update_osm_keys_tags.py
|
|
dump_return $?
|
|
}
|
|
|
|
docs.prebuild() {
|
|
build_msg DOCS "build ${DOCS_BUILD}/includes"
|
|
(
|
|
set -e
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
mkdir -p "${DOCS_BUILD}/includes"
|
|
./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
|
|
./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
|
|
./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
docker.push() {
|
|
docker.build push
|
|
}
|
|
|
|
docker.buildx() {
|
|
docker.build buildx
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
docker.build() {
|
|
pyenv.install
|
|
|
|
local SEARX_GIT_VERSION
|
|
local VERSION_GITCOMMIT
|
|
local GITHUB_USER
|
|
local SEARX_IMAGE_NAME
|
|
local BUILD
|
|
|
|
build_msg DOCKER build
|
|
# run installation in a subprocess and activate pyenv
|
|
|
|
# See https://www.shellcheck.net/wiki/SC1001 and others ..
|
|
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
|
|
( set -e
|
|
# shellcheck source=/dev/null
|
|
source "${PY_ENV_BIN}/activate"
|
|
|
|
# Check if it is a git repository
|
|
if [ ! -d .git ]; then
|
|
die 1 "This is not Git repository"
|
|
fi
|
|
if [ ! -x "$(which git)" ]; then
|
|
die 1 "git is not installed"
|
|
fi
|
|
|
|
if ! git remote get-url origin 2> /dev/null; then
|
|
die 1 "there is no remote origin"
|
|
fi
|
|
|
|
# This is a git repository
|
|
git update-index -q --refresh
|
|
pyenv.cmd python -m searx.version freeze
|
|
eval "$(pyenv.cmd python -m searx.version)"
|
|
|
|
# Get the last git commit id
|
|
VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3)
|
|
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
|
|
|
|
# define the docker image name
|
|
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
|
|
SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
|
|
|
|
BUILD="build"
|
|
if [ "$1" = "buildx" ]; then
|
|
# buildx includes the push option
|
|
CACHE_TAG="${SEARX_IMAGE_NAME}:latest-build-cache"
|
|
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
|
|
shift
|
|
fi
|
|
build_msg DOCKER "Build command: ${BUILD}"
|
|
|
|
# build Docker image
|
|
build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
|
|
# shellcheck disable=SC2086
|
|
docker $BUILD \
|
|
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
|
|
--build-arg GIT_URL="${GIT_URL}" \
|
|
--build-arg SEARX_GIT_VERSION="${VERSION_STRING}" \
|
|
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
|
|
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
|
|
--build-arg LABEL_VCS_URL="${GIT_URL}" \
|
|
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
|
|
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
|
|
-t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${VERSION_STRING}" .
|
|
|
|
if [ "$1" = "push" ]; then
|
|
docker push "${SEARX_IMAGE_NAME}:latest"
|
|
docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
|
|
fi
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
# shellcheck disable=SC2119
|
|
gecko.driver() {
|
|
pyenv.install
|
|
|
|
build_msg INSTALL "gecko.driver"
|
|
# run installation in a subprocess and activate pyenv
|
|
( set -e
|
|
# shellcheck source=/dev/null
|
|
source "${PY_ENV_BIN}/activate"
|
|
|
|
# TODO : check the current geckodriver version
|
|
geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
|
|
set +e
|
|
if [ -z "$NOTFOUND" ]; then
|
|
build_msg INSTALL "geckodriver already installed"
|
|
return
|
|
fi
|
|
PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
|
|
case "$PLATFORM" in
|
|
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
|
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
|
"windows 32 bit") ARCH="win32";;
|
|
"windows 64 bit") ARCH="win64";;
|
|
"mac 64bit") ARCH="macos";;
|
|
esac
|
|
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
|
|
|
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
|
|
|
|
FILE="$(mktemp)"
|
|
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
|
|
rm -- "$FILE"
|
|
chmod 755 -- "${PY_ENV_BIN}/geckodriver"
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
node.env() {
|
|
if ! required_commands npm; then
|
|
info_msg "to install build tools use::"
|
|
info_msg " sudo -H ./utils/searx.sh install buildhost"
|
|
die 1 "install needed build tools first"
|
|
fi
|
|
|
|
( set -e
|
|
|
|
build_msg INSTALL "searx/static/themes/oscar/package.json"
|
|
npm --prefix searx/static/themes/oscar install
|
|
|
|
build_msg INSTALL "searx/static/themes/simple/package.json"
|
|
npm --prefix searx/static/themes/simple install
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
node.clean() {
|
|
if ! required_commands npm 2>/dev/null; then
|
|
build_msg CLEAN "npm is not installed / ignore npm dependencies"
|
|
return 0
|
|
fi
|
|
build_msg CLEAN "locally installed npm dependencies"
|
|
( set -e
|
|
npm --prefix searx/static/themes/oscar run clean
|
|
npm --prefix searx/static/themes/simple run clean
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
pygments.less() {
|
|
build_msg PYGMENTS "searx_extra/update/update_pygments.py"
|
|
if ! pyenv.cmd python searx_extra/update/update_pygments.py; then
|
|
build_msg PYGMENTS "building LESS files for pygments failed"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
py.build() {
|
|
build_msg BUILD "python package ${PYDIST}"
|
|
pyenv.cmd python setup.py \
|
|
sdist -d "${PYDIST}" \
|
|
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
|
}
|
|
|
|
py.clean() {
|
|
build_msg CLEAN pyenv
|
|
( set -e
|
|
pyenv.drop
|
|
[ "$VERBOSE" = "1" ] && set -x
|
|
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
|
|
find . -name '*.pyc' -exec rm -f {} +
|
|
find . -name '*.pyo' -exec rm -f {} +
|
|
find . -name __pycache__ -exec rm -rf {} +
|
|
)
|
|
}
|
|
|
|
pyenv.check() {
|
|
cat <<EOF
|
|
import yaml
|
|
print('import yaml --> OK')
|
|
EOF
|
|
}
|
|
|
|
pyenv.install() {
|
|
|
|
if ! pyenv.OK; then
|
|
py.clean > /dev/null
|
|
fi
|
|
if pyenv.install.OK > /dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
( set -e
|
|
pyenv
|
|
build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
|
|
"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
|
|
buildenv
|
|
)
|
|
local exit_val=$?
|
|
if [ ! $exit_val -eq 0 ]; then
|
|
die 42 "error while pip install (${PY_ENV_BIN})"
|
|
fi
|
|
}
|
|
|
|
pyenv.uninstall() {
|
|
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
|
|
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
|
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
|
|
|
}
|
|
|
|
pypi.upload() {
|
|
py.clean
|
|
py.build
|
|
# https://github.com/pypa/twine
|
|
pyenv.cmd twine upload "${PYDIST}"/*
|
|
}
|
|
|
|
pypi.upload.test() {
|
|
py.clean
|
|
py.build
|
|
pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
|
|
}
|
|
|
|
test.yamllint() {
|
|
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
|
pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
|
|
}
|
|
|
|
test.pylint() {
|
|
# shellcheck disable=SC2086
|
|
( set -e
|
|
build_msg TEST "[pylint] \$PYLINT_FILES"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
"${PYLINT_FILES[@]}"
|
|
|
|
build_msg TEST "[pylint] searx/engines"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARX_DISABLE_OPTION}" \
|
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
searx/engines
|
|
|
|
build_msg TEST "[pylint] searx tests"
|
|
pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
--disable="${PYLINT_SEARX_DISABLE_OPTION}" \
|
|
--ignore=searx/engines \
|
|
searx tests
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.pep8() {
|
|
build_msg TEST 'pycodestyle (formerly pep8)'
|
|
local _exclude=""
|
|
printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
|
|
pyenv.cmd pycodestyle \
|
|
--exclude="searx/static, searx/languages.py, $_exclude " \
|
|
--max-line-length=120 \
|
|
--ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
|
|
searx tests
|
|
dump_return $?
|
|
}
|
|
|
|
test.unit() {
|
|
build_msg TEST 'tests/unit'
|
|
pyenv.cmd python -m nose2 -s tests/unit
|
|
dump_return $?
|
|
}
|
|
|
|
test.coverage() {
|
|
build_msg TEST 'unit test coverage'
|
|
( set -e
|
|
pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
|
pyenv.cmd coverage report
|
|
pyenv.cmd coverage html
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
test.robot() {
|
|
build_msg TEST 'robot'
|
|
buildenv.unset_env
|
|
gecko.driver
|
|
PYTHONPATH=. pyenv.cmd python searx/testing.py robot
|
|
dump_return $?
|
|
}
|
|
|
|
test.clean() {
|
|
build_msg CLEAN "test stuff"
|
|
rm -rf geckodriver.log .coverage coverage/
|
|
dump_return $?
|
|
}
|
|
|
|
themes.all() {
|
|
( set -e
|
|
pygments.less
|
|
node.env
|
|
themes.oscar
|
|
themes.simple
|
|
)
|
|
dump_return $?
|
|
}
|
|
|
|
themes.oscar() {
|
|
build_msg GRUNT "theme: oscar"
|
|
npm --prefix searx/static/themes/oscar run build
|
|
dump_return $?
|
|
}
|
|
|
|
themes.simple() {
|
|
build_msg GRUNT "theme: simple"
|
|
npm --prefix searx/static/themes/simple run build
|
|
dump_return $?
|
|
}
|
|
|
|
PYLINT_FILES=()
|
|
while IFS= read -r line; do
|
|
PYLINT_FILES+=("$line")
|
|
done <<< "$(pylint.FILES)"
|
|
|
|
# shellcheck disable=SC2119
|
|
main() {
|
|
|
|
local _type
|
|
local cmd="$1"; shift
|
|
|
|
if [ "$cmd" == "" ]; then
|
|
help
|
|
err_msg "missing command"
|
|
return 42
|
|
fi
|
|
|
|
case "$cmd" in
|
|
--getenv) var="$1"; echo "${!var}";;
|
|
--help) help;;
|
|
--*)
|
|
help
|
|
err_msg "unknown option $cmd"
|
|
return 42
|
|
;;
|
|
*)
|
|
_type="$(type -t "$cmd")"
|
|
if [ "$_type" != 'function' ]; then
|
|
err_msg "unknown command: $cmd / use --help"
|
|
return 42
|
|
else
|
|
"$cmd" "$@"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|