From abd5eee3922b18a25ba59bb5a5d59a969ab5deb9 Mon Sep 17 00:00:00 2001
From: Markus Heiser <markus.heiser@darmarit.de>
Date: Tue, 16 Nov 2021 18:00:47 +0100
Subject: [PATCH 1/2] [mod] Tools to install and maintain golang [1] binaries &
 packages.

[1] https://golang.org/doc/devel/release#policy

A simple *helloworld* test with user 'my_user' :

  sudo -H adduser my_user
  ./manage go.golang go1.17.3 my_user
  ./manage go.install github.com/go-training/helloworld@latest my_user
  ./manage go.bash my_user
  $ helloword
  Hello World!!

Don't forget to remove 'my_user':  sudo -H deluser --remove-home my_user

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 Makefile        |   1 +
 manage          |   4 +
 utils/lib_go.sh | 214 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 219 insertions(+)
 create mode 100755 utils/lib_go.sh

diff --git a/Makefile b/Makefile
index 80132336..eb0fa3f7 100644
--- a/Makefile
+++ b/Makefile
@@ -65,6 +65,7 @@ test.shell:
 		utils/lib.sh \
 		utils/lib_install.sh \
 		utils/lib_static.sh \
+		utils/lib_go.sh \
 	        utils/filtron.sh \
 	        utils/searx.sh \
 	        utils/morty.sh \
diff --git a/manage b/manage
index 0c7b04fb..b61b4a19 100755
--- a/manage
+++ b/manage
@@ -9,6 +9,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
 # shellcheck source=utils/lib_static.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
 
+# shellcheck source=utils/lib_go.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
+
 # config
 
 PYOBJECTS="searx"
@@ -91,6 +94,7 @@ themes.:
 pygments.:
   less      : build LESS files for pygments
 EOF
+    go.help
     static_help
 }
 
diff --git a/utils/lib_go.sh b/utils/lib_go.sh
new file mode 100755
index 00000000..314204e1
--- /dev/null
+++ b/utils/lib_go.sh
@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Tools to install and maintain golang [1] binaries & packages.
+#
+# [1] https://golang.org/doc/devel/release#policy
+#
+# A simple *helloworld* test with user 'my_user' :
+#
+#   sudo -H adduser my_user
+#   ./manage go.golang go1.17.3 my_user
+#   ./manage go.install github.com/go-training/helloworld@latest my_user
+#   ./manage go.bash my_user
+#   $ helloword
+#   Hello World!!
+#
+# Don't forget to remove 'my_user':  sudo -H deluser --remove-home my_user
+
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+# shellcheck disable=SC2034
+declare main_cmd
+
+# configure golang environment
+# ----------------------------
+
+[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"
+
+GO_DL_URL="https://golang.org/dl"
+
+# implement go functions
+# -----------------------
+
+go.help(){
+    cat <<EOF
+go.:
+  ls        : list golang binary archives (stable)
+  golang    : (re-) install golang binary in user's \$HOME/local folder
+  install   : install go package in user's \$HOME/go-apps folder
+  bash      : start bash interpreter with golang environment sourced
+EOF
+}
+
+go.ls(){
+    python3 <<EOF
+import sys, json, requests
+resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if not ver['stable']:
+        continue
+    for f in ver['files']:
+        if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
+            continue
+        print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
+EOF
+}
+
+go.ver_info(){
+
+    # print informations about a golang distribution. To print filename
+    # sha256 and size of the archive that fits to your OS and host:
+    #
+    #   go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    #
+    # usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]
+    #
+    # kind:  [archive|source|installer]
+    # os:    [darwin|freebsd|linux|windows]
+    # arch:  [amd64|arm64|386|armv6l|ppc64le|s390x]
+
+    python3 - "$@" <<EOF
+import sys, json, requests
+resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if ver['version'] != sys.argv[1]:
+        continue
+    for f in ver['files']:
+        if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
+            continue
+        for x in sys.argv[5:]:
+           print(f[x])
+        sys.exit(0)
+sys.exit(42)
+EOF
+}
+
+go.os() {
+  local OS
+  case "$(command uname -a)xx" in
+    Linux\ *) OS=linux ;;
+    Darwin\ *) OS=darwin ;;
+    FreeBSD\ *) OS=freebsd ;;
+    CYGWIN* | MSYS* | MINGW*) OS=windows ;;
+    *)  die 42 "OS is unknown: $(command uname -a)" ;;
+  esac
+  echo "${OS}"
+}
+
+go.arch() {
+    local ARCH
+    case "$(command uname -m)" in
+        "x86_64") ARCH=amd64 ;;
+        "aarch64") ARCH=arm64 ;;
+        "armv6" | "armv7l") ARCH=armv6l ;;
+        "armv8") ARCH=arm64 ;;
+        .*386.*) ARCH=386 ;;
+        ppc64*) ARCH=ppc64le ;;
+    *)  die 42 "ARCH is unknown: $(command uname -m)" ;;
+    esac
+    echo "${ARCH}"
+}
+
+go.golang() {
+
+    # install golang binary in user's $HOME/local folder:
+    #
+    #   go.golang ${GO_VERSION} ${SERVICE_USER}
+    #
+    # usage:  go.golang <go-vers> [<username>]
+
+    local version fname sha size user userpr
+    local buf=()
+
+    version="${1:-${GO_VERSION}}"
+    user="${2:-${USERNAME}}"
+    userpr="  ${_Yellow}|${user}|${_creset} "
+
+    rst_title "Install Go in ${user}'s HOME" section
+
+    mapfile -t buf < <(
+        go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    )
+
+    if [ ${#buf[@]} -eq 0 ]; then
+        die 42 "can't find info of golang version: ${version}"
+    fi
+    fname="${buf[0]}"
+    sha="${buf[1]}"
+    size="$(numfmt --to=iec "${buf[2]}")"
+
+    info_msg "Download go binary ${fname} (${size}B)"
+    cache_download "${GO_DL_URL}/${fname}" "${fname}"
+
+    pushd "${CACHE}" &> /dev/null
+    echo "${sha}  ${fname}" > "${fname}.sha256"
+    if ! sha256sum -c "${fname}.sha256" >/dev/null; then
+        die 42 "downloaded file ${fname} checksum does not match"
+    else
+        info_msg "${fname} checksum OK"
+    fi
+    popd &> /dev/null
+
+    info_msg "install golang"
+    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
+mkdir -p \$HOME/local
+rm -rf \$HOME/local/go
+tar -C \$HOME/local -xzf ${CACHE}/${fname}
+echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env
+echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env
+EOF
+    info_msg "test golang installation"
+    sudo -i -u "${user}" <<EOF
+source \$HOME/.go_env
+command -v go
+go version
+EOF
+}
+
+go.install() {
+
+    # install go package in user's $HOME/go-apps folder:
+    #
+    #   go.install github.com/go-training/helloworld@lates ${SERVICE_USER}
+    #
+    # usage:  go.install <package> [<username>]
+
+    local package user userpr
+
+    package="${1}"
+    user="${2:-${USERNAME}}"
+    userpr="  ${_Yellow}|${user}|${_creset} "
+
+    if [ -z "${package}" ]; then
+        die 42 "${FUNCNAME[0]}() - missing argument: <package>"
+    fi
+    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
+source \$HOME/.go_env
+go install -v ${package}
+EOF
+}
+
+go.bash() {
+
+    # start bash interpreter with golang environment sourced
+    #
+    #   go.bash ${SERVICE_USER}
+    #
+    # usage:  go.bash [<username>]
+
+    local user
+    user="${1:-${USERNAME}}"
+    sudo -i -u "${user}" bash --init-file "~${user}/.go_env"
+}
+
+go.version(){
+    local user
+    user="${1:-${USERNAME}}"
+    sudo -i -u "${user}" <<EOF
+source \$HOME/.go_env
+go version | cut -d' ' -f 3
+EOF
+}

From 79b41478ac7290453b73bf00c4cdcd84e588f051 Mon Sep 17 00:00:00 2001
From: Markus Heiser <markus.heiser@darmarit.de>
Date: Tue, 16 Nov 2021 18:03:34 +0100
Subject: [PATCH 2/2] [fix] filtron & morty - install golang binary that fits
 to arch & os

Closes: https://github.com/searxng/searxng/issues/507
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 docs/build-templates/filtron.rst |  9 ++++----
 docs/build-templates/morty.rst   |  9 ++++----
 utils/filtron.sh                 | 26 ++++++------------------
 utils/lib.sh                     | 35 --------------------------------
 utils/morty.sh                   | 26 ++++++------------------
 5 files changed, 22 insertions(+), 83 deletions(-)

diff --git a/docs/build-templates/filtron.rst b/docs/build-templates/filtron.rst
index 79b25439..6a39d1f5 100644
--- a/docs/build-templates/filtron.rst
+++ b/docs/build-templates/filtron.rst
@@ -19,7 +19,8 @@
 
 .. tabs::
 
-  .. group-tab:: bash
+  .. group-tab:: os: linux / arch: amd64
+
 
     .. code-block:: bash
 
@@ -30,9 +31,9 @@
        $ sudo -i -u \"${SERVICE_USER}\"
        (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
        (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
-       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_TAR}\" \\
-                   \"${GO_PKG_URL}\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_TAR}\"
+       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
+                   \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
+       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
        (${SERVICE_USER}) $ which go
        ${SERVICE_HOME}/local/go/bin/go
 
diff --git a/docs/build-templates/morty.rst b/docs/build-templates/morty.rst
index 2be24da5..b03a54f9 100644
--- a/docs/build-templates/morty.rst
+++ b/docs/build-templates/morty.rst
@@ -19,7 +19,8 @@
 
 .. tabs::
 
-  .. group-tab:: bash
+  .. group-tab:: os: linux / arch: amd64
+
 
     .. code-block:: bash
 
@@ -30,9 +31,9 @@
        $ sudo -i -u \"${SERVICE_USER}\"
        (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
        (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
-       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_TAR}\" \\
-                   \"${GO_PKG_URL}\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_TAR}\"
+       (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
+                   \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
+       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
        (${SERVICE_USER}) $ which go
        ${SERVICE_HOME}/local/go/bin/go
 
diff --git a/utils/filtron.sh b/utils/filtron.sh
index 4e8d6aa5..ff380a32 100755
--- a/utils/filtron.sh
+++ b/utils/filtron.sh
@@ -4,6 +4,8 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+# shellcheck source=utils/lib_go.sh
+source "${REPO_ROOT}/utils/lib_go.sh"
 # shellcheck source=utils/lib_install.sh
 source "${REPO_ROOT}/utils/lib_install.sh"
 
@@ -43,8 +45,6 @@ SERVICE_GROUP="${SERVICE_USER}"
 
 GO_ENV="${SERVICE_HOME}/.go_env"
 GO_VERSION="go1.17.2"
-GO_PKG_URL="https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"
-GO_TAR=$(basename "$GO_PKG_URL")
 
 APACHE_FILTRON_SITE="searxng.conf"
 NGINX_FILTRON_SITE="searxng.conf"
@@ -218,7 +218,7 @@ install_all() {
     rst_title "Install $SERVICE_NAME (service)"
     assert_user
     wait_key
-    install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
+    go.golang "${GO_VERSION}" "${SERVICE_USER}"
     wait_key
     install_filtron
     install_rules
@@ -301,9 +301,7 @@ install_check() {
 }
 
 go_version(){
-    sudo -i -u "$SERVICE_USER" <<EOF
-go version | cut -d' ' -f 3
-EOF
+    go.version "${SERVICE_USER}"
 }
 
 remove_all() {
@@ -338,12 +336,6 @@ EOF
     export SERVICE_HOME
     echo "export SERVICE_HOME=$SERVICE_HOME"
 
-    cat > "$GO_ENV" <<EOF
-export GOPATH=\$HOME/go-apps
-export PATH=\$HOME/local/go/bin:\$GOPATH/bin:\$PATH
-EOF
-    echo "Environment $GO_ENV has been setup."
-
     tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
 grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
 EOF
@@ -353,22 +345,16 @@ filtron_is_installed() {
     [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
 }
 
-_svcpr="  ${_Yellow}|${SERVICE_USER}|${_creset} "
-
 install_filtron() {
     rst_title "Install filtron in user's ~/go-apps" section
     echo
-    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
-go install -v github.com/asciimoo/filtron@latest
-EOF
+    go.install github.com/asciimoo/filtron@latest "${SERVICE_USER}"
 }
 
 update_filtron() {
     rst_title "Update filtron" section
     echo
-    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
-go install -v github.com/asciimoo/filtron@latest
-EOF
+    go.install github.com/asciimoo/filtron@latest "${SERVICE_USER}"
 }
 
 install_rules() {
diff --git a/utils/lib.sh b/utils/lib.sh
index 7f6ff58c..4472b9d3 100755
--- a/utils/lib.sh
+++ b/utils/lib.sh
@@ -504,7 +504,6 @@ install_template() {
     done
 }
 
-
 service_is_available() {
 
     # usage:  service_is_available <URL>
@@ -825,40 +824,6 @@ EOF
     build_msg GH-PAGES "deployed"
 }
 
-# golang
-# ------
-
-go_is_available() {
-
-    # usage:  go_is_available $SERVICE_USER && echo "go is installed!"
-
-    sudo -i -u "${1}" which go &>/dev/null
-}
-
-install_go() {
-
-    # usage:  install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
-
-    local _svcpr="  ${_Yellow}|${3}|${_creset} "
-
-    rst_title "Install Go in user's HOME" section
-
-    rst_para "download and install go binary .."
-    cache_download "${1}" "${2}"
-
-    tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr"
-echo \$PATH
-echo \$GOPATH
-mkdir -p \$HOME/local
-rm -rf \$HOME/local/go
-tar -C \$HOME/local -xzf ${CACHE}/${2}
-EOF
-    sudo -i -u "${3}" <<EOF | prefix_stdout
-! which go >/dev/null &&  echo "ERROR - Go Installation not found in PATH!?!"
-which go >/dev/null &&  go version && echo "congratulations -- Go installation OK :)"
-EOF
-}
-
 # system accounts
 # ---------------
 
diff --git a/utils/morty.sh b/utils/morty.sh
index c04df7a0..c1f2bc96 100755
--- a/utils/morty.sh
+++ b/utils/morty.sh
@@ -3,6 +3,8 @@
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+# shellcheck source=utils/lib_go.sh
+source "${REPO_ROOT}/utils/lib_go.sh"
 # shellcheck source=utils/lib_install.sh
 source "${REPO_ROOT}/utils/lib_install.sh"
 
@@ -29,8 +31,6 @@ SERVICE_ENV_DEBUG=false
 
 GO_ENV="${SERVICE_HOME}/.go_env"
 GO_VERSION="go1.17.2"
-GO_PKG_URL="https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"
-GO_TAR=$(basename "$GO_PKG_URL")
 
 # shellcheck disable=SC2034
 CONFIG_FILES=()
@@ -234,7 +234,7 @@ install_all() {
     rst_title "Install $SERVICE_NAME (service)"
     assert_user
     wait_key
-    install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
+    go.golang "${GO_VERSION}" "${SERVICE_USER}"
     wait_key
     install_morty
     wait_key
@@ -306,9 +306,7 @@ install_check() {
 }
 
 go_version(){
-    sudo -i -u "$SERVICE_USER" <<EOF
-go version | cut -d' ' -f 3
-EOF
+    go.version "${SERVICE_USER}"
 }
 
 remove_all() {
@@ -338,12 +336,6 @@ EOF
     export SERVICE_HOME
     echo "export SERVICE_HOME=$SERVICE_HOME"
 
-    cat > "$GO_ENV" <<EOF
-export GOPATH=\$HOME/go-apps
-export PATH=\$HOME/local/go/bin:\$GOPATH/bin:\$PATH
-EOF
-    echo "Environment $GO_ENV has been setup."
-
     tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
 grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
 EOF
@@ -353,22 +345,16 @@ morty_is_installed() {
     [[ -f $SERVICE_HOME/go-apps/bin/morty ]]
 }
 
-_svcpr="  ${_Yellow}|${SERVICE_USER}|${_creset} "
-
 install_morty() {
     rst_title "Install morty in user's ~/go-apps" section
     echo
-    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
-go install -v github.com/asciimoo/morty@latest
-EOF
+    go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
 }
 
 update_morty() {
     rst_title "Update morty" section
     echo
-    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
-go install -v github.com/asciimoo/morty@latest
-EOF
+    go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
 }
 
 set_service_env_debug() {