From ed8a1690296cfc55f67dc0a4dac6dee06a47bd06 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 14 Jun 2022 10:02:50 +0200 Subject: [PATCH] [doc] update documentation of the installation procedures Signed-off-by: Markus Heiser --- docs/admin/arch_public.dot | 39 +- docs/admin/architecture.rst | 20 +- docs/admin/buildhosts.rst | 8 +- docs/admin/engines/nosql-engines.rst | 8 +- docs/admin/engines/settings.rst | 13 + docs/admin/engines/sql-engines.rst | 8 +- docs/admin/filtron.rst | 193 -------- docs/admin/index.rst | 5 +- docs/admin/installation-apache.rst | 425 ++++++------------ docs/admin/installation-docker.rst | 59 ++- docs/admin/installation-nginx.rst | 287 ++++-------- docs/admin/installation-scripts.rst | 62 +++ docs/admin/installation-searxng.rst | 58 ++- docs/admin/installation-switch2ng.rst | 8 +- docs/admin/installation-uwsgi.rst | 206 +++++++-- docs/admin/installation.rst | 100 +---- docs/admin/morty.rst | 40 -- docs/admin/update-searxng.rst | 60 +-- .../{searx.rst => searxng.rst} | 24 +- docs/conf.py | 2 +- docs/dev/engine_overview.rst | 8 +- docs/dev/lxcdev.rst | 185 ++++---- docs/dev/makefile.rst | 26 +- docs/dev/offline_engines.rst | 6 +- docs/dev/plugins.rst | 27 +- docs/dev/quickstart.rst | 14 +- docs/utils/filtron.sh.rst | 80 ---- docs/utils/index.rst | 41 +- docs/utils/lxc.sh.rst | 16 +- docs/utils/morty.sh.rst | 80 ---- docs/utils/searx.sh.rst | 39 -- docs/utils/searxng.sh.rst | 36 ++ manage | 2 +- utils/searxng.sh | 123 +++++ 34 files changed, 921 insertions(+), 1387 deletions(-) delete mode 100644 docs/admin/filtron.rst create mode 100644 docs/admin/installation-scripts.rst delete mode 100644 docs/admin/morty.rst rename docs/build-templates/{searx.rst => searxng.rst} (88%) delete mode 100644 docs/utils/filtron.sh.rst delete mode 100644 docs/utils/morty.sh.rst delete mode 100644 docs/utils/searx.sh.rst create mode 100644 docs/utils/searxng.sh.rst diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index c4ee5f3c..526fb53d 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -1,33 +1,30 @@ digraph G { - node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"]; + node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans]; edge [fontname="Sans"]; - browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="https://docs.searxng.org/utils/filtron.sh.html#public-reverse-proxy"]; - filtron [label="Filtron", href="https://docs.searxng.org/utils/filtron.sh.html"]; - morty [label="Morty", href="https://docs.searxng.org/utils/morty.sh.html"]; - static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="https://docs.searxng.org/utils/searx.sh.html"] - searx1 [label="Searx #1"]; - searx2 [label="Searx #2"]; - searx3 [label="Searx #3"]; - searx4 [label="Searx #4"]; + browser [label="browser", shape=tab, fillcolor=aliceblue]; + rp [label="reverse proxy"]; + static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray]; + uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"] + redis [label="redis DB", shape=cylinder]; + searxng1 [label="SearXNG #1", fontcolor=blue3]; + searxng2 [label="SearXNG #2", fontcolor=blue3]; + searxng3 [label="SearXNG #3", fontcolor=blue3]; + searxng4 [label="SearXNG #4", fontcolor=blue3]; browser -> rp [label="HTTPS"] - subgraph cluster_searx { - label = "Searx instance" fontname="Sans"; + subgraph cluster_searxng { + label = "SearXNG instance" fontname=Sans; bgcolor="#fafafa"; { rank=same; static rp }; - rp -> morty [label="optional: images and HTML pages proxy"]; - rp -> static [label="optional: reverse proxy serves directly static files"]; - rp -> filtron [label="HTTP"]; - filtron -> uwsgi [label="HTTP"]; - uwsgi -> searx1; - uwsgi -> searx2; - uwsgi -> searx3; - uwsgi -> searx4; + rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray]; + rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"]; + uwsgi -> searxng1 -> redis; + uwsgi -> searxng2 -> redis; + uwsgi -> searxng3 -> redis; + uwsgi -> searxng4 -> redis; } } diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index db99c9f5..be99aa99 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -8,17 +8,19 @@ Architecture - Reverse Proxy: :ref:`Apache ` & :ref:`nginx ` - - Filtron: :ref:`searxng filtron` - - Morty: :ref:`searxng morty` - uWSGI: :ref:`searxng uwsgi` - SearXNG: :ref:`installation basic` Herein you will find some hints and suggestions about typical architectures of SearXNG infrastructures. -We start with a contribution from :pull-searx:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public SearXNG instances which can build up and -maintained by the scripts from our :ref:`toolboxing`. +.. _architecture uWSGI: + +uWSGI Setup +=========== + +We start with a *reference* setup for public SearXNG instances which can build +up and maintained by the scripts from our :ref:`toolboxing`. .. _arch public: @@ -26,3 +28,11 @@ maintained by the scripts from our :ref:`toolboxing`. :alt: arch_public.dot Reference architecture of a public SearXNG setup. + +The reference installation activates ``server.limiter``, ``server.image_proxy`` +and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml +`) + +.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # preferences: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index e23327ba..6926469e 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -15,19 +15,19 @@ Buildhosts :backlinks: entry To get best results from build, its recommend to install additional packages -on build hosts (see :ref:`searx.sh`).:: +on build hosts (see :ref:`searxng.sh`).:: - sudo -H ./utils/searx.sh install buildhost + sudo -H ./utils/searxng.sh install buildhost This will install packages needed by searx: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages and packages needed to build docuemtation and run tests: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START build-packages :end-before: END build-packages diff --git a/docs/admin/engines/nosql-engines.rst b/docs/admin/engines/nosql-engines.rst index a50b9c36..68fd0e8b 100644 --- a/docs/admin/engines/nosql-engines.rst +++ b/docs/admin/engines/nosql-engines.rst @@ -42,11 +42,11 @@ Extra Dependencies For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine redis_server: diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 71ad29b4..0b4b984d 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -207,10 +207,14 @@ Global Settings ``secret_key`` : ``$SEARXNG_SECRET`` Used for cryptography purpose. +.. _limiter: + ``limiter`` : Rate limit the number of request on the instance, block some bots. The :ref:`limiter plugin` requires a :ref:`settings redis` database. +.. _image_proxy: + ``image_proxy`` : Allow your instance of SearXNG of being able to proxy images. Uses memory space. @@ -225,9 +229,13 @@ Global Settings ``ui:`` ------- +.. _cache busting: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting + .. code:: yaml ui: + static_use_hash: false default_locale: "" query_in_title: false infinite_scroll: false @@ -236,6 +244,11 @@ Global Settings theme_args: simple_style: auto +.. _static_use_hash: + +``static_use_hash`` : + Enables `cache busting`_ of static files. + ``default_locale`` : SearXNG interface language. If blank, the locale is detected by using the browser language. If it doesn't work, or you are deploying a language diff --git a/docs/admin/engines/sql-engines.rst b/docs/admin/engines/sql-engines.rst index d9138321..6b6a4cb6 100644 --- a/docs/admin/engines/sql-engines.rst +++ b/docs/admin/engines/sql-engines.rst @@ -98,11 +98,11 @@ Extra Dependencies For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine postgresql: diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst deleted file mode 100644 index 60d7cbeb..00000000 --- a/docs/admin/filtron.rst +++ /dev/null @@ -1,193 +0,0 @@ - -.. _searxng filtron: - -========================== -How to protect an instance -========================== - -.. tip:: - - To protect your instance a installation of filtron (as described here) is no - longer needed, alternatively activate the :ref:`limiter plugin` in your - ``settings.yml``. Note that the :ref:`limiter plugin` requires a :ref:`Redis - ` database. - - -.. sidebar:: further reading - - - :ref:`filtron.sh` - - :ref:`nginx searxng site` - -.. _filtron: https://github.com/searxng/filtron - -SearXNG depends on external search services. To avoid the abuse of these services -it is advised to limit the number of requests processed by SearXNG. - -An application firewall, filtron_ solves exactly this problem. Filtron is just -a middleware between your web server (nginx, apache, ...) and searx, we describe -such infrastructures in chapter: :ref:`architecture`. - - -filtron & go -============ - -.. _Go: https://golang.org/ -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply -installed by ``go get`` package management (see `filtron README`_). If you use -filtron as middleware, a more isolated setup is recommended. To simplify such -an installation and the maintenance of, use our script :ref:`filtron.sh`. - -.. _Sample configuration of filtron: - -Sample configuration of filtron -=============================== - -.. sidebar:: Tooling box - - - :origin:`/etc/filtron/rules.json ` - -An example configuration can be find below. This configuration limits the access -of: - -- scripts or applications (roboagent limit) -- webcrawlers (botlimit) -- IPs which send too many requests (IP limit) -- too many json, csv, etc. requests (rss/json limit) -- the same UserAgent of if too many requests (useragent limit) - -.. code:: json - - [ - { - "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": "", - "limit": "", - "subrules": [ - { - "name": "missing Accept-Language", - "filters": ["!Header:Accept-Language"], - "limit": "", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "suspiciously Connection=close header", - "filters": ["Header:Connection=close"], - "limit": "", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": "", - "limit": "", - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "filters": [ - "Param:format=(csv|json|rss)" - ], - "interval": "", - "limit": "", - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": "", - "limit": "", - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } - ] - - -.. _filtron route request: - -Route request through filtron -============================= - -.. sidebar:: further reading - - - :ref:`filtron.sh overview` - - :ref:`installation nginx` - - :ref:`installation apache` - -Filtron can be started using the following command: - -.. code:: sh - - $ filtron -rules rules.json - -It listens on ``127.0.0.1:4004`` and forwards filtered requests to -``127.0.0.1:8888`` by default. - -Use it along with ``nginx`` with the following example configuration. - -.. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static { - /usr/local/searx/searx-src/searx/static; - } - - -Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a SearXNG is being run. For a complete setup see: :ref:`nginx -searxng site`. diff --git a/docs/admin/index.rst b/docs/admin/index.rst index e6c0636b..17a75888 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -7,17 +7,16 @@ Administrator documentation :caption: Contents installation + installation-docker + installation-scripts installation-searxng installation-uwsgi installation-nginx installation-apache - installation-docker installation-switch2ng update-searxng engines/index api architecture - filtron - morty plugins buildhosts diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst index b60e20ad..ca919812 100644 --- a/docs/admin/installation-apache.rst +++ b/docs/admin/installation-apache.rst @@ -1,13 +1,13 @@ .. _installation apache: -=================== -Install with apache -=================== +====== +Apache +====== .. _Apache: https://httpd.apache.org/ .. _Apache Debian: https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x): -.. _README.Debian: +.. _apache2.README.Debian: https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian .. _Apache Arch Linux: https://wiki.archlinux.org/index.php/Apache_HTTP_Server @@ -23,7 +23,9 @@ Install with apache https://httpd.apache.org/docs/current/en/configuring.html .. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost .. _LoadModule: - https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule + https://httpd.apache.org/docs/mod/mod_so.html#loadmodule +.. _IncludeOptional: + https://httpd.apache.org/docs/mod/core.html#includeoptional .. _DocumentRoot: https://httpd.apache.org/docs/trunk/mod/core.html#documentroot .. _Location: @@ -32,11 +34,30 @@ Install with apache https://uwsgi-docs.readthedocs.io/en/latest/Apache.html .. _mod_proxy_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi +.. _mod_proxy_http: + https://httpd.apache.org/docs/current/mod/mod_proxy_http.html +.. _mod_proxy: + https://httpd.apache.org/docs/current/mod/mod_proxy.html + + +This section explains how to set up a SearXNG site using the HTTP server Apache_. +If you have used the :ref:`installation scripts` and do not have any preference +you can install the :ref:`SearXNG site ` using +:ref:`searxng.sh `: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install apache + +If you have special interests or problems with setting up Apache, the following +section might give you some guidance. + .. sidebar:: further read - `Apache Arch Linux`_ - - `Apache Debian`_ and `README.Debian`_ + - `Apache Debian`_ + - `apache2.README.Debian`_ - `Apache Fedora`_ - `Apache directives`_ @@ -45,23 +66,8 @@ Install with apache :local: :backlinks: entry ----- - -**Install** :ref:`apache searxng site` using :ref:`filtron.sh ` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh apache install - -**Install** :ref:`apache searxng site` using :ref:`morty.sh ` - -.. code:: bash - $ sudo -H ./utils/morty.sh apache install - ----- - -The apache HTTP server +The Apache HTTP server ====================== If Apache_ is not installed, install it now. If apache_ is new to you, the @@ -73,13 +79,13 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install apache2 .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H pacman -S apache sudo -H systemctl enable httpd @@ -87,7 +93,7 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H dnf install httpd sudo -H systemctl enable httpd @@ -101,7 +107,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/apache2/sites-enabled/000-default.conf @@ -115,7 +121,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -130,8 +136,8 @@ How this default intro site is configured, depends on the linux distribution Require all granted - The *welcome* page of Arch Linux is a page showing directory located at - ``DocumentRoot``. This is *directory* page is generated by the Module + The *welcome* page of Arch Linux is a page showing the directory located + at ``DocumentRoot``. This *directory* page is generated by the Module `mod_autoindex `_: .. code:: apache @@ -142,7 +148,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -163,323 +169,204 @@ How this default intro site is configured, depends on the linux distribution less /etc/httpd/conf.d/welcome.conf -.. _apache searxng site: -Apache Reverse Proxy -==================== +.. _Debian's Apache layout: -.. sidebar:: public to the internet? +Debian's Apache layout +---------------------- - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy ` and :ref:`result proxy morty `, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Be aware, Debian's Apache layout is quite different from the standard Apache +configuration. For details look at the apache2.README.Debian_ +(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on +Debian: -To setup a Apache revers proxy you have to enable the *headers* and *proxy* -modules and create a `Location`_ configuration for the SearXNG site. In most -distributions you have to un-comment the lines in the main configuration file, -except in :ref:`The Debian Layout`. +* :man:`apache2ctl`: Apache HTTP server control interface +* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules +* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations +* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites + +.. _apache modules: + +Apache modules +-------------- + +To load additional modules, in most distributions you have to un-comment the +lines with the corresponding LoadModule_ directive, except in :ref:`Debian's +Apache layout`. .. tabs:: .. group-tab:: Ubuntu / debian - In the Apache setup, enable headers and proxy modules: + :ref:`Debian's Apache layout` uses :man:`a2enmod` and :man:`a2dismod` to + activate or disable modules: - .. code:: sh + .. code:: bash + sudo -H a2enmod ssl sudo -H a2enmod headers sudo -H a2enmod proxy sudo -H a2enmod proxy_http - - In :ref:`The Debian Layout` you create a ``searxng.conf`` with the - ```` directive and save this file in the *sites - available* folder at ``/etc/apache2/sites-available``. To enable the - ``searxng.conf`` use :man:`a2ensite`: - - .. code:: sh - - sudo -H a2ensite searxng.conf + sudo -H a2enmod proxy_uwsgi .. group-tab:: Arch Linux - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test - - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so .. group-tab:: Fedora / RHEL - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so -With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the -proxied host. +.. _apache sites: -.. _apache searxng via filtron plus morty: - -.. tabs:: - - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture ` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron `, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: apache - - - - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://127.0.0.1:4004 - RequestHeader set X-Script-Name /searx - - - - 2. Configure reverse proxy for :ref:`morty `, listening on - *localhost 3000* - - .. code:: apache - - ProxyPreserveHost On - - - - # SetEnvIf Request_URI "/morty" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPass http://127.0.0.1:3000 - RequestHeader set X-Script-Name /morty - - - - For a fully result proxification add :ref:`morty's ` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - -uWSGI support -============= - -Be warned, with this setup, your instance isn't :ref:`protected `, nevertheless it is good enough for intranet usage. In modern Linux -distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache -package and you need to install only the :ref:`uWSGI ` package: +Apache sites +------------ .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + In :ref:`Debian's Apache layout` you create a ``searxng.conf`` with the + ```` directive and save this file in the *sites + available* folder at ``/etc/apache2/sites-available``. To enable the + ``searxng.conf`` use :man:`a2ensite`: - sudo -H apt-get install uwsgi + .. code:: bash - # Ubuntu =< 18.04 - sudo -H apt-get install libapache2-mod-proxy-uwsgi + sudo -H a2ensite searxng.conf .. group-tab:: Arch Linux - .. code:: sh - - sudo -H pacman -S uwsgi - - .. group-tab:: Fedora / RHEL - - .. code:: sh - - sudo -H dnf install uwsgi - -The next example shows a configuration using the `uWSGI Apache support`_ via -unix sockets and `mod_proxy_uwsgi`_. - -For socket communication, you have to activate ``socket = -/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888`` -configuration in your :ref:`uwsgi ini file `. If not -already exists, create a folder for the unix sockets, which can be used by the -SearXNG account (see :ref:`create searxng user`): - -.. code:: bash - - sudo -H mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - -If the server is public; to limit access to your intranet replace ``Allow from -all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class. - -.. tabs:: - - .. group-tab:: Ubuntu / debian + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: .. code:: apache - LoadModule headers_module /usr/lib/apache2/mod_headers.so - LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so - LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + IncludeOptional sites-enabled/*.conf - + Create two folders, one for the *available sites* and one for the *enabled sites*: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: bash - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - .. group-tab:: Arch Linux - - .. code:: apache + .. code:: bash - FIXME needs test + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + .. group-tab:: Fedora / RHEL - + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: apache - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + IncludeOptional sites-enabled/*.conf - + Create two folders, one for the *available sites* and one for the *enabled sites*: - .. group-tab:: Fedora / RHEL + .. code:: bash - .. code:: apache + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - FIXME needs test + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - + .. code:: bash - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all +.. _apache searxng site: - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ +Apache's SearXNG site +===================== - +.. _mod_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi - +.. sidebar:: uWSGI - .. group-tab:: old mod_wsgi + Use mod_proxy_uwsgi_ / don't use the old mod_uwsgi_ anymore. - We show this only for historical reasons, DON'T USE `mod_uwsgi - `_. - ANYMORE! +To proxy the incoming requests to the SearXNG instance Apache needs the +mod_proxy_ module (:ref:`apache modules`). - .. code:: apache +.. sidebar:: HTTP headers - + With ProxyPreserveHost_ the incoming ``Host`` header is passed to the proxied + host. - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +Depending on what your SearXNG installation is listen, you need a http +mod_proxy_http_) or socket (mod_proxy_uwsgi_) communication to upstream. - +The :ref:`installation scripts` installs by default the :ref:`reference setup +` and a :ref:`uwsgi setup` that listens on a socket. +You can install and activate your own ``searxng.conf`` like shown in +:ref:`apache sites`. - Require all granted +.. tabs:: - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket + .. group-tab:: socket - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache socket + :end-before: END apache socket - + .. group-tab:: http - + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache http + :end-before: END apache http .. _restart apache: -Restart service -=============== +Restart service: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart apache2 - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd sudo -H touch /etc/uwsgi.d/searxng.ini @@ -489,27 +376,13 @@ disable logs ============ For better privacy you can disable Apache logs. In the examples above activate -one of the lines and `restart apache`_:: - +one of the lines and `restart apache`_: - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +.. code:: apache -The ``CustomLog`` directive disable logs for the whole (virtual) server, use it -when the URL of the service does not have a path component (``/searx``) / is -located at root (``/``). + SetEnvIf Request_URI "/searxng" dontlog + # CustomLog /dev/null combined env=dontlog -.. _The Debian Layout: - -The Debian Layout -================= - -Be aware that the Debian layout is quite different from the standard Apache -configuration. For details look at the README.Debian_ -(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on -Debian: - -* :man:`apache2ctl`: Apache HTTP server control interface -* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules -* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations -* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites +The ``CustomLog`` directive disable logs of the entire (virtual) server, use it +when the URL of the service does not have a path component (``/searxng``), when +SearXNG is located at root (``/``). diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst index cd91af38..7800472e 100644 --- a/docs/admin/installation-docker.rst +++ b/docs/admin/installation-docker.rst @@ -1,37 +1,60 @@ - .. _installation docker: -=================== -Docker installation -=================== +================ +Docker Container +================ .. _ENTRYPOINT: https://docs.docker.com/engine/reference/builder/#entrypoint +.. _searxng/searxng @dockerhub: https://hub.docker.com/r/searxng/searxng .. _searxng-docker: https://github.com/searxng/searxng-docker -.. _[filtron]: https://hub.docker.com/r/dalf/filtron -.. _[morty]: https://hub.docker.com/r/dalf/morty .. _[caddy]: https://hub.docker.com/_/caddy +.. _Redis: https://redis.io/ + +---- .. sidebar:: info + - `searxng/searxng @dockerhub`_ - :origin:`Dockerfile` - - `searxng/searxng @dockerhub `_ - `Docker overview `_ - - `Docker Cheat Sheet `_ - - `Alpine Linux `_ `(wiki) `__ `apt packages `_ + - `Docker Cheat Sheet `_ + - `Alpine Linux `_ + `(wiki) `__ + `apt packages `_ - Alpine's ``/bin/sh`` is :man:`dash` -.. tip:: +**If you intend to create a public instance using Docker, use our well maintained +docker container** + +- `searxng/searxng @dockerhub`_. + +.. sidebar:: hint + + The rest of this article is of interest only to those who want to create and + maintain their own Docker images. + +The sources are hosted at searxng-docker_ and the container includes: + +- a HTTPS reverse proxy `[caddy]`_ and +- a Redis_ DB + +The `default SearXNG setup `_ +of this container: + +- enables :ref:`limiter ` to protect against bots +- enables :ref:`image proxy ` for better privacy +- enables :ref:`cache busting ` to save bandwith + +---- - If you intend to create a public instance using Docker, use our well - maintained searxng-docker_ image which includes - - :ref:`protection ` `[filtron]`_, - - a :ref:`result proxy ` `[morty]`_ and - - a HTTPS reverse proxy `[caddy]`_. +Get Docker +========== -Make sure you have `installed Docker `_ and -on Linux, don't forget to add your user to the docker group (log out and log -back in so that your group membership is re-evaluated): +If you plan to build and maintain a docker image by your own, make sure you have +`Docker installed `_ and on Linux, don't +forget to add your user to the docker group (log out and log back in so that +your group membership is re-evaluated): .. code:: sh diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst index fdbcdf03..33b6df53 100644 --- a/docs/admin/installation-nginx.rst +++ b/docs/admin/installation-nginx.rst @@ -1,8 +1,8 @@ .. _installation nginx: -================== -Install with nginx -================== +===== +NGINX +===== .. _nginx: https://docs.nginx.com/nginx/admin-guide/ @@ -19,6 +19,19 @@ Install with nginx .. _SCRIPT_NAME: https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name +This section explains how to set up a SearXNG site using the HTTP server nginx_. +If you have used the :ref:`installation scripts` and do not have any preference +you can install the :ref:`SearXNG site ` using +:ref:`searxng.sh `: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install nginx + +If you have special interests or problems with setting up nginx, the following +section might give you some guidance. + + .. sidebar:: further reading - nginx_ @@ -27,39 +40,23 @@ Install with nginx - `Getting Started wiki`_ - `uWSGI support from nginx`_ + .. contents:: Contents :depth: 2 :local: :backlinks: entry ----- - -**Install** :ref:`nginx searxng site` using :ref:`filtron.sh ` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh nginx install - -**Install** :ref:`nginx searxng site` using :ref:`morty.sh ` - -.. code:: bash - - $ sudo -H ./utils/morty.sh nginx install - ----- - The nginx HTTP server ===================== -If nginx_ is not installed (uwsgi will not work with the package nginx-light), -install it now. +If nginx_ is not installed, install it now. .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install nginx @@ -88,11 +85,11 @@ depends on the linux distribution: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx @@ -104,7 +101,7 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - in there is a configuration section named ``server``: + There is a configuration section named ``server``: .. code-block:: nginx @@ -120,249 +117,121 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx include /etc/nginx/conf.d/*.conf; + .. _nginx searxng site: -A nginx SearXNG site +NGINX's SearXNG site ==================== -.. sidebar:: public to the internet? - - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy ` and :ref:`result proxy morty `, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Now you have to create a configuration file (``searxng.conf``) for the SearXNG +site. If nginx_ is new to you, the `nginx beginners guide`_ is a good starting +point and the `Getting Started wiki`_ is always a good resource *to keep in the +pocket*. -Now you have to create a configuration for the SearXNG site. If nginx_ is new to -you, the `nginx beginners guide`_ is a good starting point and the `Getting -Started wiki`_ is always a good resource *to keep in the pocket*. +Depending on what your SearXNG installation is listen, you need a http or socket +communication to upstream. .. tabs:: - .. group-tab:: Ubuntu / debian - - Create configuration at ``/etc/nginx/sites-available/searxng`` and place a - symlink to sites-enabled: - - .. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searxng /etc/nginx/sites-enabled/searxng - - .. group-tab:: Arch Linux + .. group-tab:: socket - In the ``/etc/nginx/nginx.conf`` file, replace the configuration section - named ``server``. + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx socket + :end-before: END nginx socket - .. group-tab:: Fedora / RHEL + .. group-tab:: http - Create configuration at ``/etc/nginx/conf.d/searxng`` and place a - symlink to sites-enabled: + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx http + :end-before: END nginx http -.. _nginx searxng via filtron plus morty: +The :ref:`installation scripts` installs by default the :ref:`reference setup +` and a :ref:`uwsgi setup` that listens on a socket. .. tabs:: - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture ` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron `, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } - - - 2. Configure reverse proxy for :ref:`morty `, listening on - *localhost 3000*: - - .. code:: nginx - - # https://example.org/morty - - location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - } - - For a fully result proxification add :ref:`morty's ` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - - - .. group-tab:: proxy or uWSGI - - Be warned, with this setup, your instance isn't :ref:`protected `. Nevertheless it is good enough for intranet usage and it is a - excellent example of; *how different services can be set up*. The next - example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI - application `, listening on ``http = - 127.0.0.1:8888``. - - .. code:: nginx + .. group-tab:: Ubuntu / debian - # https://hostname.local/ + Create configuration at ``/etc/nginx/sites-available/`` and place a + symlink to ``sites-enabled``: - location / { - proxy_pass http://127.0.0.1:8888; + .. code:: bash - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_buffering off; - } + sudo -H ln -s /etc/nginx/sites-available/searxng.conf \ + /etc/nginx/sites-enabled/searxng.conf - Alternatively you can use the `uWSGI support from nginx`_ via unix - sockets. For socket communication, you have to activate ``socket = - /run/uwsgi/app/searx/socket`` and comment out the ``http = - 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file `. + .. group-tab:: Arch Linux - The example shows a nginx virtual ``server`` configuration, listening on - port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at - location ``/`` by importing the `uwsgi_params`_ and passing requests to - the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the - :ref:`searx-src clone ` and wraps directly the - :origin:`searx/static/` content at ``location /static``. + In the ``/etc/nginx/nginx.conf`` file, in the ``server`` section add a + `include `_ + directive: .. code:: nginx server { - # replace hostname.local with your server's name - server_name hostname.local; - - listen 80; - listen [::]:80; - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - root /usr/local/searx/searx-src/searx; - location /static { } + # ... + include /etc/nginx/default.d/*.conf; + # ... } - If not already exists, create a folder for the unix sockets, which can be - used by the SearXNG account: + Create two folders, one for the *available sites* and one for the *enabled sites*: .. code:: bash - mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - - .. group-tab:: \.\. at subdir URL - - Be warned, with these setups, your instance isn't :ref:`protected `. The examples are just here to demonstrate how to export the - SearXNG application from a subdirectory URL ``https://example.org/searx/``. - - .. code:: nginx - - # https://hostname.local/searx - - location /searx { - proxy_pass http://127.0.0.1:8888; + mkdir -p /etc/nginx/default.d + mkdir -p /etc/nginx/default.apps-available - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``default.d``: - The ``X-Script-Name /searx`` is needed by the SearXNG implementation to - calculate relative URLs correct. The next example shows a uWSGI - configuration. Since there are no HTTP headers in a (u)WSGI protocol, the - value is shipped via the SCRIPT_NAME_ in the WSGI environment. + .. code:: bash - .. code:: nginx + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/default.d/searxng.conf - # https://hostname.local/searx + .. group-tab:: Fedora / RHEL - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } + Create a folder for the *available sites*: - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/; - } + .. code:: bash - For SearXNG to work correctly the ``base_url`` must be set in the - :origin:`searx/settings.yml`. + mkdir -p /etc/nginx/default.apps-available - .. code:: yaml + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``conf.d``: - server: - # replace example.org with your server's public name - base_url : https://example.org/searx/ + .. code:: bash + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/conf.d/searxng.conf -Restart service: +Restart services: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng - .. group-tab:: Fedora + .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx sudo -H touch /etc/uwsgi.d/searxng.ini diff --git a/docs/admin/installation-scripts.rst b/docs/admin/installation-scripts.rst new file mode 100644 index 00000000..0efeee2a --- /dev/null +++ b/docs/admin/installation-scripts.rst @@ -0,0 +1,62 @@ +.. _installation scripts: + +=================== +Installation Script +=================== + +.. sidebar:: Update OS first! + + To avoid unwanted side effects, update your OS before installing SearXNG. + +The following will install a setup as shown in :ref:`the reference architecture +`. First you need to get a clone. The clone is only needed for +the installation procedure and some maintenance tasks. + +.. sidebar:: further read + + - :ref:`toolboxing` + +Jump to a folder that is readable by *others* and start to clone SearXNG, +alternatively you can create your own fork and clone from there. + +.. code:: bash + + $ cd ~/Downloads + $ git clone https://github.com/searxng/searxng.git searxng + $ cd searxng + +.. sidebar:: further read + + - :ref:`inspect searxng` + +To install a SearXNG :ref:`reference setup ` +including a :ref:`uWSGI setup ` as described in the +:ref:`installation basic` and in the :ref:`searxng uwsgi` section type: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install all + +.. attention:: + + For the installation procedure, use a *sudoer* login to run the scripts. If + you install from ``root``, take into account that the scripts are creating a + ``searxng`` user. In the installation procedure this new created user do + need read access to the clone of searx, which is not the case if you clone + into a folder below ``/root``! + +.. sidebar:: further read + + - :ref:`update searxng` + +.. _caddy: https://hub.docker.com/_/caddy + +When all services are installed and running fine, you can add SearXNG to your +HTTP server. We do not have any preferences for the HTTP server, you can use +whatever you prefer. + +We use caddy in our :ref:`docker image ` and we have +implemented installation procedures for: + +- :ref:`installation nginx` +- :ref:`installation apache` diff --git a/docs/admin/installation-searxng.rst b/docs/admin/installation-searxng.rst index b1413931..db760bc3 100644 --- a/docs/admin/installation-searxng.rst +++ b/docs/admin/installation-searxng.rst @@ -9,15 +9,16 @@ Step by step installation :local: :backlinks: entry -Step by step installation with virtualenv. For Ubuntu, be sure to have enable -universe repository. + +In this section we show the setup of a SearXNG instance that will be installed +by the :ref:`installation scripts`. .. _install packages: Install packages ================ -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages @@ -30,24 +31,24 @@ Install packages Create user =========== -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create user :end-before: END create user -.. _searx-src: +.. _searxng-src: Install SearXNG & dependencies ============================== -Start a interactive shell from new created user and clone searx: +Start a interactive shell from new created user and clone SearXNG: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START clone searxng :end-before: END clone searxng In the same shell create *virtualenv*: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create virtualenv :end-before: END create virtualenv @@ -55,7 +56,7 @@ To install searx's dependencies, exit the SearXNG *bash* session you opened abov and restart a new. Before install, first check if your *virtualenv* was sourced from the login (*~/.profile*): -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START manage.sh update_packages :end-before: END manage.sh update_packages @@ -77,30 +78,41 @@ Configuration - :ref:`settings use_default_settings` - :origin:`/etc/searxng/settings.yml ` -To create a initial ``/etc/searxng/settings.yml`` you can start with a copy of -the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup +To create a initial ``/etc/searxng/settings.yml`` we recommend to start with a +copy of the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup :ref:`use default settings ` from -:origin:`searx/settings.yml`. +:origin:`searx/settings.yml` and is shown in the tab *"Use default settings"* +below. This setup: -For a *minimal setup*, configure like shown below – replace ``searx@$(uname --n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit -``/etc/searxng/settings.yml`` to your needs. +- enables :ref:`limiter ` to protect against bots +- enables :ref:`image proxy ` for better privacy +- enables :ref:`cache busting ` to save bandwith -.. kernel-include:: $DOCS_BUILD/includes/searx.rst - :start-after: START searxng config - :end-before: END searxng config +Modify the ``/etc/searxng/settings.yml`` to your needs: .. tabs:: .. group-tab:: Use default settings - .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml - :language: yaml + .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml` .. group-tab:: searx/settings.yml - .. literalinclude:: ../../searx/settings.yml - :language: yaml + .. literalinclude:: ../../searx/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`searx/settings.yml` + +For a *minimal setup* you need to set ``server:secret_key``. + +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START searxng config + :end-before: END searxng config Check @@ -110,7 +122,7 @@ To check your SearXNG setup, optional enable debugging and start the *webapp*. SearXNG looks at the exported environment ``$SEARXNG_SETTINGS_PATH`` for a configuration file. -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START check searxng installation :end-before: END check searxng installation diff --git a/docs/admin/installation-switch2ng.rst b/docs/admin/installation-switch2ng.rst index 8863cd71..0e25b960 100644 --- a/docs/admin/installation-switch2ng.rst +++ b/docs/admin/installation-switch2ng.rst @@ -22,7 +22,11 @@ If your searx instance was installed *"Step by step"* or by the *"Installation scripts"*, you need to undo the installation procedure completely. If you have morty & filtron installed, it is recommended to uninstall these services also. In case of scripts, to uninstall use the scripts from the origin you installed -searx from. +searx from or try:: + + $ sudo -H ./utils/filtron.sh remve all + $ sudo -H ./utils/morty.sh remve all + $ sudo -H ./utils/searx.sh remve all If you have removed the old searx installation, clone from SearXNG and and start with your installation procedure (e.g. :ref:`installation scripts`): @@ -53,7 +57,7 @@ Once you have done your installation, you can run a SearXNG *check* procedure, to see if there are some left overs. In this example there exists a *old* ``/etc/searx/settings.yml``:: - $ sudo -H ./utils/searx.sh install check + $ sudo -H ./utils/searxng.sh instance check ============================ SearXNG (check installation) diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index 1021283f..eea52ab3 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -1,7 +1,7 @@ .. _searxng uwsgi: ===== -uwsgi +uWSGI ===== .. sidebar:: further reading @@ -29,37 +29,51 @@ uwsgi Origin uWSGI ============ -How uWSGI is implemented by distributors is different. uWSGI itself -recommend two methods +.. _Tyrant mode: + https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting -`systemd.unit`_ template files as described here `One service per app in systemd`_. +How uWSGI is implemented by distributors varies. The uWSGI project itself +recommends two methods: - There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app - placed at dedicated locations. Take archlinux and a searxng.ini as example:: +1. `systemd.unit`_ template file as described here `One service per app in systemd`_: - unit template --> /usr/lib/systemd/system/uwsgi@.service - uwsgi ini files --> /etc/uwsgi/searxng.ini + There is one `systemd unit template`_ on the system installed and one `uwsgi + ini file`_ per uWSGI-app placed at dedicated locations. Take archlinux and a + ``searxng.ini`` as example:: - The SearXNG app can be maintained as know from common systemd units:: + systemd template unit: /usr/lib/systemd/system/uwsgi@.service + contains: [Service] + ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini - systemctl enable uwsgi@searx - systemctl start uwsgi@searx - systemctl restart uwsgi@searx - systemctl stop uwsgi@searx + SearXNG application: /etc/uwsgi/searxng.ini + links to: /etc/uwsgi/apps-available/searxng.ini -The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps. + The SearXNG app (template ``/etc/uwsgi/%I.ini``) can be maintained as known + from common systemd units: + + .. code:: sh + + $ systemctl enable uwsgi@searxng + $ systemctl start uwsgi@searxng + $ systemctl restart uwsgi@searxng + $ systemctl stop uwsgi@searxng + +2. The `uWSGI Emperor`_ which fits for maintaining a large range of uwsgi + apps and there is a `Tyrant mode`_ to secure multi-user hosting. The Emperor mode is a special uWSGI instance that will monitor specific - events. The Emperor mode (service) is started by a (common, not template) - systemd unit. The Emperor service will scan specific directories for `uwsgi - ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the - timestamp is modified, a corresponding action takes place: a new uWSGI - instance is started, reload or stopped. Take Fedora and a searxng.ini as - example:: + events. The Emperor mode (the service) is started by a (common, not template) + systemd unit. + + The Emperor service will scan specific directories for `uwsgi ini file`_\s + (also know as *vassals*). If a *vassal* is added, removed or the timestamp is + modified, a corresponding action takes place: a new uWSGI instance is started, + reload or stopped. Take Fedora and a ``searxng.ini`` as example:: + + to install & start SearXNG instance create --> /etc/uwsgi.d/searxng.ini + to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini + to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini - to start a new SearXNG instance create --> /etc/uwsgi.d/searxng.ini - to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini - to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini Distributors ============ @@ -70,10 +84,22 @@ modes and their defaults. Another point they might differ is the packaging of plugins (if so, compare :ref:`install packages`) and what the default python interpreter is (python2 vs. python3). -Fedora starts a Emperor by default, while archlinux does not start any uwsgi -service by default. Worth to know; debian (ubuntu) follow a complete different -approach. *debian*: your are familiar with the apache infrastructure? .. they -do similar for the uWSGI infrastructure (with less comfort), the folders are:: +While archlinux does not start a uWSGI service by default, Fedora (RHEL) starts +a Emperor in `Tyrant mode`_ by default (you should have read :ref:`uWSGI Tyrant +mode pitfalls`). Worth to know; debian (ubuntu) follow a complete different +approach, read see :ref:`Debian's uWSGI layout`. + +.. _Debian's uWSGI layout: + +Debian's uWSGI layout +--------------------- + +.. _uwsgi.README.Debian: + https://salsa.debian.org/uwsgi-team/uwsgi/-/raw/debian/latest/debian/uwsgi.README.Debian + +Be aware, Debian's uWSGI layout is quite different from the standard uWSGI +configuration. Your are familiar with :ref:`Debian's Apache layout`? .. they do +similar for the uWSGI infrastructure. The folders are:: /etc/uwsgi/apps-available/ /etc/uwsgi/apps-enabled/ @@ -82,29 +108,52 @@ The `uwsgi ini file`_ is enabled by a symbolic link:: ln -s /etc/uwsgi/apps-available/searxng.ini /etc/uwsgi/apps-enabled/ -From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You -could control specific instance(s) by issuing:: +More details you will find in the uwsgi.README.Debian_ +(``/usr/share/doc/uwsgi/README.Debian.gz``). Some commands you should know on +Debian: - service uwsgi ... +.. code:: none - sudo -H service uwsgi start searx - sudo -H service uwsgi stop searx + Commands recognized by init.d script + ==================================== -My experience is, that this command is a bit buggy. + You can issue to init.d script following commands: + * start | starts daemon + * stop | stops daemon + * reload | sends to daemon SIGHUP signal + * force-reload | sends to daemon SIGTERM signal + * restart | issues 'stop', then 'start' commands + * status | shows status of daemon instance (running/not running) -.. _uwsgi configuration: + 'status' command must be issued with exactly one argument: ''. -Alltogether -=========== + Controlling specific instances of uWSGI + ======================================= -Create the configuration ini-file according to your distribution (see below) and -restart the uwsgi application. + You could control specific instance(s) by issuing: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi ... + + where: + * is one of 'start', 'stop' etc. + * is the name of configuration file (without extension) + + For example, this is how instance for /etc/uwsgi/apps-enabled/hello.xml is + started: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi start hello + + +.. _uWSGI maintenance: + +uWSGI maintenance +================= .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description ubuntu-20.04 :end-before: END searxng uwsgi-description ubuntu-20.04 @@ -112,7 +161,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description arch :end-before: END searxng uwsgi-description arch @@ -120,16 +169,28 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description fedora :end-before: END searxng uwsgi-description fedora +.. _uwsgi setup: + +uWSGI setup +=========== + +Create the configuration ini-file according to your distribution and restart the +uwsgi application. As shown below, the :ref:`installation scripts` installs by +default: + +- a uWSGI setup that listens on a socket and +- enables :ref:`cache busting `. + .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini ubuntu-20.04 :end-before: END searxng uwsgi-appini ubuntu-20.04 @@ -137,7 +198,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini arch :end-before: END searxng uwsgi-appini arch @@ -145,6 +206,63 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini fedora :end-before: END searxng uwsgi-appini fedora + + +.. _uWSGI Tyrant mode pitfalls: + +Pitfalls of the Tyrant mode +=========================== + +The implementation of the process owners and groups in the `Tyrant mode`_ is +somewhat unusual and requires special consideration. In `Tyrant mode`_ mode the +Emperor will run the vassal using the UID/GID of the vassal configuration file +(user and group of the app ``.ini`` file). + +.. _#2099@uWSGI: https://github.com/unbit/uwsgi/issues/2099 +.. _#752@uWSGI: https://github.com/unbit/uwsgi/pull/752 +.. _#2425uWSGI: https://github.com/unbit/uwsgi/issues/2425 + +Without option ``emperor-tyrant-initgroups=true`` in ``/etc/uwsgi.ini`` the +process won't get the additional groups, but this option is not available in +2.0.x branch (see `#2099@uWSGI`_) the feature `#752@uWSGI`_ has been merged (on +Oct. 2014) to the master branch of uWSGI but had never been released; the last +major release is from Dec. 2013, since the there had been only bugfix releases +(see `#2425uWSGI`_). To shorten up: + + **In Tyrant mode, there is no way to get additional groups, and the uWSGI + process misses additional permissions that may be needed.** + +By example, on Fedora (RHEL): If you try to install a redis DB with socket +communication and you want to connect from the SearXNG uWSGI, you will see a +*Permission denied* in the log of your instance:: + + ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + ERROR:searx.plugins.limiter: init limiter DB failed!!! + +Even if your *searxng* user of the uWSGI process is added to additional groups +to give access to the socket from the redis DB:: + + $ groups searxng + searxng : searxng searxng-redis + +To see the effective groups of the uwsgi process, you have to look at the status +of the process, by example:: + + $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + +Here you can see that the additional "Groups" of PID 186 are unset (missing gid +of ``searxng-redis``):: + + $ cat /proc/186/task/186/status + ... + Uid: 993 993 993 993 + Gid: 993 993 993 993 + FDSize: 128 + Groups: + ... diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 91f82e50..f650480c 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -14,99 +14,7 @@ Installation - :ref:`installation scripts` - :ref:`installation basic` -The :ref:`installation basic` is good enough for intranet usage and it is a -excellent illustration of *how a SearXNG instance is build up*. If you place your -instance public to the internet you should really consider to install a -:ref:`filtron reverse proxy ` and for privacy a :ref:`result proxy -` is mandatory. - -Therefore, if you do not have any special preferences, its recommend to use the -:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling -box ` as described below. - -.. _installation scripts: - -Installation scripts -==================== - -.. sidebar:: Update OS first! - - To avoid unwanted side effects, update your OS before installing SearXNG. - -The following will install a setup as shown in :ref:`architecture`. First you -need to get a clone. The clone is only needed for the installation procedure -and some maintenance tasks (alternatively you can create your own fork). - -For the installation procedure, use a *sudoer* login to run the scripts. If you -install from ``root``, take into account that the scripts are creating a -``searx``, a ``filtron`` and a ``morty`` user. In the installation procedure -these new created users do need read access to the clone of searx, which is not -the case if you clone into a folder below ``/root``. - -.. code:: bash - - $ cd ~/Downloads - $ git clone https://github.com/searxng/searxng.git searxng - $ cd searxng - -.. sidebar:: further read - - - :ref:`toolboxing` - - :ref:`update searxng` - - :ref:`inspect searxng` - -**Install** :ref:`SearXNG service ` - -This installs SearXNG as described in :ref:`installation basic`. - -.. code:: bash - - $ sudo -H ./utils/searx.sh install all - -**Install** :ref:`filtron reverse proxy ` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh install all - -**Install** :ref:`result proxy ` - -.. code:: bash - - $ sudo -H ./utils/morty.sh install all - -If all services are running fine, you can add it to your HTTP server: - -**Install** HTTP - -- :ref:`installation apache` -- :ref:`installation nginx` - -**Install** :ref:`external plugins ` - -Use SearXNG's ``shell`` to install external plugins. In the example below we -install the SearXNG plugins from **The Green Web Foundation** `[ref] -`__: - -.. code:: bash - - $ sudo -H ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) searx@ryzen:~$ pip install git+https://github.com/return42/tgwf-searx-plugins - -In the :ref:`settings.yml` activate the ``plugins:`` section and add module -``only_show_green_results`` from tgwf-searx-plugins. - -.. code:: yaml - - plugins: - - only_show_green_results - -.. _git stash: https://git-scm.com/docs/git-stash - -.. tip:: - - About script's installation options have a look at chapter :ref:`toolboxing - setup`. How to brand your instance see chapter :ref:`settings global`. To - *stash* your instance's setup, `git stash`_ your clone's :origin:`.config.sh` - file . +The :ref:`installation basic` is a excellent illustration of *how a SearXNG +instance is build up* (see :ref:`architecture uWSGI`). If you do not have any +special preferences, its recommend to use the :ref:`installation docker` or the +:ref:`installation scripts`. diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst deleted file mode 100644 index b6bd2ea5..00000000 --- a/docs/admin/morty.rst +++ /dev/null @@ -1,40 +0,0 @@ - -.. _searxng morty: - -========================= -How to setup result proxy -========================= - -.. sidebar:: further reading - - - :ref:`morty.sh` - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty - -By default SearXNG can only act as an image proxy for result images, but it is -possible to proxify all the result URLs with an external service, morty_. - -To use this feature, morty has to be installed and activated in SearXNG's -``settings.yml``. Add the following snippet to your ``settings.yml`` and -restart searx: - -.. code:: yaml - - result_proxy: - url : http://127.0.0.1:3000/ - key : !!binary "insert_your_morty_proxy_key_here" - -Note that the example above (``http://127.0.0.1:3000``) is only for single-user -instances without a HTTP proxy. If your morty service is public, the url is the -address of the reverse proxy (e.g ``https://example.org/morty``). - -For more information about *result proxy* have a look at *"SearXNG via filtron -plus morty"* in the :ref:`nginx ` and -:ref:`apache ` sections. - -``url`` - Is the address of the running morty service. - -``key`` - Is an optional argument, see `morty's README`_ for more information. diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 4c541bf5..24bdf4f5 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -1,59 +1,39 @@ -.. _update searxng: +=================== +SearXNG maintenance +=================== -============= -How to update -============= +.. sidebar:: further read -How to update depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``update`` command from the scripts. + - :ref:`toolboxing` + - :ref:`uWSGI maintenance` -**Update** :ref:`SearXNG service ` +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry -.. code:: sh +.. _update searxng: - sudo -H ./utils/searx.sh update searx +How to update +============= -**Update** :ref:`filtron reverse proxy ` +How to update depends on the :ref:`installation` method. If you have used the +:ref:`installation scripts`, use ``update`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/filtron.sh update filtron - -**Update** :ref:`result proxy ` - -.. code:: bash - - $ sudo -H ./utils/morty.sh update morty + sudo -H ./utils/searxng.sh instance update .. _inspect searxng: -====================== How to inspect & debug ====================== -.. sidebar:: further read - - - :ref:`toolboxing` - - :ref:`Makefile` - How to debug depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``inspect`` command from the scripts. - -**Inspect** :ref:`SearXNG service ` - -.. code:: sh - - sudo -H ./utils/searx.sh inspect service - -**Inspect** :ref:`filtron reverse proxy ` +:ref:`installation scripts`, use ``inspect`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/filtron.sh inspect service - -**Inspect** :ref:`result proxy ` - -.. code:: bash - - $ sudo -H ./utils/morty.sh inspect service - + sudo -H ./utils/searxng.sh instance inspect diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searxng.rst similarity index 88% rename from docs/build-templates/searx.rst rename to docs/build-templates/searxng.rst index 0e7877ba..14b38546 100644 --- a/docs/build-templates/searx.rst +++ b/docs/build-templates/searxng.rst @@ -1,4 +1,4 @@ -.. template evaluated by: ./utils/searx.sh doc +.. template evaluated by: ./utils/searxng.sh searxng.doc.rst .. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$ .. START distro-packages @@ -65,7 +65,8 @@ ${fedora_build} $ sudo -H useradd --shell /bin/bash --system \\ --home-dir \"$SERVICE_HOME\" \\ - --comment 'Privacy-respecting metasearch engine' $SERVICE_USER + --comment 'Privacy-respecting metasearch engine' \\ + $SERVICE_USER $ sudo -H mkdir \"$SERVICE_HOME\" $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\" @@ -81,7 +82,8 @@ ${fedora_build} .. code-block:: sh $ sudo -H -u ${SERVICE_USER} -i - (${SERVICE_USER})$ git clone \"$GIT_URL\" \"$SEARXNG_SRC\" + (${SERVICE_USER})$ git clone \"$GIT_URL\" \\ + \"$SEARXNG_SRC\" .. END clone searxng @@ -94,7 +96,8 @@ ${fedora_build} .. code-block:: sh (${SERVICE_USER})$ python3 -m venv \"${SEARXNG_PYENV}\" - (${SERVICE_USER})$ echo \". ${SEARXNG_PYENV}/bin/activate\" >> \"$SERVICE_HOME/.profile\" + (${SERVICE_USER})$ echo \". ${SEARXNG_PYENV}/bin/activate\" \\ + >> \"$SERVICE_HOME/.profile\" .. END create virtualenv @@ -137,21 +140,12 @@ ${fedora_build} $ sudo -H cp \"$SEARXNG_SRC/utils/templates/etc/searxng/settings.yml\" \\ \"${SEARXNG_SETTINGS_PATH}\" - .. group-tab:: searx/settings.yml - - .. code-block:: sh - - $ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARXNG_SRC/searx/settings.yml\" \\ - \"${SEARXNG_SETTINGS_PATH}\" - -.. tabs:: - .. group-tab:: minimal setup .. code-block:: sh - $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARXNG_SETTINGS_PATH\" + $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \\ + \"$SEARXNG_SETTINGS_PATH\" .. END searxng config diff --git a/docs/conf.py b/docs/conf.py index 44d1c2ad..8e0c3ab1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,5 +195,5 @@ html_show_sourcelink = True # LaTeX ---------------------------------------------------------------- latex_documents = [ - (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual") + (master_doc, "searxng-{}.tex".format(VERSION_STRING), html_title, author, "manual") ] diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 43987549..e950ae66 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -66,11 +66,11 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin .. table:: Common options in the engine setup (``settings.yml``) :width: 100% - ======================= =========== =============================================== + ======================= =========== ================================================== argument type information - ======================= =========== =============================================== + ======================= =========== ================================================== name string name of search-engine - engine string name of searx-engine (filename without ``.py``) + engine string name of searxng-engine (file name without ``.py``) enable_http bool enable HTTP (by default only HTTPS is enabled). shortcut string shortcut of search-engine timeout string specific timeout for search-engine @@ -78,7 +78,7 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin proxies dict set proxies for a specific engine (e.g. ``proxies : {http: socks5://proxy:port, https: socks5://proxy:port}``) - ======================= =========== =============================================== + ======================= =========== ================================================== .. _engine overrides: diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index 85b26221..6195a575 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -45,9 +45,7 @@ be set on a *production* system. The scripts from :ref:`searx_utils` can divide in those to install and maintain software: -- :ref:`searx.sh` -- :ref:`filtron.sh` -- :ref:`morty.sh` +- :ref:`searxng.sh` and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or even development tasks over a stack of isolated containers / what we call the: @@ -73,7 +71,7 @@ once: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ snap install lxd $ lxd init --auto @@ -85,7 +83,7 @@ fork: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ cd ~/Downloads $ git clone https://github.com/searxng/searxng.git searxng @@ -94,19 +92,19 @@ fork: The :ref:`lxc-searxng.env` consists of several images, see ``export LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post we exercise on a archlinux_ image. The container of this image is named -``searx-archlinux``. Lets build the container, but be sure that this container +``searxng-archlinux``. Lets build the container, but be sure that this container does not already exists, so first lets remove possible old one: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh remove searx-archlinux - $ sudo -H ./utils/lxc.sh build searx-archlinux + $ sudo -H ./utils/lxc.sh remove searxng-archlinux + $ sudo -H ./utils/lxc.sh build searxng-archlinux -.. sidebar:: The ``searx-archlinux`` container +.. sidebar:: The ``searxng-archlinux`` container is the base of all our exercises here. @@ -117,9 +115,9 @@ In this container we install all services :ref:`including searx, morty & filtron .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To proxy HTTP from filtron and morty in the container to the outside of the container, install nginx into the container. Once for the bot blocker filtron: @@ -128,9 +126,9 @@ container, install nginx into the container. Once for the bot blocker filtron: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/filtron.sh nginx install ... INFO: got 429 from http://10.174.184.156/searx @@ -141,9 +139,9 @@ and once for the content sanitizer (content proxy morty): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/morty.sh nginx install ... INFO: got 200 from http://10.174.184.156/morty/ @@ -154,7 +152,7 @@ and once for the content sanitizer (content proxy morty): blocker (filtron) and WEB content sanitizer (content proxy morty), both are needed for a *privacy protecting* search engine. -On your system, the IP of your ``searx-archlinux`` container differs from +On your system, the IP of your ``searxng-archlinux`` container differs from http://10.174.184.156/searx, just open the URL reported in your installation protocol in your WEB browser from the desktop to test the instance from outside of the container. @@ -169,27 +167,27 @@ In containers, work as usual Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers open the root-bash in the container using ``./utils/lxc.sh cmd -searx-archlinux``: +searxng-archlinux``: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash - INFO: [searx-archlinux] bash - [root@searx-archlinux searx]# pwd + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux searx]# pwd /share/searxng -The prompt ``[root@searx-archlinux ...]`` signals, that you are the root user in -the searx-container. To debug the running SearXNG instance use: +The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user in +the searxng-container. To debug the running SearXNG instance use: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash $ ./utils/searx.sh inspect service ... @@ -202,56 +200,42 @@ above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug option"* in SearXNG's settings file and restarts the SearXNG uwsgi application. To debug services from filtron and morty analogous use: -.. tabs:: - - .. group-tab:: root@searx-archlinux - - .. code:: sh - - $ ./utils/filtron.sh inspect service - $ ./utils/morty.sh inspect service - -Another point we have to notice is that each service (:ref:`SearXNG `, -:ref:`filtron ` and :ref:`morty `) runs under dedicated -system user account with the same name (compare :ref:`create searxng user`). To -get a shell from theses accounts, simply call one of the scripts: +Another point we have to notice is that the service (:ref:`SearXNG ` +runs under dedicated system user account with the same name (compare +:ref:`create searxng user`). To get a shell from theses accounts, simply call: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash - $ ./utils/searx.sh shell - $ ./utils/filtron.sh shell - $ ./utils/morty.sh shell + $ ./utils/searxng.sh instance cmd bash -To get in touch, open a shell from the service user (searx@searx-archlinux): +To get in touch, open a shell from the service user (searxng@searxng-archlinux): .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) [searx@searx-archlinux ~]$ ... + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash + INFO: [searxng-archlinux] ./utils/searxng.sh instance cmd bash + [searxng@searxng-archlinux ~]$ -The prompt ``[searx@searx-archlinux]`` signals that you are logged in as system -user ``searx`` in the ``searx-archlinux`` container and the python *virtualenv* -``(searx-pyenv)`` environment is activated. +The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system +user ``searx`` in the ``searxng-archlinux`` container and the python *virtualenv* +``(searxng-pyenv)`` environment is activated. .. tabs:: - .. group-tab:: searx@searx-archlinux - - .. code:: sh + .. group-tab:: searxng@searxng-archlinux - (searx-pyenv) [searx@searx-archlinux ~]$ pwd - /usr/local/searx + .. code:: bash + (searxng-pyenv) [searxng@searxng-archlinux ~]$ pwd + /usr/local/searxng Wrap production into developer suite @@ -262,23 +246,22 @@ from a LXC container (which is quite ready for production) into a developer suite. For this, we have to keep an eye on the :ref:`installation basic`: - SearXNG setup in: ``/etc/searxng/settings.yml`` -- SearXNG user's home: ``/usr/local/searx`` -- virtualenv in: ``/usr/local/searx/searx-pyenv`` -- SearXNG software in: ``/usr/local/searx/searx-src`` +- SearXNG user's home: ``/usr/local/searxng`` +- virtualenv in: ``/usr/local/searxng/searxng-pyenv`` +- SearXNG software in: ``/usr/local/searxng/searxng-src`` -With the use of the :ref:`searx.sh` the SearXNG service was installed as +With the use of the :ref:`searxng.sh` the SearXNG service was installed as :ref:`uWSGI application `. To maintain this service, we can use -``systemctl`` (compare :ref:`service architectures on distributions `). +``systemctl`` (compare :ref:`uWSGI maintenance`). .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - systemctl stop uwsgi@searx + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + systemctl stop uwsgi@searxng With the command above, we stopped the SearXNG uWSGI-App in the archlinux container. @@ -291,29 +274,29 @@ least you should attend the settings of ``uid``, ``chdir``, ``env`` and env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml http = 127.0.0.1:8888 - chdir = /usr/local/searx/searx-src/searx - virtualenv = /usr/local/searx/searx-pyenv - pythonpath = /usr/local/searx/searx-src + chdir = /usr/local/searxng/searxng-src/searx + virtualenv = /usr/local/searxng/searxng-pyenv + pythonpath = /usr/local/searxng/searxng-src If you have read the :ref:`"Good to know section" ` you remember, that each container shares the root folder of the repository and the command ``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the SearXNG installation into a developer one, we simple have to create a smylink to the **transparent** reposetory from the desktop. Now lets replace the -repository at ``searx-src`` in the container with the working tree from outside +repository at ``searxng-src`` in the container with the working tree from outside of the container: .. tabs:: .. group-tab:: container becomes a developer suite - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ln -s /share/searx/ /usr/local/searx/searx-src + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ln -s /share/searx/ /usr/local/searxng/searxng-src Now we can develop as usual in the working tree of our desktop system. Every time the software was changed, you have to restart the SearXNG service (in the @@ -323,9 +306,9 @@ conatiner): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ systemctl restart uwsgi@searx @@ -338,30 +321,30 @@ daily usage: To *inspect* the SearXNG instance (already described above): - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/searx.sh inspect service Run :ref:`makefile`, e.g. to test inside the container: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make test To install all prerequisites needed for a :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh install buildhost + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ./utils/searxng.sh install buildhost To build the docs on a buildhost :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make docs.html .. _lxcdev summary: @@ -371,18 +354,18 @@ Summary We build up a fully functional SearXNG suite in a archlinux container: -.. code:: sh +.. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To access HTTP from the desktop we installed nginx for the services inside the conatiner: .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash $ ./utils/filtron.sh nginx install $ ./utils/morty.sh nginx install @@ -393,12 +376,12 @@ the container : .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash - $ mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old - $ ln -s /share/searx/ /usr/local/searx/searx-src + $ mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old + $ ln -s /share/searx/ /usr/local/searxng/searxng-src $ systemctl restart uwsgi@searx To get information about the searxNG suite in the archlinux container we can @@ -408,13 +391,13 @@ use: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh show suite searx-archlinux + $ sudo -H ./utils/lxc.sh show suite searxng-archlinux ... - [searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx - [searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ - [searx-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ - [searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] + [searxng-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx + [searxng-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ + [searxng-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ + [searxng-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] ... diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index ceb76bf1..68c708a8 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -40,7 +40,7 @@ We do no longer need to build up the virtualenv manually. Jump into your git working tree and release a ``make install`` to get a virtualenv with a *developer install* of SearXNG (:origin:`setup.py`). :: - $ cd ~/searx-clone + $ cd ~/searxng-clone $ make install PYENV [virtualenv] installing ./requirements*.txt into local/py3 ... @@ -288,27 +288,3 @@ To filter out HTTP redirects (3xx_):: https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0 https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None -- - - -``make pybuild`` -================ - -.. _PyPi: https://pypi.org/ -.. _twine: https://twine.readthedocs.io/en/latest/ - -Build Python packages in ``./dist/py``:: - - $ make pybuild - ... - BUILD pybuild - running sdist - running egg_info - ... - running bdist_wheel - - $ ls ./dist - searx-0.18.0-py3-none-any.whl searx-0.18.0.tar.gz - -To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use -``pypi.upload.test``). Since you are not the owner of :pypi:`searx` you will -never need to upload. diff --git a/docs/dev/offline_engines.rst b/docs/dev/offline_engines.rst index 9320c626..bfb2664f 100644 --- a/docs/dev/offline_engines.rst +++ b/docs/dev/offline_engines.rst @@ -55,10 +55,10 @@ admins can install packages in advance. If there is a need to install additional packages in *Python's Virtual Environment* of your SearXNG instance you need to switch into the environment -(:ref:`searx-src`) first, for this you can use :ref:`searx.sh`:: +(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... Private engines (Security) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 44401e34..36a44323 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -33,17 +33,26 @@ Example plugin External plugins ================ -External plugins are standard python modules implementing all the requirements of the standard plugins. -Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section. -Example external plugin can be found `here `_. +SearXNG supports *external plugins* / there is no need to install one, SearXNG +runs out of the box. But to demonstrate; in the example below we install the +SearXNG plugins from *The Green Web Foundation* `[ref] +`__: -Register your plugin -==================== +.. code:: bash + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install git+https://github.com/return42/tgwf-searx-plugins + +In the :ref:`settings.yml` activate the ``plugins:`` section and add module +``only_show_green_results`` from ``tgwf-searx-plugins``. + +.. code:: yaml + + plugins: + ... + - only_show_green_results + ... -To enable your plugin register your plugin in -searx > plugin > __init__.py. -And at the bottom of the file add your plugin like. -``plugins.register(name_of_python_file)`` Plugin entry points =================== diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index db52a2d8..921384aa 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -10,7 +10,7 @@ Development Quickstart SearXNG loves developers, just clone and start hacking. All the rest is done for you simply by using :ref:`make `. -.. code:: sh +.. code:: bash git clone https://github.com/searxng/searxng.git searxng @@ -27,21 +27,21 @@ to our ":ref:`how to contribute`" guideline. If you implement themes, you will need to setup a :ref:`make node.env` once: -.. code:: sh +.. code:: bash make node.env Before you call *make run* (2.), you need to compile the modified styles and JavaScript: -.. code:: sh +.. code:: bash make themes.all Alternatively you can also compile selective the theme you have modified, e.g. the *simple* theme. -.. code:: sh +.. code:: bash make themes.simple @@ -52,7 +52,7 @@ e.g. the *simple* theme. If you finished your *tests* you can start to commit your changes. To separate the modified source code from the build products first run: -.. code:: sh +.. code:: bash make static.build.restore @@ -60,13 +60,13 @@ This will restore the old build products and only your changes of the code remain in the working tree which can now be added & commited. When all sources are commited, you can commit the build products simply by: -.. code:: sh +.. code:: bash make static.build.commit Commiting the build products should be the last step, just before you send us your PR. There is also a make target to rewind this last build commit: -.. code:: sh +.. code:: bash make static.build.drop diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst deleted file mode 100644 index 1f730239..00000000 --- a/docs/utils/filtron.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _filtron.sh: - -==================== -``utils/filtron.sh`` -==================== - -.. sidebar:: further reading - - - :ref:`searxng filtron` - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx ` & :ref:`apache - `) - -.. _Go: https://golang.org/ -.. _filtron: https://github.com/searxng/filtron -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -To simplify installation and maintenance of a filtron instance you can use the -script :origin:`utils/filtron.sh`. In most cases you will install filtron_ -simply by running the command: - -.. code:: bash - - sudo -H ./utils/filtron.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ -into this user account: - -#. Create a separated user account (``filtron``). -#. Download and install Go_ binary in user's $HOME (``~filtron``). -#. Install filtron with the package management from Go_ (``go get -v -u - github.com/searxng/filtron``) -#. Setup a proper rule configuration :origin:`[ref] - ` (``/etc/filtron/rules.json``). -#. Setup a systemd service unit :origin:`[ref] - ` - (``/lib/systemd/system/filtron.service``). - - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install go - :end-before: END install go - - -Install filtron -=============== - -Install :origin:`rules.json ` at -``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and -install filtron software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install filtron - :end-before: END install filtron - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _filtron.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/filtron.sh --help diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 62e02655..2da26ed1 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -1,36 +1,24 @@ .. _searx_utils: .. _toolboxing: -=================== -Admin's tooling box -=================== +================== +DevOps tooling box +================== -In the folder :origin:`utils/` we maintain some tools useful for administrators. +In the folder :origin:`utils/` we maintain some tools useful for administrators +and developers. .. toctree:: :maxdepth: 2 :caption: Contents - searx.sh - filtron.sh - morty.sh + searxng.sh lxc.sh -.. _toolboxing common: +Common command environments +=========================== -Common commands & environment -============================= - -Scripts to maintain services often dispose of common commands and environments. - -``shell`` : command - Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for - troubleshooting. - -``inspect service`` : command - Shows status and log of the service, most often you have a option to enable - more verbose debug logs. Very helpful for debugging, but be careful not to - enable debugging in a production environment! +The scripts in our tooling box often dispose of common environments: ``FORCE_TIMEOUT`` : environment Sets timeout for interactive prompts. If you want to run a script in batch @@ -40,14 +28,3 @@ Scripts to maintain services often dispose of common commands and environments. sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx - -.. _toolboxing setup: - -Tooling box setup -================= - -The main setup is done in the :origin:`.config.sh` (read also :ref:`settings -global`). - -.. literalinclude:: ../../.config.sh - :language: bash diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index 291c4f14..4308a12c 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -49,7 +49,7 @@ help>`. If you do not want to build all containers, **you can build just one**:: - $ sudo -H ./utils/lxc.sh build searx-archlinux + $ sudo -H ./utils/lxc.sh build searxng-archlinux *Good to know ...* @@ -62,9 +62,9 @@ of:: In the containers, you can run what ever you want, e.g. to start a bash use:: - $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash - INFO: [searx-archlinux] bash - root@searx-archlinux:/share/searxng# + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux SearXNG]# If there comes the time you want to **get rid off all** the containers and **clean up local images** just type:: @@ -150,8 +150,8 @@ Running commands :ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the Makefile target ``test`` in the archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searxng.sh install buildhost - sudo -H ./utils/lxc.sh cmd searx-archlinux make test + sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost + sudo -H ./utils/lxc.sh cmd searxng-archlinux make test Setup SearXNG buildhost @@ -166,7 +166,7 @@ of coffee).:: To build (live) documentation inside a archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live + sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live ... [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080 @@ -174,7 +174,7 @@ To get IP of the container and the port number *live docs* is listening:: $ sudo ./utils/lxc.sh show suite | grep docs.live ... - [searx-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ + [searxng-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ .. _lxc.sh help: diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst deleted file mode 100644 index 99103b1c..00000000 --- a/docs/utils/morty.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty -.. _Go: https://golang.org/ - -.. _morty.sh: - -================== -``utils/morty.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx ` & :ref:`apache - `) - - :ref:`searxng morty` - -To simplify installation and maintenance of a morty_ instance you can use the -script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by -running the command: - -.. code:: bash - - sudo -H ./utils/morty.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ -into this user account: - -#. Create a separated user account (``morty``). -#. Download and install Go_ binary in user's $HOME (``~morty``). -#. Install morty_ with the package management from Go_ (``go get -v -u - github.com/asciimoo/morty``) -#. Setup a systemd service unit :origin:`[ref] - ` - (``/lib/systemd/system/morty.service``). - -.. hint:: - - To add morty to your SearXNG instance read chapter :ref:`searxng morty`. - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install go - :end-before: END install go - - -Install morty -============= - -Install morty software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install morty - :end-before: END install morty - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _morty.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/morty.sh --help - diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst deleted file mode 100644 index 6d412d23..00000000 --- a/docs/utils/searx.sh.rst +++ /dev/null @@ -1,39 +0,0 @@ - -.. _searx.sh: - -================== -``utils/searx.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` - - :ref:`installation nginx` - - :ref:`installation apache` - -To simplify installation and maintenance of a SearXNG instance you can use the -script :origin:`utils/searx.sh`. - -Install -======= - -In most cases you will install SearXNG simply by running the command: - -.. code:: bash - - sudo -H ./utils/searx.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs SearXNG -into this user account. The installation is described in chapter -:ref:`installation basic`. - -.. _intranet reverse proxy: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/searx.sh --help diff --git a/docs/utils/searxng.sh.rst b/docs/utils/searxng.sh.rst new file mode 100644 index 00000000..0558f22a --- /dev/null +++ b/docs/utils/searxng.sh.rst @@ -0,0 +1,36 @@ + +.. _searxng.sh: + +==================== +``utils/searxng.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` + - :ref:`installation nginx` + - :ref:`installation apache` + +To simplify installation and maintenance of a SearXNG instance you can use the +script :origin:`utils/searxng.sh`. + +Install +======= + +In most cases you will install SearXNG simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The installation is described in chapter :ref:`installation basic`. + +.. _searxng.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory: + +.. program-output:: ../utils/searxng.sh --help diff --git a/manage b/manage index e9e5e7ca..5805481a 100755 --- a/manage +++ b/manage @@ -416,7 +416,7 @@ docs.prebuild() { set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" - ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst" + ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst" ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst" pyenv.cmd searxng_extra/docs_prebuild diff --git a/utils/searxng.sh b/utils/searxng.sh index 11dd0d67..d9ecdcca 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -889,6 +889,129 @@ _searxng.instance.inspect() { fi } +searxng.doc.rst() { + local debian="${SEARXNG_PACKAGES_debian}" + local arch="${SEARXNG_PACKAGES_arch}" + local fedora="${SEARXNG_PACKAGES_fedora}" + local debian_build="${SEARXNG_BUILD_PACKAGES_debian}" + local arch_build="${SEARXNG_BUILD_PACKAGES_arch}" + local fedora_build="${SEARXNG_BUILD_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + uwsgi_variant=':socket' + else + uwsgi_variant=':socket' + fi + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searxng.rst")\"" + + # I use ubuntu-20.04 here to demonstrate that versions are also suported, + # normaly debian-* and ubuntu-* are most the same. + + for DIST_NAME in ubuntu-20.04 arch fedora; do + ( + DIST_ID=${DIST_NAME%-*} + DIST_VERS=${DIST_NAME#*-} + [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + uWSGI_distro_setup + + echo -e "\n.. START searxng uwsgi-description $DIST_NAME" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) cat < /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + + create ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} + enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ + start: sudo -H service uwsgi start ${SEARXNG_UWSGI_APP%.*} + restart: sudo -H service uwsgi restart ${SEARXNG_UWSGI_APP%.*} + stop: sudo -H service uwsgi stop ${SEARXNG_UWSGI_APP%.*} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + arch-*) cat < /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + enable: sudo -H systemctl enable uwsgi@${SEARXNG_UWSGI_APP%.*} + start: sudo -H systemctl start uwsgi@${SEARXNG_UWSGI_APP%.*} + restart: sudo -H systemctl restart uwsgi@${SEARXNG_UWSGI_APP%.*} + stop: sudo -H systemctl stop uwsgi@${SEARXNG_UWSGI_APP%.*} + disable: sudo -H systemctl disable uwsgi@${SEARXNG_UWSGI_APP%.*} + +EOF + ;; + fedora-*|centos-7) cat < /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + esac + echo -e ".. END searxng uwsgi-description $DIST_NAME" + + local _show_cursor="" # prevent from prefix_stdout's trailing show-cursor + + echo -e "\n.. START searxng uwsgi-appini $DIST_NAME" + echo ".. code:: bash" + echo + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}${uwsgi_variant}")\"" | prefix_stdout " " + echo -e "\n.. END searxng uwsgi-appini $DIST_NAME" + + echo -e "\n.. START nginx socket" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END nginx socket" + + echo -e "\n.. START nginx http" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END nginx http" + + echo -e "\n.. START apache socket" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END apache socket" + + echo -e "\n.. START apache http" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END apache http" + ) + done + +} + # ---------------------------------------------------------------------------- main "$@" # ----------------------------------------------------------------------------