forked from Archives/searxng
[mod] replace /help by /info pages and include pages in project docs
This patch implements a bolierplate to share content from info-pages of the
SearXNG instance (URL /info) with the project documentation (path /docs/user).
The info pages are using Markdown (CommonMark), to include them in the project
documentation (reST) the myst-parser [1] is used in the Sphinx-doc build chain.
If base_url is known (defined in settings.yml) links to the instance are also
inserted into the project documentation::
searxng_extra/docs_prebuild
[1] https://www.sphinx-doc.org/en/master/usage/markdown.html
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
dependabot/pip/master/sphinx-6.1.3
Markus Heiser
2 years ago
parent
bb71ebc394
commit
b1912607ae
@ -0,0 +1,8 @@
|
|||||||
|
.. _searx.infopage:
|
||||||
|
|
||||||
|
================
|
||||||
|
Online ``/info``
|
||||||
|
================
|
||||||
|
|
||||||
|
.. automodule:: searx.infopage
|
||||||
|
:members:
|
||||||
@ -0,0 +1 @@
|
|||||||
|
*.md
|
||||||
@ -1,39 +0,0 @@
|
|||||||
|
|
||||||
.. _search-syntax:
|
|
||||||
|
|
||||||
=============
|
|
||||||
Search syntax
|
|
||||||
=============
|
|
||||||
|
|
||||||
SearXNG allows you to modify the default categories, engines and search language
|
|
||||||
via the search query.
|
|
||||||
|
|
||||||
Prefix ``!``
|
|
||||||
to set Category/engine
|
|
||||||
|
|
||||||
Prefix: ``:``
|
|
||||||
to set language
|
|
||||||
|
|
||||||
Abbrevations of the engines and languages are also accepted. Engine/category
|
|
||||||
modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
|
|
||||||
<?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
|
|
||||||
**and** wikipedia for ``qwer``).
|
|
||||||
|
|
||||||
See the :search:`/preferences page <preferences>` for the list of engines,
|
|
||||||
categories and languages.
|
|
||||||
|
|
||||||
Examples
|
|
||||||
========
|
|
||||||
|
|
||||||
Search in wikipedia for ``qwer``:
|
|
||||||
|
|
||||||
- :search:`!wp qwer <?q=%21wp%20qwer>` or
|
|
||||||
- :search:`!wikipedia qwer :search:<?q=%21wikipedia%20qwer>`
|
|
||||||
|
|
||||||
Image search:
|
|
||||||
|
|
||||||
- :search:`!images Cthulhu <?q=%21images%20Cthulhu>`
|
|
||||||
|
|
||||||
Custom language in wikipedia:
|
|
||||||
|
|
||||||
- :search:`:hu !wp hackerspace <?q=%3Ahu%20%21wp%20hackerspace>`
|
|
||||||
@ -0,0 +1,35 @@
|
|||||||
|
# Search syntax
|
||||||
|
|
||||||
|
SearXNG allows you to modify the default categories, engines and search language
|
||||||
|
via the search query.
|
||||||
|
|
||||||
|
Prefix `!` to set category and engine names.
|
||||||
|
|
||||||
|
Prefix: `:` to set the language.
|
||||||
|
|
||||||
|
Abbrevations of the engines and languages are also accepted. Engine/category
|
||||||
|
modifiers are chainable and inclusive. E.g. with {{search('!map !ddg !wp paris')}}
|
||||||
|
search in map category **and** duckduckgo **and** wikipedia for
|
||||||
|
`paris`.
|
||||||
|
|
||||||
|
See the {{link('preferences', 'preferences')}} for the list of engines,
|
||||||
|
categories and languages.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
Search in wikipedia for `paris`:
|
||||||
|
|
||||||
|
* {{search('!wp paris')}}
|
||||||
|
* {{search('!wikipedia paris')}}
|
||||||
|
|
||||||
|
Search in category `map` for `paris`:
|
||||||
|
|
||||||
|
* {{search('!map paris')}}
|
||||||
|
|
||||||
|
Image search:
|
||||||
|
|
||||||
|
* {{search('!images Wau Holland')}}
|
||||||
|
|
||||||
|
Custom language in wikipedia:
|
||||||
|
|
||||||
|
* {{search(':fr !wp Wau Holland')}}
|
||||||
@ -0,0 +1,191 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
# lint: pylint
|
||||||
|
# pyright: basic
|
||||||
|
"""Render SearXNG instance documentation.
|
||||||
|
|
||||||
|
Usage in a Flask app route:
|
||||||
|
|
||||||
|
.. code:: python
|
||||||
|
|
||||||
|
from searx import infopage
|
||||||
|
|
||||||
|
_INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage)
|
||||||
|
|
||||||
|
@app.route('/info/<pagename>', methods=['GET'])
|
||||||
|
def info(pagename):
|
||||||
|
|
||||||
|
locale = request.preferences.get_value('locale')
|
||||||
|
page = _INFO_PAGES.get_page(pagename, locale)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
__all__ = ['InfoPage', 'MistletoePage', 'InfoPageSet']
|
||||||
|
|
||||||
|
import os.path
|
||||||
|
import logging
|
||||||
|
from functools import cached_property
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import urllib.parse
|
||||||
|
import jinja2
|
||||||
|
from flask.helpers import url_for
|
||||||
|
import mistletoe
|
||||||
|
|
||||||
|
from .. import get_setting
|
||||||
|
from ..version import GIT_URL
|
||||||
|
|
||||||
|
logger = logging.getLogger('doc')
|
||||||
|
|
||||||
|
|
||||||
|
class InfoPage:
|
||||||
|
"""A page of the :py:obj:`online documentation <InfoPageSet>`."""
|
||||||
|
|
||||||
|
def __init__(self, fname, base_url=None):
|
||||||
|
self.fname = fname
|
||||||
|
self.base_url = base_url
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def raw_content(self):
|
||||||
|
"""Raw content of the page (without any jinja rendering)"""
|
||||||
|
with open(self.fname, 'r', encoding='utf-8') as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def content(self):
|
||||||
|
"""Content of the page (rendered in a Jinja conntext)"""
|
||||||
|
ctx = self.get_ctx()
|
||||||
|
template = jinja2.Environment().from_string(self.raw_content)
|
||||||
|
return template.render(**ctx)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def title(self):
|
||||||
|
"""Title of the content (without any markup)"""
|
||||||
|
t = ""
|
||||||
|
for l in self.raw_content.split('\n'):
|
||||||
|
if l.startswith('# '):
|
||||||
|
t = l.strip('# ')
|
||||||
|
return t
|
||||||
|
|
||||||
|
def get_ctx(self): # pylint: disable=no-self-use
|
||||||
|
"""Jinja context to render :py:obj:`InfoPage.content`"""
|
||||||
|
|
||||||
|
def _md_link(name, url):
|
||||||
|
url = url_for(url)
|
||||||
|
if self.base_url:
|
||||||
|
url = self.base_url + url
|
||||||
|
return "[%s](%s)" % (name, url)
|
||||||
|
|
||||||
|
def _md_search(query):
|
||||||
|
url = '%s?q=%s' % (url_for('search'), urllib.parse.quote(query))
|
||||||
|
if self.base_url:
|
||||||
|
url = self.base_url + url
|
||||||
|
return '[%s](%s)' % (query, url)
|
||||||
|
|
||||||
|
ctx = {}
|
||||||
|
ctx['GIT_URL'] = GIT_URL
|
||||||
|
ctx['get_setting'] = get_setting
|
||||||
|
ctx['link'] = _md_link
|
||||||
|
ctx['search'] = _md_search
|
||||||
|
|
||||||
|
return ctx
|
||||||
|
|
||||||
|
def render(self):
|
||||||
|
"""Render / return content"""
|
||||||
|
return self.content
|
||||||
|
|
||||||
|
|
||||||
|
class MistletoePage(InfoPage):
|
||||||
|
"""A HTML page of the :py:obj:`online documentation <InfoPageSet>`."""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def html(self):
|
||||||
|
"""HTML representation of this page"""
|
||||||
|
return self.render()
|
||||||
|
|
||||||
|
def render(self):
|
||||||
|
"""Render Markdown (CommonMark_) to HTML by using mistletoe_.
|
||||||
|
|
||||||
|
.. _CommonMark: https://commonmark.org/
|
||||||
|
.. _mistletoe: https://github.com/miyuchina/mistletoe
|
||||||
|
|
||||||
|
"""
|
||||||
|
return mistletoe.markdown(self.content)
|
||||||
|
|
||||||
|
|
||||||
|
_INFO_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'info'))
|
||||||
|
|
||||||
|
|
||||||
|
class InfoPageSet: # pylint: disable=too-few-public-methods
|
||||||
|
"""Cached rendering of the online documentation a SearXNG instance has.
|
||||||
|
|
||||||
|
:param page_class: render online documentation by :py:obj:`InfoPage` parser.
|
||||||
|
:type page_class: :py:obj:`InfoPage`
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, page_class: typing.Type[InfoPage], base_url=None):
|
||||||
|
self.page_class = page_class
|
||||||
|
self.base_url = base_url
|
||||||
|
self.CACHE: typing.Dict[tuple, InfoPage] = {}
|
||||||
|
|
||||||
|
# future: could be set from settings.xml
|
||||||
|
|
||||||
|
self.folder: str = _INFO_FOLDER
|
||||||
|
"""location of the Markdwon files"""
|
||||||
|
|
||||||
|
self.i18n_origin: str = 'en'
|
||||||
|
"""default language"""
|
||||||
|
|
||||||
|
self.l10n: typing.List = [
|
||||||
|
'en',
|
||||||
|
]
|
||||||
|
"""list of supported languages (aka locales)"""
|
||||||
|
|
||||||
|
self.toc: typing.List = [
|
||||||
|
'search-syntax',
|
||||||
|
'about',
|
||||||
|
]
|
||||||
|
"""list of articles in the online documentation"""
|
||||||
|
|
||||||
|
def get_page(self, pagename: str, locale: typing.Optional[str] = None):
|
||||||
|
"""Return ``pagename`` instance of :py:obj:`InfoPage`
|
||||||
|
|
||||||
|
:param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc`
|
||||||
|
:type pagename: str
|
||||||
|
|
||||||
|
:param locale: language of the page, e.g. ``en``, ``zh_Hans_CN``
|
||||||
|
(default: :py:obj:`InfoPageSet.i18n_origin`)
|
||||||
|
:type locale: str
|
||||||
|
|
||||||
|
"""
|
||||||
|
if pagename not in self.toc:
|
||||||
|
return None
|
||||||
|
if locale is not None and locale not in self.l10n:
|
||||||
|
return None
|
||||||
|
|
||||||
|
locale = locale or self.i18n_origin
|
||||||
|
cache_key = (pagename, locale)
|
||||||
|
page = self.CACHE.get(cache_key)
|
||||||
|
|
||||||
|
if page is not None:
|
||||||
|
return page
|
||||||
|
|
||||||
|
# not yet instantiated
|
||||||
|
|
||||||
|
fname = os.path.join(self.folder, locale, pagename) + '.md'
|
||||||
|
if not os.path.exists(fname):
|
||||||
|
logger.error('file %s does not exists', fname)
|
||||||
|
return None
|
||||||
|
|
||||||
|
page = self.page_class(fname, self.base_url)
|
||||||
|
self.CACHE[cache_key] = page
|
||||||
|
return page
|
||||||
|
|
||||||
|
def all_pages(self, locale: typing.Optional[str] = None):
|
||||||
|
"""Iterate over all pages"""
|
||||||
|
locale = locale or self.i18n_origin
|
||||||
|
for pagename in self.toc:
|
||||||
|
page = self.get_page(pagename, locale)
|
||||||
|
yield pagename, page
|
||||||
@ -1,12 +0,0 @@
|
|||||||
{% extends "oscar/base.html" %}
|
|
||||||
{% block title %}{{ page.title }} - {% endblock %}
|
|
||||||
{% block content %}
|
|
||||||
<ul class="nav nav-tabs">
|
|
||||||
{% for name, page in all_pages %}
|
|
||||||
<li {% if name == page_filename %}class="active"{% endif %}>
|
|
||||||
<a href="{{name}}">{{page.title}}</a>
|
|
||||||
</li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
{{ page.content | safe }}
|
|
||||||
{% endblock %}
|
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
{% extends "oscar/base.html" %}
|
||||||
|
{% block title %}{{ active_page.title }} - {% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<ul class="nav nav-tabs">
|
||||||
|
{% for pagename, page in all_pages('en') %}
|
||||||
|
<li {% if pagename == active_pagename %}class="active"{% endif %}>
|
||||||
|
<a href="{{pagename}}">{{page.title}}</a>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
{{ active_page.html | safe }}
|
||||||
|
{% endblock %}
|
||||||
@ -1,12 +0,0 @@
|
|||||||
{% extends 'simple/page_with_header.html' %}
|
|
||||||
{% block title %}{{ page.title }} - {% endblock %}
|
|
||||||
{% block content %}
|
|
||||||
<ul class="tabs">
|
|
||||||
{% for name, page in all_pages %}
|
|
||||||
<li>
|
|
||||||
<a href="{{name}}" {% if name == page_filename %}class="active"{% endif %}>{{page.title}}</a>
|
|
||||||
</li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
{{ page.content | safe }}
|
|
||||||
{% endblock %}
|
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
{% extends 'simple/page_with_header.html' %}
|
||||||
|
{% block title %}{{ active_page.title }} - {% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<ul class="tabs">
|
||||||
|
{% for pagename, page in all_pages('en') %}
|
||||||
|
<li>
|
||||||
|
<a href="{{pagename}}" {% if pagename == active_pagename %}class="active"{% endif %}>{{page.title}}</a>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
<div class="info-page {{pagename}}">
|
||||||
|
{{- active_page.html | safe -}}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
@ -1,61 +0,0 @@
|
|||||||
# pyright: basic
|
|
||||||
from typing import Dict, NamedTuple
|
|
||||||
import pkg_resources
|
|
||||||
|
|
||||||
import flask
|
|
||||||
from flask.helpers import url_for
|
|
||||||
import mistletoe
|
|
||||||
|
|
||||||
from . import get_setting
|
|
||||||
from .version import GIT_URL
|
|
||||||
|
|
||||||
|
|
||||||
class HelpPage(NamedTuple):
|
|
||||||
title: str
|
|
||||||
content: str
|
|
||||||
|
|
||||||
|
|
||||||
# Whenever a new .md file is added to help/ it needs to be added here
|
|
||||||
_TOC = ('about',)
|
|
||||||
|
|
||||||
PAGES: Dict[str, HelpPage] = {}
|
|
||||||
""" Maps a filename under help/ without the file extension to the rendered page. """
|
|
||||||
|
|
||||||
|
|
||||||
def render(app: flask.Flask):
|
|
||||||
"""
|
|
||||||
Renders the user documentation. Must be called after all Flask routes have been
|
|
||||||
registered, because the documentation might try to link to them with Flask's `url_for`.
|
|
||||||
|
|
||||||
We render the user documentation once on startup to improve performance.
|
|
||||||
"""
|
|
||||||
|
|
||||||
link_targets = {
|
|
||||||
'brand.git_url': GIT_URL,
|
|
||||||
'brand.public_instances': get_setting('brand.public_instances'),
|
|
||||||
'brand.docs_url': get_setting('brand.docs_url'),
|
|
||||||
}
|
|
||||||
|
|
||||||
base_url = get_setting('server.base_url') or None
|
|
||||||
# we specify base_url so that url_for works for base_urls that have a non-root path
|
|
||||||
|
|
||||||
with app.test_request_context(base_url=base_url):
|
|
||||||
link_targets['url_for:index'] = url_for('index')
|
|
||||||
link_targets['url_for:preferences'] = url_for('preferences')
|
|
||||||
link_targets['url_for:stats'] = url_for('stats')
|
|
||||||
|
|
||||||
define_link_targets = ''.join(f'[{name}]: {url}\n' for name, url in link_targets.items())
|
|
||||||
|
|
||||||
for pagename in _TOC:
|
|
||||||
file_content = pkg_resources.resource_string(__name__, 'help/en/' + pagename + '.md').decode()
|
|
||||||
markdown = define_link_targets + file_content
|
|
||||||
assert file_content.startswith('# ')
|
|
||||||
title = file_content.split('\n', maxsplit=1)[0].strip('# ')
|
|
||||||
content: str = mistletoe.markdown(markdown)
|
|
||||||
|
|
||||||
if pagename == 'about':
|
|
||||||
try:
|
|
||||||
content += pkg_resources.resource_string(__name__, 'templates/__common__/aboutextend.html').decode()
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
PAGES[pagename] = HelpPage(title=title, content=content)
|
|
||||||
@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# lint: pylint
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
"""Script that implements some prebuild tasks needed by target docs.prebuild
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os.path
|
||||||
|
import time
|
||||||
|
from searx import settings, get_setting
|
||||||
|
from searx.infopage import InfoPageSet, InfoPage
|
||||||
|
|
||||||
|
_doc_user = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'docs', 'user'))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
DOC = None
|
||||||
|
base_url = get_setting('server.base_url', None)
|
||||||
|
|
||||||
|
if base_url:
|
||||||
|
DOC = _render_all_with_flask_ctx(base_url)
|
||||||
|
else:
|
||||||
|
DOC = _render_all()
|
||||||
|
for pagename, page in DOC.all_pages('en'):
|
||||||
|
fname = os.path.join(_doc_user, os.path.basename(page.fname))
|
||||||
|
with open(fname, 'w') as f:
|
||||||
|
f.write(page.content)
|
||||||
|
|
||||||
|
|
||||||
|
class OfflinePage(InfoPage):
|
||||||
|
|
||||||
|
def get_ctx(self): # pylint: disable=no-self-use
|
||||||
|
"""Jinja context to render :py:obj:`DocPage.content` for offline purpose (no
|
||||||
|
links to SearXNG instance)"""
|
||||||
|
|
||||||
|
ctx = super().get_ctx()
|
||||||
|
ctx['link'] = lambda name, url: '`%s`' % name
|
||||||
|
ctx['search'] = lambda query: '`%s`' % query
|
||||||
|
|
||||||
|
return ctx
|
||||||
|
|
||||||
|
|
||||||
|
def _render_all():
|
||||||
|
DOC = InfoPageSet(OfflinePage)
|
||||||
|
for pagename, page in DOC.all_pages('en'):
|
||||||
|
page.render()
|
||||||
|
return DOC
|
||||||
|
|
||||||
|
|
||||||
|
def _render_all_with_flask_ctx(base_url):
|
||||||
|
|
||||||
|
DOC = InfoPageSet(InfoPage, base_url)
|
||||||
|
|
||||||
|
# The url_for functions in the jinja templates need all routes to be
|
||||||
|
# registered in the Flask app.
|
||||||
|
|
||||||
|
settings['server']['secret_key'] = "x"
|
||||||
|
from searx.webapp import app
|
||||||
|
|
||||||
|
# Specify base_url so that url_for() works for base_urls. If base_url is
|
||||||
|
# specified, then these values from are given preference over any Flask's
|
||||||
|
# generics (see flaskfix.py).
|
||||||
|
|
||||||
|
with app.test_request_context(base_url=base_url):
|
||||||
|
for pagename, page in DOC.all_pages('en'):
|
||||||
|
page.render()
|
||||||
|
|
||||||
|
# The searx.webapp import from above fires some HTTP requests, thats
|
||||||
|
# why we get a RuntimeError::
|
||||||
|
#
|
||||||
|
# RuntimeError: The connection pool was closed while 1 HTTP \
|
||||||
|
# requests/responses were still in-flight.
|
||||||
|
#
|
||||||
|
# Closing network won't help ..
|
||||||
|
# from searx.network import network
|
||||||
|
# network.done()
|
||||||
|
|
||||||
|
# waiting some seconds before ending the comand line was the only solution I
|
||||||
|
# found ..
|
||||||
|
|
||||||
|
time.sleep(3)
|
||||||
|
return DOC
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
Loading…
Reference in New Issue