build from commit 522ba9a14b
@ -0,0 +1,4 @@
|
||||
# Sphinx build info version 1
|
||||
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||
config: 78ff65dd4ea6762d8a38b76abe50b87c
|
||||
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
@ -0,0 +1,109 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Page not found — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="/_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="/_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="/_static/tabs.css" />
|
||||
<script data-url_root="#" id="documentation_options" src="/_static/documentation_options.js"></script>
|
||||
<script src="/_static/jquery.js"></script>
|
||||
<script src="/_static/underscore.js"></script>
|
||||
<script src="/_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="/_static/doctools.js"></script>
|
||||
<script src="/_static/sphinx_highlight.js"></script>
|
||||
<script src="/_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="/genindex.html" />
|
||||
<link rel="search" title="Search" href="/search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="/genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="/py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="/index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">Page not found</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Page not found</h1>
|
||||
|
||||
Unfortunately we couldn't find the content you were looking for.
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="/index.html">
|
||||
<img class="logo" src="/_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="/index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="/donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="/index.html">Overview</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="/search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="/_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,3 @@
|
||||
digraph foo {
|
||||
"bar" -> "baz";
|
||||
}
|
@ -0,0 +1,31 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: foo Pages: 1 -->
|
||||
<svg width="62pt" height="116pt"
|
||||
viewBox="0.00 0.00 62.00 116.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 112)">
|
||||
<title>foo</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-112 58,-112 58,4 -4,4"/>
|
||||
<!-- bar -->
|
||||
<g id="node1" class="node">
|
||||
<title>bar</title>
|
||||
<ellipse fill="none" stroke="black" cx="27" cy="-90" rx="27" ry="18"/>
|
||||
<text text-anchor="middle" x="27" y="-86.3" font-family="Times,serif" font-size="14.00">bar</text>
|
||||
</g>
|
||||
<!-- baz -->
|
||||
<g id="node2" class="node">
|
||||
<title>baz</title>
|
||||
<ellipse fill="none" stroke="black" cx="27" cy="-18" rx="27" ry="18"/>
|
||||
<text text-anchor="middle" x="27" y="-14.3" font-family="Times,serif" font-size="14.00">baz</text>
|
||||
</g>
|
||||
<!-- bar->baz -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>bar->baz</title>
|
||||
<path fill="none" stroke="black" d="M27,-71.7C27,-63.98 27,-54.71 27,-46.11"/>
|
||||
<polygon fill="black" stroke="black" points="30.5,-46.1 27,-36.1 23.5,-46.1 30.5,-46.1"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 1.3 KiB |
@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
|
||||
baseProfile="full" width="70px" height="40px"
|
||||
viewBox="0 0 700 400"
|
||||
>
|
||||
<line x1="180" y1="370"
|
||||
x2="500" y2="50"
|
||||
stroke="black" stroke-width="15px"
|
||||
/>
|
||||
<polygon points="585 0 525 25 585 50"
|
||||
transform="rotate(135 525 25)"
|
||||
/>
|
||||
</svg>
|
After Width: | Height: | Size: 385 B |
@ -0,0 +1,30 @@
|
||||
digraph G {
|
||||
|
||||
node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans];
|
||||
edge [fontname="Sans"];
|
||||
|
||||
browser [label="browser", shape=tab, fillcolor=aliceblue];
|
||||
rp [label="reverse proxy"];
|
||||
static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray];
|
||||
uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"]
|
||||
redis [label="redis DB", shape=cylinder];
|
||||
searxng1 [label="SearXNG #1", fontcolor=blue3];
|
||||
searxng2 [label="SearXNG #2", fontcolor=blue3];
|
||||
searxng3 [label="SearXNG #3", fontcolor=blue3];
|
||||
searxng4 [label="SearXNG #4", fontcolor=blue3];
|
||||
|
||||
browser -> rp [label="HTTPS"]
|
||||
|
||||
subgraph cluster_searxng {
|
||||
label = "SearXNG instance" fontname=Sans;
|
||||
bgcolor="#fafafa";
|
||||
{ rank=same; static rp };
|
||||
rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray];
|
||||
rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"];
|
||||
uwsgi -> searxng1 -> redis;
|
||||
uwsgi -> searxng2 -> redis;
|
||||
uwsgi -> searxng3 -> redis;
|
||||
uwsgi -> searxng4 -> redis;
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,149 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: G Pages: 1 -->
|
||||
<svg width="543pt" height="401pt"
|
||||
viewBox="0.00 0.00 543.00 401.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 397)">
|
||||
<title>G</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-397 539,-397 539,4 -4,4"/>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_searxng</title>
|
||||
<polygon fill="#fafafa" stroke="black" points="8,-8 8,-316 527,-316 527,-8 8,-8"/>
|
||||
<text text-anchor="middle" x="267.5" y="-300.8" font-family="Sans" font-size="14.00">SearXNG instance</text>
|
||||
</g>
|
||||
<!-- browser -->
|
||||
<g id="node1" class="node">
|
||||
<title>browser</title>
|
||||
<polygon fill="aliceblue" stroke="black" points="108,-393 48,-393 48,-397 36,-397 36,-357 108,-357 108,-393"/>
|
||||
<polyline fill="none" stroke="black" points="36,-393 48,-393 "/>
|
||||
<text text-anchor="middle" x="72" y="-371.3" font-family="Sans" font-size="14.00">browser</text>
|
||||
</g>
|
||||
<!-- rp -->
|
||||
<g id="node2" class="node">
|
||||
<title>rp</title>
|
||||
<polygon fill="#ffffcc" stroke="black" points="128,-285 16,-285 16,-249 128,-249 128,-285"/>
|
||||
<text text-anchor="middle" x="72" y="-263.3" font-family="Sans" font-size="14.00">reverse proxy</text>
|
||||
</g>
|
||||
<!-- browser->rp -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>browser->rp</title>
|
||||
<path fill="none" stroke="black" d="M72,-356.97C72,-340.38 72,-314.88 72,-295.43"/>
|
||||
<polygon fill="black" stroke="black" points="75.5,-295.34 72,-285.34 68.5,-295.34 75.5,-295.34"/>
|
||||
<text text-anchor="middle" x="94.5" y="-327.8" font-family="Sans" font-size="14.00">HTTPS</text>
|
||||
</g>
|
||||
<!-- static -->
|
||||
<g id="node3" class="node">
|
||||
<title>static</title>
|
||||
<g id="a_node3"><a xlink:href="url to configure static files" xlink:title="static files">
|
||||
<polygon fill="lightgray" stroke="black" points="518.5,-285 515.5,-289 494.5,-289 491.5,-285 431.5,-285 431.5,-249 518.5,-249 518.5,-285"/>
|
||||
<text text-anchor="middle" x="475" y="-263.3" font-family="Sans" font-size="14.00">static files</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- rp->static -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>rp->static</title>
|
||||
<path fill="none" stroke="black" d="M128.04,-267C205.6,-267 344.74,-267 421.13,-267"/>
|
||||
<polygon fill="slategray" stroke="black" points="421.48,-270.5 431.48,-267 421.48,-263.5 421.48,-270.5"/>
|
||||
<text text-anchor="middle" x="279.75" y="-273.8" font-family="Sans" font-size="14.00" fill="slategray">optional: reverse proxy serves static files</text>
|
||||
</g>
|
||||
<!-- uwsgi -->
|
||||
<g id="node4" class="node">
|
||||
<title>uwsgi</title>
|
||||
<g id="a_node4"><a xlink:href="https://docs.searxng.org/utils/searx.sh.html" xlink:title="uwsgi">
|
||||
<polygon fill="#ffffcc" stroke="black" points="244.14,-198 158.02,-198 135.86,-162 221.98,-162 244.14,-198"/>
|
||||
<text text-anchor="middle" x="190" y="-176.3" font-family="Sans" font-size="14.00">uwsgi</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- rp->uwsgi -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>rp->uwsgi</title>
|
||||
<path fill="none" stroke="black" d="M95.88,-248.8C113.71,-235.95 138.28,-218.26 157.84,-204.17"/>
|
||||
<polygon fill="black" stroke="black" points="160.09,-206.86 166.16,-198.18 156,-201.18 160.09,-206.86"/>
|
||||
<text text-anchor="middle" x="240.5" y="-219.8" font-family="Sans" font-size="14.00">http:// (tcp) or unix:// (socket)</text>
|
||||
</g>
|
||||
<!-- searxng1 -->
|
||||
<g id="node6" class="node">
|
||||
<title>searxng1</title>
|
||||
<polygon fill="#ffffcc" stroke="black" points="120,-125 16,-125 16,-89 120,-89 120,-125"/>
|
||||
<text text-anchor="middle" x="68" y="-103.3" font-family="Sans" font-size="14.00" fill="#0000cd">SearXNG #1</text>
|
||||
</g>
|
||||
<!-- uwsgi->searxng1 -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>uwsgi->searxng1</title>
|
||||
<path fill="none" stroke="black" d="M160.78,-161.99C144.45,-152.49 123.91,-140.54 106.31,-130.29"/>
|
||||
<polygon fill="black" stroke="black" points="107.81,-127.12 97.41,-125.11 104.29,-133.17 107.81,-127.12"/>
|
||||
</g>
|
||||
<!-- searxng2 -->
|
||||
<g id="node7" class="node">
|
||||
<title>searxng2</title>
|
||||
<polygon fill="#ffffcc" stroke="black" points="242,-125 138,-125 138,-89 242,-89 242,-125"/>
|
||||
<text text-anchor="middle" x="190" y="-103.3" font-family="Sans" font-size="14.00" fill="#0000cd">SearXNG #2</text>
|
||||
</g>
|
||||
<!-- uwsgi->searxng2 -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>uwsgi->searxng2</title>
|
||||
<path fill="none" stroke="black" d="M190,-161.81C190,-153.79 190,-144.05 190,-135.07"/>
|
||||
<polygon fill="black" stroke="black" points="193.5,-135.03 190,-125.03 186.5,-135.03 193.5,-135.03"/>
|
||||
</g>
|
||||
<!-- searxng3 -->
|
||||
<g id="node8" class="node">
|
||||
<title>searxng3</title>
|
||||
<polygon fill="#ffffcc" stroke="black" points="364,-125 260,-125 260,-89 364,-89 364,-125"/>
|
||||
<text text-anchor="middle" x="312" y="-103.3" font-family="Sans" font-size="14.00" fill="#0000cd">SearXNG #3</text>
|
||||
</g>
|
||||
<!-- uwsgi->searxng3 -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>uwsgi->searxng3</title>
|
||||
<path fill="none" stroke="black" d="M219.22,-161.99C235.55,-152.49 256.09,-140.54 273.69,-130.29"/>
|
||||
<polygon fill="black" stroke="black" points="275.71,-133.17 282.59,-125.11 272.19,-127.12 275.71,-133.17"/>
|
||||
</g>
|
||||
<!-- searxng4 -->
|
||||
<g id="node9" class="node">
|
||||
<title>searxng4</title>
|
||||
<polygon fill="#ffffcc" stroke="black" points="486,-125 382,-125 382,-89 486,-89 486,-125"/>
|
||||
<text text-anchor="middle" x="434" y="-103.3" font-family="Sans" font-size="14.00" fill="#0000cd">SearXNG #4</text>
|
||||
</g>
|
||||
<!-- uwsgi->searxng4 -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>uwsgi->searxng4</title>
|
||||
<path fill="none" stroke="black" d="M226,-168.52C264.37,-157.36 325.82,-139.48 372,-126.04"/>
|
||||
<polygon fill="black" stroke="black" points="372.99,-129.4 381.61,-123.24 371.03,-122.68 372.99,-129.4"/>
|
||||
</g>
|
||||
<!-- redis -->
|
||||
<g id="node5" class="node">
|
||||
<title>redis</title>
|
||||
<path fill="#ffffcc" stroke="black" d="M288.5,-48.73C288.5,-50.53 271.69,-52 251,-52 230.31,-52 213.5,-50.53 213.5,-48.73 213.5,-48.73 213.5,-19.27 213.5,-19.27 213.5,-17.47 230.31,-16 251,-16 271.69,-16 288.5,-17.47 288.5,-19.27 288.5,-19.27 288.5,-48.73 288.5,-48.73"/>
|
||||
<path fill="none" stroke="black" d="M288.5,-48.73C288.5,-46.92 271.69,-45.45 251,-45.45 230.31,-45.45 213.5,-46.92 213.5,-48.73"/>
|
||||
<text text-anchor="middle" x="251" y="-30.3" font-family="Sans" font-size="14.00">redis DB</text>
|
||||
</g>
|
||||
<!-- searxng1->redis -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>searxng1->redis</title>
|
||||
<path fill="none" stroke="black" d="M111.83,-88.99C139.55,-78.24 175.36,-64.35 203.71,-53.35"/>
|
||||
<polygon fill="black" stroke="black" points="205.35,-56.47 213.4,-49.59 202.81,-49.94 205.35,-56.47"/>
|
||||
</g>
|
||||
<!-- searxng2->redis -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>searxng2->redis</title>
|
||||
<path fill="none" stroke="black" d="M204.77,-88.81C212.26,-80.09 221.5,-69.34 229.74,-59.75"/>
|
||||
<polygon fill="black" stroke="black" points="232.51,-61.89 236.37,-52.03 227.2,-57.33 232.51,-61.89"/>
|
||||
</g>
|
||||
<!-- searxng3->redis -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>searxng3->redis</title>
|
||||
<path fill="none" stroke="black" d="M297.23,-88.81C289.74,-80.09 280.5,-69.34 272.26,-59.75"/>
|
||||
<polygon fill="black" stroke="black" points="274.8,-57.33 265.63,-52.03 269.49,-61.89 274.8,-57.33"/>
|
||||
</g>
|
||||
<!-- searxng4->redis -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>searxng4->redis</title>
|
||||
<path fill="none" stroke="black" d="M390.17,-88.99C362.45,-78.24 326.64,-64.35 298.29,-53.35"/>
|
||||
<polygon fill="black" stroke="black" points="299.19,-49.94 288.6,-49.59 296.65,-56.47 299.19,-49.94"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 7.4 KiB |
@ -0,0 +1,3 @@
|
||||
graph G {
|
||||
Hello -- World
|
||||
}
|
@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: G Pages: 1 -->
|
||||
<svg width="85pt" height="116pt"
|
||||
viewBox="0.00 0.00 84.69 116.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 112)">
|
||||
<title>G</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-112 80.69,-112 80.69,4 -4,4"/>
|
||||
<!-- Hello -->
|
||||
<g id="node1" class="node">
|
||||
<title>Hello</title>
|
||||
<ellipse fill="none" stroke="black" cx="38.35" cy="-90" rx="35.19" ry="18"/>
|
||||
<text text-anchor="middle" x="38.35" y="-86.3" font-family="Times,serif" font-size="14.00">Hello</text>
|
||||
</g>
|
||||
<!-- World -->
|
||||
<g id="node2" class="node">
|
||||
<title>World</title>
|
||||
<ellipse fill="none" stroke="black" cx="38.35" cy="-18" rx="38.19" ry="18"/>
|
||||
<text text-anchor="middle" x="38.35" y="-14.3" font-family="Times,serif" font-size="14.00">World</text>
|
||||
</g>
|
||||
<!-- Hello--World -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>Hello--World</title>
|
||||
<path fill="none" stroke="black" d="M38.35,-71.7C38.35,-60.85 38.35,-46.92 38.35,-36.1"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 1.2 KiB |
@ -0,0 +1,25 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<!-- This file was generated by dvisvgm 2.8.1 -->
|
||||
<svg version='1.1' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='36.523255pt' height='48.422712pt' viewBox='57.608781 53.798251 36.523255 48.422712'>
|
||||
<defs>
|
||||
<path id='g0-0' d='M9.191532-3.20797C9.428643-3.20797 9.679701-3.20797 9.679701-3.486924S9.428643-3.765878 9.191532-3.765878H1.645828C1.408717-3.765878 1.157659-3.765878 1.157659-3.486924S1.408717-3.20797 1.645828-3.20797H9.191532Z'/>
|
||||
<path id='g1-120' d='M6.611208-5.69066C6.164882-5.606974 5.997509-5.272229 5.997509-5.007223C5.997509-4.672478 6.262516-4.560897 6.457783-4.560897C6.876214-4.560897 7.169116-4.923537 7.169116-5.300125C7.169116-5.885928 6.499626-6.150934 5.913823-6.150934C5.063014-6.150934 4.588792-5.314072 4.463263-5.049066C4.142466-6.095143 3.277709-6.150934 3.02665-6.150934C1.603985-6.150934 .850809-4.323786 .850809-4.016936C.850809-3.961146 .9066-3.891407 1.004234-3.891407C1.115816-3.891407 1.143711-3.975093 1.171606-4.030884C1.645828-5.579078 2.580324-5.87198 2.984807-5.87198C3.612453-5.87198 3.737983-5.286177 3.737983-4.951432C3.737983-4.644583 3.654296-4.323786 3.486924-3.654296L3.012702-1.743462C2.803487-.9066 2.399004-.139477 1.659776-.139477C1.590037-.139477 1.241345-.139477 .948443-.320797C1.45056-.418431 1.562142-.836862 1.562142-1.004234C1.562142-1.283188 1.352927-1.45056 1.08792-1.45056C.753176-1.45056 .390535-1.157659 .390535-.711333C.390535-.125529 1.046077 .139477 1.645828 .139477C2.315318 .139477 2.789539-.390535 3.082441-.962391C3.305604-.139477 4.002989 .139477 4.519054 .139477C5.941719 .139477 6.694894-1.687671 6.694894-1.994521C6.694894-2.064259 6.639103-2.12005 6.555417-2.12005C6.429888-2.12005 6.41594-2.050311 6.374097-1.93873C5.997509-.711333 5.188543-.139477 4.560897-.139477C4.072727-.139477 3.807721-.502117 3.807721-1.073973C3.807721-1.380822 3.863512-1.603985 4.086675-2.524533L4.574844-4.42142C4.78406-5.258281 5.258281-5.87198 5.899875-5.87198C5.927771-5.87198 6.318306-5.87198 6.611208-5.69066Z'/>
|
||||
<path id='g1-121' d='M3.668244 1.562142C3.291656 2.092154 2.747696 2.566376 2.064259 2.566376C1.896887 2.566376 1.227397 2.538481 1.018182 1.896887C1.060025 1.910834 1.129763 1.910834 1.157659 1.910834C1.57609 1.910834 1.855044 1.548194 1.855044 1.227397S1.590037 .795019 1.380822 .795019C1.157659 .795019 .669489 .962391 .669489 1.645828C.669489 2.357161 1.26924 2.84533 2.064259 2.84533C3.459029 2.84533 4.867746 1.562142 5.258281 .013948L6.625156-5.425654C6.639103-5.495392 6.666999-5.579078 6.666999-5.662765C6.666999-5.87198 6.499626-6.011457 6.290411-6.011457C6.164882-6.011457 5.87198-5.955666 5.760399-5.537235L4.728269-1.436613C4.658531-1.185554 4.658531-1.157659 4.546949-1.004234C4.267995-.613699 3.807721-.139477 3.138232-.139477C2.357161-.139477 2.287422-.9066 2.287422-1.283188C2.287422-2.078207 2.66401-3.152179 3.040598-4.156413C3.194022-4.560897 3.277709-4.756164 3.277709-5.035118C3.277709-5.620922 2.859278-6.150934 2.175841-6.150934C.892653-6.150934 .376588-4.128518 .376588-4.016936C.376588-3.961146 .432379-3.891407 .530012-3.891407C.655542-3.891407 .669489-3.947198 .72528-4.142466C1.060025-5.314072 1.590037-5.87198 2.133998-5.87198C2.259527-5.87198 2.496638-5.87198 2.496638-5.411706C2.496638-5.049066 2.343213-4.644583 2.133998-4.11457C1.45056-2.287422 1.45056-1.827148 1.45056-1.492403C1.45056-.167372 2.399004 .139477 3.096389 .139477C3.500872 .139477 4.002989 .013948 4.491158-.502117L4.505106-.488169C4.29589 .334745 4.156413 .878705 3.668244 1.562142Z'/>
|
||||
<path id='g1-122' d='M1.771357-1.129763C2.371108-1.8132 2.859278-2.245579 3.556663-2.873225C4.393524-3.598506 4.756164-3.947198 4.951432-4.156413C5.927771-5.118804 6.41594-5.927771 6.41594-6.039352S6.304359-6.150934 6.276463-6.150934C6.178829-6.150934 6.150934-6.095143 6.081196-5.997509C5.732503-5.397758 5.397758-5.104857 5.035118-5.104857C4.742217-5.104857 4.588792-5.230386 4.323786-5.565131C4.030884-5.913823 3.793773-6.150934 3.38929-6.150934C2.371108-6.150934 1.75741-4.881694 1.75741-4.588792C1.75741-4.546949 1.771357-4.463263 1.896887-4.463263C2.008468-4.463263 2.022416-4.519054 2.064259-4.616687C2.30137-5.174595 2.970859-5.272229 3.235866-5.272229C3.528767-5.272229 3.807721-5.174595 4.100623-5.049066C4.630635-4.825903 4.853798-4.825903 4.993275-4.825903C5.090909-4.825903 5.1467-4.825903 5.216438-4.839851C4.756164-4.29589 4.002989-3.626401 3.375342-3.054545L1.966625-1.75741C1.115816-.892653 .599751-.069738 .599751 .027895C.599751 .111582 .669489 .139477 .753176 .139477S.850809 .125529 .948443-.041843C1.171606-.390535 1.617933-.9066 2.133998-.9066C2.426899-.9066 2.566376-.808966 2.84533-.460274C3.110336-.153425 3.347447 .139477 3.793773 .139477C5.160648 .139477 5.941719-1.63188 5.941719-1.952677C5.941719-2.008468 5.927771-2.092154 5.788294-2.092154C5.676712-2.092154 5.662765-2.036364 5.620922-1.896887C5.314072-1.073973 4.491158-.739228 3.947198-.739228C3.654296-.739228 3.375342-.836862 3.082441-.962391C2.524533-1.185554 2.371108-1.185554 2.189788-1.185554C2.050311-1.185554 1.896887-1.185554 1.771357-1.129763Z'/>
|
||||
<path id='g2-43' d='M5.565131-3.221918H9.414695C9.609963-3.221918 9.861021-3.221918 9.861021-3.472976C9.861021-3.737983 9.62391-3.737983 9.414695-3.737983H5.565131V-7.587547C5.565131-7.782814 5.565131-8.033873 5.314072-8.033873C5.049066-8.033873 5.049066-7.796762 5.049066-7.587547V-3.737983H1.199502C1.004234-3.737983 .753176-3.737983 .753176-3.486924C.753176-3.221918 .990286-3.221918 1.199502-3.221918H5.049066V.627646C5.049066 .822914 5.049066 1.073973 5.300125 1.073973C5.565131 1.073973 5.565131 .836862 5.565131 .627646V-3.221918Z'/>
|
||||
<path id='g2-49' d='M4.016936-8.940473C4.016936-9.26127 4.016936-9.275218 3.737983-9.275218C3.403238-8.89863 2.705853-8.382565 1.26924-8.382565V-7.978082C1.590037-7.978082 2.287422-7.978082 3.054545-8.340722V-1.073973C3.054545-.571856 3.012702-.404483 1.785305-.404483H1.352927V0C1.729514-.027895 3.082441-.027895 3.542715-.027895S5.341968-.027895 5.718555 0V-.404483H5.286177C4.05878-.404483 4.016936-.571856 4.016936-1.073973V-8.940473Z'/>
|
||||
</defs>
|
||||
<g id='page1'>
|
||||
<use x='59.270442' y='62.786766' xlink:href='#g2-49'/>
|
||||
<rect x='58.804295' y='68.456606' height='.55789' width='7.760768'/>
|
||||
<use x='58.804295' y='81.789907' xlink:href='#g1-120'/>
|
||||
<use x='70.860038' y='72.222482' xlink:href='#g2-43'/>
|
||||
<use x='85.942577' y='62.786766' xlink:href='#g2-49'/>
|
||||
<rect x='85.777106' y='68.456606' height='.55789' width='7.159416'/>
|
||||
<use x='85.777106' y='81.789907' xlink:href='#g1-121'/>
|
||||
<rect x='57.608781' y='86.175621' height='.55789' width='36.523255'/>
|
||||
<use x='60.284264' y='99.508922' xlink:href='#g1-121'/>
|
||||
<use x='70.543155' y='99.508922' xlink:href='#g0-0'/>
|
||||
<use x='84.490862' y='99.508922' xlink:href='#g1-122'/>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 6.6 KiB |
@ -0,0 +1,25 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<!-- This file was generated by dvisvgm 2.8.1 -->
|
||||
<svg version='1.1' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='23.092891pt' height='29.523663pt' viewBox='57.608781 53.79825 23.092891 29.523663'>
|
||||
<defs>
|
||||
<path id='g0-0' d='M6.434044-2.245569C6.600021-2.245569 6.775761-2.245569 6.775761-2.440836S6.600021-2.636103 6.434044-2.636103H1.152075C.986098-2.636103 .810358-2.636103 .810358-2.440836S.986098-2.245569 1.152075-2.245569H6.434044Z'/>
|
||||
<path id='g1-120' d='M3.260957-2.94853C3.319537-3.202377 3.544094-4.100605 4.227528-4.100605C4.276345-4.100605 4.510665-4.100605 4.715696-3.973681C4.442322-3.924865 4.247055-3.680781 4.247055-3.446461C4.247055-3.290247 4.354452-3.104744 4.618062-3.104744C4.832856-3.104744 5.145283-3.280484 5.145283-3.671018C5.145283-4.178712 4.569245-4.315398 4.237292-4.315398C3.671018-4.315398 3.329301-3.797941 3.21214-3.573384C2.968057-4.217765 2.440836-4.315398 2.157699-4.315398C1.142311-4.315398 .585801-3.055927 .585801-2.811843C.585801-2.71421 .683434-2.71421 .702961-2.71421C.781068-2.71421 .810358-2.733737 .829884-2.821607C1.161838-3.856521 1.806219-4.100605 2.138172-4.100605C2.323676-4.100605 2.665393-4.012735 2.665393-3.446461C2.665393-3.143797 2.499416-2.489653 2.138172-1.122785C1.981959-.517457 1.640242-.107397 1.210655-.107397C1.152075-.107397 .927518-.107397 .722488-.23432C.966571-.283137 1.181365-.488167 1.181365-.761541C1.181365-1.025151 .966571-1.103258 .820121-1.103258C.527221-1.103258 .283137-.849411 .283137-.536984C.283137-.08787 .771304 .107397 1.200891 .107397C1.845272 .107397 2.196753-.576037 2.226043-.634617C2.343203-.273374 2.694683 .107397 3.280484 .107397C4.286108 .107397 4.842619-1.152075 4.842619-1.396158C4.842619-1.493792 4.754749-1.493792 4.725459-1.493792C4.637589-1.493792 4.618062-1.454738 4.598535-1.386395C4.276345-.341717 3.612438-.107397 3.300011-.107397C2.91924-.107397 2.763027-.419824 2.763027-.751778C2.763027-.966571 2.821607-1.181365 2.929003-1.610952L3.260957-2.94853Z'/>
|
||||
<path id='g1-121' d='M4.744986-3.719834C4.784039-3.856521 4.784039-3.876048 4.784039-3.944391C4.784039-4.120131 4.647352-4.208002 4.500902-4.208002C4.403268-4.208002 4.247055-4.149422 4.159185-4.002971C4.139658-3.954155 4.061551-3.651491 4.022498-3.475751C3.954155-3.221904 3.885811-2.958293 3.827231-2.694683L3.387881-.937281C3.348827-.790831 2.929003-.107397 2.284623-.107397C1.786692-.107397 1.679295-.536984 1.679295-.898228C1.679295-1.347342 1.845272-1.952669 2.177226-2.811843C2.333439-3.21214 2.372493-3.319537 2.372493-3.514804C2.372493-3.954155 2.060066-4.315398 1.571899-4.315398C.644381-4.315398 .283137-2.899713 .283137-2.811843C.283137-2.71421 .38077-2.71421 .400297-2.71421C.497931-2.71421 .507694-2.733737 .556511-2.88995C.820121-3.807704 1.210655-4.100605 1.542608-4.100605C1.620715-4.100605 1.786692-4.100605 1.786692-3.788178C1.786692-3.544094 1.689059-3.290247 1.620715-3.104744C1.230181-2.069829 1.054441-1.513318 1.054441-1.054441C1.054441-.185504 1.669532 .107397 2.245569 .107397C2.62634 .107397 2.958293-.05858 3.231667-.331954C3.104744 .17574 2.987583 .654144 2.59705 1.171601C2.343203 1.503555 1.972196 1.786692 1.523082 1.786692C1.386395 1.786692 .947044 1.757402 .781068 1.376632C.937281 1.376632 1.064205 1.376632 1.200891 1.259471C1.298525 1.171601 1.396158 1.044678 1.396158 .859174C1.396158 .556511 1.132548 .517457 1.034915 .517457C.810358 .517457 .488167 .673671 .488167 1.152075C.488167 1.640242 .917754 2.001486 1.523082 2.001486C2.528706 2.001486 3.534331 1.113021 3.807704 .009763L4.744986-3.719834Z'/>
|
||||
<path id='g1-122' d='M1.298525-.810358C1.825745-1.376632 2.108882-1.620715 2.460363-1.923379C2.460363-1.933142 3.06569-2.4506 3.417171-2.80208C4.344688-3.710071 4.559482-4.178712 4.559482-4.217765C4.559482-4.315398 4.471612-4.315398 4.452085-4.315398C4.383742-4.315398 4.354452-4.295872 4.305635-4.208002C4.012735-3.739361 3.807704-3.583148 3.573384-3.583148S3.221904-3.729598 3.075454-3.895575C2.88995-4.120131 2.723973-4.315398 2.401783-4.315398C1.669532-4.315398 1.220418-3.407407 1.220418-3.202377C1.220418-3.15356 1.249708-3.09498 1.337578-3.09498S1.444975-3.143797 1.464502-3.202377C1.650005-3.651491 2.216279-3.661254 2.294386-3.661254C2.499416-3.661254 2.68492-3.592911 2.909477-3.514804C3.300011-3.368354 3.407407-3.368354 3.661254-3.368354C3.309774-2.94853 2.489653-2.245569 2.304149-2.089356L1.425448-1.269235C.761541-.615091 .419824-.05858 .419824 .009763C.419824 .107397 .517457 .107397 .536984 .107397C.615091 .107397 .634617 .08787 .693197-.019527C.917754-.361244 1.210655-.624854 1.523082-.624854C1.747639-.624854 1.845272-.536984 2.089356-.253847C2.255333-.048817 2.431073 .107397 2.71421 .107397C3.680781 .107397 4.247055-1.132548 4.247055-1.396158C4.247055-1.444975 4.208002-1.493792 4.129895-1.493792C4.042025-1.493792 4.022498-1.435212 3.993208-1.366868C3.768651-.732251 3.143797-.546747 2.821607-.546747C2.62634-.546747 2.4506-.605327 2.245569-.673671C1.913616-.800594 1.767165-.839648 1.562135-.839648C1.542608-.839648 1.386395-.839648 1.298525-.810358Z'/>
|
||||
<path id='g2-43' d='M3.993208-2.245569H6.717181C6.853868-2.245569 7.039372-2.245569 7.039372-2.440836S6.853868-2.636103 6.717181-2.636103H3.993208V-5.36984C3.993208-5.506526 3.993208-5.69203 3.797941-5.69203S3.602674-5.506526 3.602674-5.36984V-2.636103H.868938C.732251-2.636103 .546747-2.636103 .546747-2.440836S.732251-2.245569 .868938-2.245569H3.602674V.488167C3.602674 .624854 3.602674 .810358 3.797941 .810358S3.993208 .624854 3.993208 .488167V-2.245569Z'/>
|
||||
<path id='g2-49' d='M2.870423-6.248541C2.870423-6.482861 2.870423-6.502388 2.645866-6.502388C2.040539-5.877534 1.181365-5.877534 .868938-5.877534V-5.57487C1.064205-5.57487 1.640242-5.57487 2.147936-5.828717V-.771304C2.147936-.419824 2.118646-.302664 1.239945-.302664H.927518V0C1.269235-.02929 2.118646-.02929 2.50918-.02929S3.749124-.02929 4.090841 0V-.302664H3.778414C2.899713-.302664 2.870423-.41006 2.870423-.771304V-6.248541Z'/>
|
||||
</defs>
|
||||
<g id='page1'>
|
||||
<use x='59.153464' y='60.090183' xlink:href='#g2-49'/>
|
||||
<rect x='58.804295' y='61.81595' height='.55789' width='5.580027'/>
|
||||
<use x='58.804295' y='70.391549' xlink:href='#g1-120'/>
|
||||
<use x='65.579836' y='65.581827' xlink:href='#g2-43'/>
|
||||
<use x='74.496784' y='60.090183' xlink:href='#g2-49'/>
|
||||
<rect x='74.369089' y='61.81595' height='.55789' width='5.137078'/>
|
||||
<use x='74.369089' y='70.391549' xlink:href='#g1-121'/>
|
||||
<rect x='57.608781' y='72.847884' height='.55789' width='23.092891'/>
|
||||
<use x='60.304908' y='81.423483' xlink:href='#g1-121'/>
|
||||
<use x='65.441996' y='81.423483' xlink:href='#g0-0'/>
|
||||
<use x='73.035735' y='81.423483' xlink:href='#g1-122'/>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 6.5 KiB |
@ -0,0 +1,22 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<!-- This file was generated by dvisvgm 2.8.1 -->
|
||||
<svg version='1.1' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='69.686469pt' height='12.418845pt' viewBox='56.413267 54.399982 69.686469 12.418845'>
|
||||
<defs>
|
||||
<path id='g2-43' d='M5.565131-3.221918H9.414695C9.609963-3.221918 9.861021-3.221918 9.861021-3.472976C9.861021-3.737983 9.62391-3.737983 9.414695-3.737983H5.565131V-7.587547C5.565131-7.782814 5.565131-8.033873 5.314072-8.033873C5.049066-8.033873 5.049066-7.796762 5.049066-7.587547V-3.737983H1.199502C1.004234-3.737983 .753176-3.737983 .753176-3.486924C.753176-3.221918 .990286-3.221918 1.199502-3.221918H5.049066V.627646C5.049066 .822914 5.049066 1.073973 5.300125 1.073973C5.565131 1.073973 5.565131 .836862 5.565131 .627646V-3.221918Z'/>
|
||||
<path id='g2-61' d='M9.414695-4.519054C9.609963-4.519054 9.861021-4.519054 9.861021-4.770112C9.861021-5.035118 9.62391-5.035118 9.414695-5.035118H1.199502C1.004234-5.035118 .753176-5.035118 .753176-4.78406C.753176-4.519054 .990286-4.519054 1.199502-4.519054H9.414695ZM9.414695-1.924782C9.609963-1.924782 9.861021-1.924782 9.861021-2.175841C9.861021-2.440847 9.62391-2.440847 9.414695-2.440847H1.199502C1.004234-2.440847 .753176-2.440847 .753176-2.189788C.753176-1.924782 .990286-1.924782 1.199502-1.924782H9.414695Z'/>
|
||||
<path id='g1-50' d='M1.239945-.751778L2.274859-1.757402C3.797941-3.104744 4.383742-3.631964 4.383742-4.608299C4.383742-5.72132 3.505041-6.502388 2.313913-6.502388C1.210655-6.502388 .488167-5.60416 .488167-4.735222C.488167-4.188475 .976334-4.188475 1.005625-4.188475C1.171601-4.188475 1.513318-4.305635 1.513318-4.705932C1.513318-4.959779 1.337578-5.213626 .995861-5.213626C.917754-5.213626 .898228-5.213626 .868938-5.203863C1.093495-5.83848 1.620715-6.199724 2.186989-6.199724C3.075454-6.199724 3.495277-5.408893 3.495277-4.608299C3.495277-3.827231 3.00711-3.055927 2.470126-2.4506L.595564-.361244C.488167-.253847 .488167-.23432 .488167 0H4.110368L4.383742-1.698822H4.139658C4.090841-1.405922 4.022498-.976334 3.924865-.829884C3.856521-.751778 3.21214-.751778 2.997347-.751778H1.239945Z'/>
|
||||
<path id='g0-97' d='M4.198257-1.659776C4.128518-1.422665 4.128518-1.39477 3.93325-1.129763C3.626401-.739228 3.012702-.139477 2.357161-.139477C1.785305-.139477 1.464508-.655542 1.464508-1.478456C1.464508-2.245579 1.896887-3.807721 2.161893-4.393524C2.636115-5.369863 3.291656-5.87198 3.835616-5.87198C4.756164-5.87198 4.937484-4.728269 4.937484-4.616687C4.937484-4.60274 4.895641-4.42142 4.881694-4.393524L4.198257-1.659776ZM5.090909-5.230386C4.937484-5.593026 4.560897-6.150934 3.835616-6.150934C2.259527-6.150934 .557908-4.11457 .557908-2.050311C.557908-.669489 1.366874 .139477 2.315318 .139477C3.082441 .139477 3.737983-.460274 4.128518-.920548C4.267995-.097634 4.923537 .139477 5.341968 .139477S6.095143-.111582 6.346202-.613699C6.569365-1.08792 6.764633-1.93873 6.764633-1.994521C6.764633-2.064259 6.708842-2.12005 6.625156-2.12005C6.499626-2.12005 6.485679-2.050311 6.429888-1.841096C6.220672-1.018182 5.955666-.139477 5.383811-.139477C4.979328-.139477 4.951432-.502117 4.951432-.781071C4.951432-1.101868 4.993275-1.255293 5.118804-1.799253C5.216438-2.147945 5.286177-2.454795 5.397758-2.859278C5.913823-4.951432 6.039352-5.453549 6.039352-5.537235C6.039352-5.732503 5.885928-5.885928 5.676712-5.885928C5.230386-5.885928 5.118804-5.397758 5.090909-5.230386Z'/>
|
||||
<path id='g0-98' d='M3.221918-9.331009C3.235866-9.3868 3.263761-9.470486 3.263761-9.540224C3.263761-9.679701 3.124284-9.679701 3.096389-9.679701C3.082441-9.679701 2.580324-9.637858 2.329265-9.609963C2.092154-9.596015 1.882939-9.56812 1.63188-9.554172C1.297136-9.526276 1.199502-9.512329 1.199502-9.26127C1.199502-9.121793 1.338979-9.121793 1.478456-9.121793C2.189788-9.121793 2.189788-8.996264 2.189788-8.856787C2.189788-8.759153 2.078207-8.35467 2.022416-8.103611L1.687671-6.764633C1.548194-6.206725 .753176-3.040598 .697385-2.789539C.627646-2.440847 .627646-2.203736 .627646-2.022416C.627646-.599751 1.422665 .139477 2.329265 .139477C3.947198 .139477 5.620922-1.93873 5.620922-3.961146C5.620922-5.244334 4.895641-6.150934 3.849564-6.150934C3.124284-6.150934 2.468742-5.551183 2.203736-5.272229L3.221918-9.331009ZM2.343213-.139477C1.896887-.139477 1.408717-.474222 1.408717-1.562142C1.408717-2.022416 1.45056-2.287422 1.701619-3.263761C1.743462-3.445081 1.966625-4.337733 2.022416-4.519054C2.050311-4.630635 2.873225-5.87198 3.821669-5.87198C4.435367-5.87198 4.714321-5.258281 4.714321-4.533001C4.714321-3.863512 4.323786-2.287422 3.975093-1.562142C3.626401-.808966 2.984807-.139477 2.343213-.139477Z'/>
|
||||
<path id='g0-99' d='M5.453549-5.244334C5.188543-5.244334 5.063014-5.244334 4.867746-5.076961C4.78406-5.007223 4.630635-4.798007 4.630635-4.574844C4.630635-4.29589 4.839851-4.128518 5.104857-4.128518C5.439601-4.128518 5.816189-4.407472 5.816189-4.96538C5.816189-5.634869 5.174595-6.150934 4.212204-6.150934C2.385056-6.150934 .557908-4.156413 .557908-2.175841C.557908-.962391 1.311083 .139477 2.733748 .139477C4.630635 .139477 5.830137-1.338979 5.830137-1.520299C5.830137-1.603985 5.746451-1.673724 5.69066-1.673724C5.648817-1.673724 5.634869-1.659776 5.50934-1.534247C4.616687-.348692 3.291656-.139477 2.761644-.139477C1.799253-.139477 1.492403-.976339 1.492403-1.673724C1.492403-2.161893 1.729514-3.514819 2.231631-4.463263C2.594271-5.118804 3.347447-5.87198 4.226152-5.87198C4.407472-5.87198 5.174595-5.844085 5.453549-5.244334Z'/>
|
||||
</defs>
|
||||
<g id='page1'>
|
||||
<use x='56.413267' y='65.753425' xlink:href='#g0-97'/>
|
||||
<use x='63.582369' y='60.691915' xlink:href='#g1-50'/>
|
||||
<use x='72.061649' y='65.753425' xlink:href='#g2-43'/>
|
||||
<use x='85.783204' y='65.753425' xlink:href='#g0-98'/>
|
||||
<use x='91.589827' y='60.691915' xlink:href='#g1-50'/>
|
||||
<use x='100.843973' y='65.753425' xlink:href='#g2-61'/>
|
||||
<use x='115.340392' y='65.753425' xlink:href='#g0-99'/>
|
||||
<use x='121.218045' y='60.691915' xlink:href='#g1-50'/>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 5.8 KiB |
@ -0,0 +1,43 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<!-- This file was generated by dvisvgm 2.8.1 -->
|
||||
<svg version='1.1' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='134.289458pt' height='28.68903pt' viewBox='165.981071 78.308016 134.289458 28.68903'>
|
||||
<defs>
|
||||
<path id='g1-105' d='M3.807721-3.249813C3.87746-3.417186 3.87746-3.445081 3.87746-3.486924S3.87746-3.556663 3.807721-3.724035L1.366874-10.139975C1.283188-10.377086 1.199502-10.460772 1.046077-10.460772S.767123-10.335243 .767123-10.181818C.767123-10.139975 .767123-10.11208 .836862-9.958655L3.305604-3.486924L.836862 2.956912C.767123 3.110336 .767123 3.138232 .767123 3.20797C.767123 3.361395 .892653 3.486924 1.046077 3.486924C1.227397 3.486924 1.283188 3.347447 1.338979 3.20797L3.807721-3.249813Z'/>
|
||||
<path id='g1-106' d='M2.217684-9.958655C2.217684-10.209714 2.217684-10.460772 1.93873-10.460772S1.659776-10.209714 1.659776-9.958655V2.984807C1.659776 3.235866 1.659776 3.486924 1.93873 3.486924S2.217684 3.235866 2.217684 2.984807V-9.958655Z'/>
|
||||
<path id='g2-32' d='M6.555417-9.344956C6.555417-9.358904 6.611208-9.540224 6.611208-9.554172C6.611208-9.679701 6.499626-9.679701 6.457783-9.679701C6.332254-9.679701 6.318306-9.609963 6.262516-9.400747L3.961146-.167372C2.803487-.306849 2.371108-.892653 2.371108-1.729514C2.371108-2.036364 2.371108-2.357161 3.02665-4.086675C3.20797-4.588792 3.277709-4.770112 3.277709-5.021171C3.277709-5.648817 2.831382-6.150934 2.175841-6.150934C.892653-6.150934 .376588-4.128518 .376588-4.016936C.376588-3.961146 .432379-3.891407 .530012-3.891407C.655542-3.891407 .669489-3.947198 .72528-4.142466C1.060025-5.355915 1.617933-5.87198 2.133998-5.87198C2.259527-5.87198 2.496638-5.858032 2.496638-5.411706C2.496638-5.355915 2.496638-5.049066 2.259527-4.435367C1.506351-2.454795 1.506351-2.147945 1.506351-1.827148C1.506351-.488169 2.622167 .027895 3.87746 .125529C3.765878 .557908 3.668244 1.004234 3.556663 1.436613C3.333499 2.273474 3.235866 2.66401 3.235866 2.719801C3.235866 2.84533 3.347447 2.84533 3.38929 2.84533C3.417186 2.84533 3.472976 2.84533 3.500872 2.789539C3.556663 2.733748 4.128518 .390535 4.184309 .139477C4.700374 .139477 5.802242 .139477 7.057534-1.157659C7.517808-1.659776 7.936239-2.30137 8.17335-2.901121C8.312827-3.263761 8.647572-4.505106 8.647572-5.216438C8.647572-6.0533 8.229141-6.150934 8.089664-6.150934C7.754919-6.150934 7.44807-5.816189 7.44807-5.537235C7.44807-5.369863 7.545704-5.272229 7.601494-5.216438C7.727024-5.090909 8.103611-4.714321 8.103611-3.989041C8.103611-3.486924 7.824658-2.454795 6.932005-1.45056C5.760399-.139477 4.686426-.139477 4.267995-.139477L6.555417-9.344956Z'/>
|
||||
<path id='g2-58' d='M2.566376-.669489C2.566376-1.073973 2.231631-1.352927 1.896887-1.352927C1.492403-1.352927 1.21345-1.018182 1.21345-.683437C1.21345-.278954 1.548194 0 1.882939 0C2.287422 0 2.566376-.334745 2.566376-.669489Z'/>
|
||||
<path id='g2-64' d='M6.332254-4.658531C6.248568-5.439601 5.760399-6.374097 4.505106-6.374097C2.538481-6.374097 .530012-4.379577 .530012-2.161893C.530012-1.311083 1.115816 .292902 3.012702 .292902C6.304359 .292902 7.713076-4.505106 7.713076-6.41594C7.713076-8.424408 6.583313-9.972603 4.798007-9.972603C2.775592-9.972603 2.175841-8.201245 2.175841-7.824658C2.175841-7.699128 2.259527-7.392279 2.650062-7.392279C3.138232-7.392279 3.347447-7.838605 3.347447-8.075716C3.347447-8.508095 2.915068-8.508095 2.733748-8.508095C3.305604-9.540224 4.365629-9.637858 4.742217-9.637858C5.969614-9.637858 6.750685-8.661519 6.750685-7.099377C6.750685-6.206725 6.485679-5.174595 6.346202-4.658531H6.332254ZM3.054545-.083686C1.743462-.083686 1.520299-1.115816 1.520299-1.701619C1.520299-2.315318 1.910834-3.75193 2.12005-4.267995C2.30137-4.686426 3.096389-6.095143 4.546949-6.095143C5.816189-6.095143 6.109091-4.993275 6.109091-4.2401C6.109091-3.20797 5.202491-.083686 3.054545-.083686Z'/>
|
||||
<path id='g2-72' d='M10.432877-8.508095C10.558406-8.982316 10.586301-9.121793 11.576588-9.121793C11.827646-9.121793 11.967123-9.121793 11.967123-9.372852C11.967123-9.526276 11.841594-9.526276 11.757908-9.526276C11.506849-9.526276 11.213948-9.498381 10.948941-9.498381H9.303113C9.038107-9.498381 8.745205-9.526276 8.480199-9.526276C8.382565-9.526276 8.215193-9.526276 8.215193-9.26127C8.215193-9.121793 8.312827-9.121793 8.577833-9.121793C9.414695-9.121793 9.414695-9.010212 9.414695-8.856787C9.414695-8.828892 9.414695-8.745205 9.358904-8.53599L8.508095-5.160648H4.29589L5.132752-8.508095C5.258281-8.982316 5.286177-9.121793 6.276463-9.121793C6.527522-9.121793 6.666999-9.121793 6.666999-9.372852C6.666999-9.526276 6.541469-9.526276 6.457783-9.526276C6.206725-9.526276 5.913823-9.498381 5.648817-9.498381H4.002989C3.737983-9.498381 3.445081-9.526276 3.180075-9.526276C3.082441-9.526276 2.915068-9.526276 2.915068-9.26127C2.915068-9.121793 3.012702-9.121793 3.277709-9.121793C4.11457-9.121793 4.11457-9.010212 4.11457-8.856787C4.11457-8.828892 4.11457-8.745205 4.05878-8.53599L2.175841-1.03213C2.050311-.54396 2.022416-.404483 1.060025-.404483C.739228-.404483 .641594-.404483 .641594-.139477C.641594 0 .795019 0 .836862 0C1.08792 0 1.380822-.027895 1.645828-.027895H3.291656C3.556663-.027895 3.849564 0 4.11457 0C4.226152 0 4.379577 0 4.379577-.265006C4.379577-.404483 4.254047-.404483 4.044832-.404483C3.194022-.404483 3.194022-.516065 3.194022-.655542C3.194022-.669489 3.194022-.767123 3.221918-.878705L4.184309-4.756164H8.410461C8.17335-3.835616 7.462017-.920548 7.434122-.836862C7.280697-.418431 7.057534-.418431 6.23462-.404483C6.067248-.404483 5.941719-.404483 5.941719-.139477C5.941719 0 6.095143 0 6.136986 0C6.388045 0 6.680946-.027895 6.945953-.027895H8.591781C8.856787-.027895 9.149689 0 9.414695 0C9.526276 0 9.679701 0 9.679701-.265006C9.679701-.404483 9.554172-.404483 9.344956-.404483C8.494147-.404483 8.494147-.516065 8.494147-.655542C8.494147-.669489 8.494147-.767123 8.522042-.878705L10.432877-8.508095Z'/>
|
||||
<path id='g2-116' d='M2.803487-5.606974H4.086675C4.351681-5.606974 4.491158-5.606974 4.491158-5.858032C4.491158-6.011457 4.407472-6.011457 4.128518-6.011457H2.901121L3.417186-8.047821C3.472976-8.243088 3.472976-8.270984 3.472976-8.368618C3.472976-8.591781 3.291656-8.71731 3.110336-8.71731C2.998755-8.71731 2.677958-8.675467 2.566376-8.229141L2.022416-6.011457H.711333C.432379-6.011457 .306849-6.011457 .306849-5.746451C.306849-5.606974 .404483-5.606974 .669489-5.606974H1.910834L.990286-1.924782C.878705-1.436613 .836862-1.297136 .836862-1.115816C.836862-.460274 1.297136 .139477 2.078207 .139477C3.486924 .139477 4.2401-1.896887 4.2401-1.994521C4.2401-2.078207 4.184309-2.12005 4.100623-2.12005C4.072727-2.12005 4.016936-2.12005 3.989041-2.064259C3.975093-2.050311 3.961146-2.036364 3.863512-1.8132C3.57061-1.115816 2.929016-.139477 2.12005-.139477C1.701619-.139477 1.673724-.488169 1.673724-.795019C1.673724-.808966 1.673724-1.073973 1.715567-1.241345L2.803487-5.606974Z'/>
|
||||
<path id='g0-126' d='M3.556663-7.671233H6.262516C6.499626-7.671233 6.653051-7.671233 6.653051-7.838605C6.653051-8.019925 6.485679-8.019925 6.220672-8.019925H3.640349C3.737983-8.368618 3.737983-8.396513 3.821669-8.745205C3.891407-8.996264 4.016936-9.484433 4.016936-9.512329C4.016936-9.637858 3.93325-9.679701 3.821669-9.679701C3.556663-9.679701 2.454795-9.56812 2.12005-9.540224C2.008468-9.526276 1.855044-9.512329 1.855044-9.247323C1.855044-9.093898 1.994521-9.093898 2.189788-9.093898C2.859278-9.093898 2.887173-8.996264 2.887173-8.856787C2.887173-8.759153 2.761644-8.298879 2.691905-8.019925H1.910834C1.63188-8.019925 1.478456-8.019925 1.478456-7.838605C1.478456-7.671233 1.603985-7.671233 1.868991-7.671233H2.608219L.822914-.557908C.767123-.348692 .767123-.320797 .767123-.237111C.767123 .069738 1.004234 .153425 1.171606 .153425C1.436613 .153425 1.63188-.041843 1.687671-.223163C1.715567-.292902 1.868991-.920548 1.952677-1.26924L2.273474-2.510585C2.315318-2.719801 2.454795-3.249813 2.496638-3.459029C2.566376-3.682192 2.66401-4.11457 2.677958-4.156413C2.789539-4.393524 3.542715-5.858032 4.937484-5.858032C5.551183-5.858032 5.69066-5.355915 5.69066-4.895641C5.69066-4.030884 5.049066-2.329265 4.798007-1.687671C4.714321-1.464508 4.644583-1.26924 4.644583-1.004234C4.644583-.278954 5.160648 .153425 5.788294 .153425C7.127273 .153425 7.615442-1.910834 7.615442-1.994521C7.615442-2.133998 7.489913-2.133998 7.44807-2.133998C7.308593-2.133998 7.308593-2.092154 7.238854-1.882939C6.834371-.460274 6.206725-.153425 5.830137-.153425C5.565131-.153425 5.481445-.320797 5.481445-.613699C5.481445-.962391 5.634869-1.352927 5.718555-1.590037C5.955666-2.203736 6.583313-3.87746 6.583313-4.658531C6.583313-5.676712 5.941719-6.164882 4.979328-6.164882C4.546949-6.164882 3.710087-6.067248 2.915068-5.104857L3.556663-7.671233Z'/>
|
||||
<path id='g3-40' d='M4.533001 3.38929C4.533001 3.347447 4.533001 3.319552 4.29589 3.082441C2.901121 1.673724 2.12005-.627646 2.12005-3.472976C2.12005-6.178829 2.775592-8.508095 4.393524-10.153923C4.533001-10.279452 4.533001-10.307347 4.533001-10.349191C4.533001-10.432877 4.463263-10.460772 4.407472-10.460772C4.226152-10.460772 3.082441-9.456538 2.399004-8.089664C1.687671-6.680946 1.366874-5.188543 1.366874-3.472976C1.366874-2.231631 1.562142-.571856 2.287422 .920548C3.110336 2.594271 4.254047 3.500872 4.407472 3.500872C4.463263 3.500872 4.533001 3.472976 4.533001 3.38929Z'/>
|
||||
<path id='g3-41' d='M3.93325-3.472976C3.93325-4.533001 3.793773-6.262516 3.012702-7.880448C2.189788-9.554172 1.046077-10.460772 .892653-10.460772C.836862-10.460772 .767123-10.432877 .767123-10.349191C.767123-10.307347 .767123-10.279452 1.004234-10.042341C2.399004-8.633624 3.180075-6.332254 3.180075-3.486924C3.180075-.781071 2.524533 1.548194 .9066 3.194022C.767123 3.319552 .767123 3.347447 .767123 3.38929C.767123 3.472976 .836862 3.500872 .892653 3.500872C1.073973 3.500872 2.217684 2.496638 2.901121 1.129763C3.612453-.292902 3.93325-1.799253 3.93325-3.472976Z'/>
|
||||
<path id='g3-61' d='M9.414695-4.519054C9.609963-4.519054 9.861021-4.519054 9.861021-4.770112C9.861021-5.035118 9.62391-5.035118 9.414695-5.035118H1.199502C1.004234-5.035118 .753176-5.035118 .753176-4.78406C.753176-4.519054 .990286-4.519054 1.199502-4.519054H9.414695ZM9.414695-1.924782C9.609963-1.924782 9.861021-1.924782 9.861021-2.175841C9.861021-2.440847 9.62391-2.440847 9.414695-2.440847H1.199502C1.004234-2.440847 .753176-2.440847 .753176-2.189788C.753176-1.924782 .990286-1.924782 1.199502-1.924782H9.414695Z'/>
|
||||
<path id='g3-94' d='M3.417186-9.679701L1.590037-7.782814L1.8132-7.573599L3.403238-9.010212L5.007223-7.573599L5.230386-7.782814L3.417186-9.679701Z'/>
|
||||
<path id='g3-105' d='M2.426899-8.591781C2.426899-8.954421 2.133998-9.275218 1.743462-9.275218C1.380822-9.275218 1.073973-8.982316 1.073973-8.605729C1.073973-8.187298 1.408717-7.922291 1.743462-7.922291C2.175841-7.922291 2.426899-8.284932 2.426899-8.591781ZM.502117-5.997509V-5.593026C1.39477-5.593026 1.520299-5.50934 1.520299-4.825903V-1.03213C1.520299-.404483 1.366874-.404483 .460274-.404483V0C.850809-.027895 1.520299-.027895 1.924782-.027895C2.078207-.027895 2.887173-.027895 3.361395 0V-.404483C2.454795-.404483 2.399004-.474222 2.399004-1.018182V-6.150934L.502117-5.997509Z'/>
|
||||
</defs>
|
||||
<g id='page1'>
|
||||
<use x='165.981071' y='97.429622' xlink:href='#g3-105'/>
|
||||
<use x='169.774676' y='97.429622' xlink:href='#g0-126'/>
|
||||
<use x='180.971707' y='87.993906' xlink:href='#g2-64'/>
|
||||
<rect x='178.505856' y='93.663745' height='.55789' width='12.899852'/>
|
||||
<use x='178.505856' y='106.997046' xlink:href='#g2-64'/>
|
||||
<use x='186.474036' y='106.997046' xlink:href='#g2-116'/>
|
||||
<use x='192.601222' y='97.429622' xlink:href='#g1-106'/>
|
||||
<use x='198.80019' y='97.429622' xlink:href='#g2-32'/>
|
||||
<use x='208.179277' y='97.429622' xlink:href='#g3-40'/>
|
||||
<use x='213.490324' y='97.429622' xlink:href='#g2-116'/>
|
||||
<use x='218.42201' y='97.429622' xlink:href='#g3-41'/>
|
||||
<use x='223.733057' y='97.429622' xlink:href='#g1-105'/>
|
||||
<use x='233.031506' y='97.429622' xlink:href='#g3-61'/>
|
||||
<use x='251.068923' y='93.903964' xlink:href='#g3-94'/>
|
||||
<use x='247.527926' y='97.429622' xlink:href='#g2-72'/>
|
||||
<use x='259.920966' y='97.429622' xlink:href='#g1-106'/>
|
||||
<use x='266.119934' y='97.429622' xlink:href='#g2-32'/>
|
||||
<use x='275.499021' y='97.429622' xlink:href='#g3-40'/>
|
||||
<use x='280.810068' y='97.429622' xlink:href='#g2-116'/>
|
||||
<use x='285.741754' y='97.429622' xlink:href='#g3-41'/>
|
||||
<use x='291.052801' y='97.429622' xlink:href='#g1-105'/>
|
||||
<use x='296.476924' y='97.429622' xlink:href='#g2-58'/>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 12 KiB |
@ -0,0 +1,10 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
|
||||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
version="1.1" baseProfile="full"
|
||||
width="70px" height="40px" viewBox="0 0 700 400">
|
||||
<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
|
||||
<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
|
||||
<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
|
||||
<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
|
||||
</svg>
|
After Width: | Height: | Size: 580 B |
After Width: | Height: | Size: 18 KiB |
@ -0,0 +1,129 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Overview: module code — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/tabs.css" />
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">Overview: module code</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>All modules for which code is available</h1>
|
||||
<ul><li><a href="searx/babel_extract.html">searx.babel_extract</a></li>
|
||||
<li><a href="searx/engines.html">searx.engines</a></li>
|
||||
<ul><li><a href="searx/engines/demo_offline.html">searx.engines.demo_offline</a></li>
|
||||
<li><a href="searx/engines/demo_online.html">searx.engines.demo_online</a></li>
|
||||
<li><a href="searx/engines/google.html">searx.engines.google</a></li>
|
||||
<li><a href="searx/engines/google_images.html">searx.engines.google_images</a></li>
|
||||
<li><a href="searx/engines/google_news.html">searx.engines.google_news</a></li>
|
||||
<li><a href="searx/engines/google_videos.html">searx.engines.google_videos</a></li>
|
||||
<li><a href="searx/engines/tineye.html">searx.engines.tineye</a></li>
|
||||
<li><a href="searx/engines/xpath.html">searx.engines.xpath</a></li>
|
||||
<li><a href="searx/engines/yahoo.html">searx.engines.yahoo</a></li>
|
||||
</ul><li><a href="searx/infopage.html">searx.infopage</a></li>
|
||||
<li><a href="searx/locales.html">searx.locales</a></li>
|
||||
<li><a href="searx/redislib.html">searx.redislib</a></li>
|
||||
<li><a href="searx/search.html">searx.search</a></li>
|
||||
<ul><li><a href="searx/search/models.html">searx.search.models</a></li>
|
||||
</ul><li><a href="searx/utils.html">searx.utils</a></li>
|
||||
<li><a href="searxng_extra/standalone_searx.html">searxng_extra.standalone_searx</a></li>
|
||||
<li><a href="searxng_extra/update/update_engine_descriptions.html">searxng_extra.update.update_engine_descriptions</a></li>
|
||||
<li><a href="searxng_extra/update/update_external_bangs.html">searxng_extra.update.update_external_bangs</a></li>
|
||||
<li><a href="searxng_extra/update/update_languages.html">searxng_extra.update.update_languages</a></li>
|
||||
</ul>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../index.html">
|
||||
<img class="logo" src="../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../index.html">Overview</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,166 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.babel_extract — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.babel_extract</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.babel_extract</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This module implements the :origin:`searxng_msg <babel.cfg>` extractor to</span>
|
||||
<span class="sd">extract messages from:</span>
|
||||
|
||||
<span class="sd">- :origin:`searx/searxng.msg`</span>
|
||||
|
||||
<span class="sd">The ``searxng.msg`` files are selected by Babel_, see Babel's configuration in</span>
|
||||
<span class="sd">:origin:`babel.cfg`::</span>
|
||||
|
||||
<span class="sd"> searxng_msg = searx.babel_extract.extract</span>
|
||||
<span class="sd"> ...</span>
|
||||
<span class="sd"> [searxng_msg: **/searxng.msg]</span>
|
||||
|
||||
<span class="sd">A ``searxng.msg`` file is a python file that is *executed* by the</span>
|
||||
<span class="sd">:py:obj:`extract` function. Additional ``searxng.msg`` files can be added by:</span>
|
||||
|
||||
<span class="sd">1. Adding a ``searxng.msg`` file in one of the SearXNG python packages and</span>
|
||||
<span class="sd">2. implement a method in :py:obj:`extract` that yields messages from this file.</span>
|
||||
|
||||
<span class="sd">.. _Babel: https://babel.pocoo.org/en/latest/index.html</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">path</span>
|
||||
|
||||
<span class="n">SEARXNG_MSG_FILE</span> <span class="o">=</span> <span class="s2">"searxng.msg"</span>
|
||||
<span class="n">_MSG_FILES</span> <span class="o">=</span> <span class="p">[</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">),</span> <span class="n">SEARXNG_MSG_FILE</span><span class="p">)]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract"><a class="viewcode-back" href="../../src/searx.babel_extract.html#searx.babel_extract.extract">[docs]</a><span class="k">def</span> <span class="nf">extract</span><span class="p">(</span>
|
||||
<span class="c1"># pylint: disable=unused-argument</span>
|
||||
<span class="n">fileobj</span><span class="p">,</span>
|
||||
<span class="n">keywords</span><span class="p">,</span>
|
||||
<span class="n">comment_tags</span><span class="p">,</span>
|
||||
<span class="n">options</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Extract messages from ``searxng.msg`` files by a custom extractor_.</span>
|
||||
|
||||
<span class="sd"> .. _extractor:</span>
|
||||
<span class="sd"> https://babel.pocoo.org/en/latest/messages.html#writing-extraction-methods</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_MSG_FILES</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"don't know how to extract messages from </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||
|
||||
<span class="n">namespace</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">exec</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span> <span class="p">{},</span> <span class="n">namespace</span><span class="p">)</span> <span class="c1"># pylint: disable=exec-used</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">namespace</span><span class="p">[</span><span class="s1">'__all__'</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">namespace</span><span class="p">[</span><span class="n">name</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">yield</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="p">[</span><span class="s2">"</span><span class="si">%s</span><span class="s2">['</span><span class="si">%s</span><span class="s2">']"</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">k</span><span class="p">)]</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,423 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This module implements the engine loader.</span>
|
||||
|
||||
<span class="sd">Load and initialize the ``engines``, see :py:func:`load_engines` and register</span>
|
||||
<span class="sd">:py:obj:`engine_shortcuts`.</span>
|
||||
|
||||
<span class="sd">usage::</span>
|
||||
|
||||
<span class="sd"> load_engines( settings['engines'] )</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">copy</span>
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">realpath</span><span class="p">,</span> <span class="n">dirname</span>
|
||||
<span class="kn">from</span> <span class="nn">babel.localedata</span> <span class="kn">import</span> <span class="n">locale_identifiers</span>
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span><span class="p">,</span> <span class="n">settings</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.data</span> <span class="kn">import</span> <span class="n">ENGINES_LANGUAGES</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.network</span> <span class="kn">import</span> <span class="n">get</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">load_module</span><span class="p">,</span> <span class="n">match_language</span><span class="p">,</span> <span class="n">gen_useragent</span>
|
||||
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'engines'</span><span class="p">)</span>
|
||||
<span class="n">ENGINE_DIR</span> <span class="o">=</span> <span class="n">dirname</span><span class="p">(</span><span class="n">realpath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
|
||||
<span class="n">BABEL_LANGS</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">lang_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">lang_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">lang_parts</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span> <span class="k">else</span> <span class="n">lang_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">lang_parts</span> <span class="ow">in</span> <span class="p">(</span><span class="n">lang_code</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span> <span class="k">for</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="n">locale_identifiers</span><span class="p">())</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">ENGINE_DEFAULT_ARGS</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"engine_type"</span><span class="p">:</span> <span class="s2">"online"</span><span class="p">,</span>
|
||||
<span class="s2">"inactive"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"disabled"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">settings</span><span class="p">[</span><span class="s2">"outgoing"</span><span class="p">][</span><span class="s2">"request_timeout"</span><span class="p">],</span>
|
||||
<span class="s2">"shortcut"</span><span class="p">:</span> <span class="s2">"-"</span><span class="p">,</span>
|
||||
<span class="s2">"categories"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"general"</span><span class="p">],</span>
|
||||
<span class="s2">"supported_languages"</span><span class="p">:</span> <span class="p">[],</span>
|
||||
<span class="s2">"language_aliases"</span><span class="p">:</span> <span class="p">{},</span>
|
||||
<span class="s2">"paging"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"safesearch"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"time_range_support"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"enable_http"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"using_tor_proxy"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"display_error_messages"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||
<span class="s2">"send_accept_language_header"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"tokens"</span><span class="p">:</span> <span class="p">[],</span>
|
||||
<span class="s2">"about"</span><span class="p">:</span> <span class="p">{},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="c1"># set automatically when an engine does not have any tab category</span>
|
||||
<span class="n">OTHER_CATEGORY</span> <span class="o">=</span> <span class="s1">'other'</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="Engine"><a class="viewcode-back" href="../../src/searx.engines.html#searx.engines.Engine">[docs]</a><span class="k">class</span> <span class="nc">Engine</span><span class="p">:</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||
<span class="w"> </span><span class="sd">"""This class is currently never initialized and only used for type hinting."""</span>
|
||||
|
||||
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">shortcut</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">categories</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">supported_languages</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span>
|
||||
<span class="n">inactive</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">disabled</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">language_support</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">safesearch</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">time_range_support</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span></div>
|
||||
|
||||
|
||||
<span class="c1"># Defaults for the namespace of an engine module, see :py:func:`load_engine`</span>
|
||||
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'general'</span><span class="p">:</span> <span class="p">[]}</span>
|
||||
<span class="n">engines</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Engine</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">engine_shortcuts</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="sd">"""Simple map of registered *shortcuts* to name of the engine (or ``None``).</span>
|
||||
|
||||
<span class="sd">::</span>
|
||||
|
||||
<span class="sd"> engine_shortcuts[engine.shortcut] = engine.name</span>
|
||||
|
||||
<span class="sd">:meta hide-value:</span>
|
||||
<span class="sd">"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="load_engine"><a class="viewcode-back" href="../../src/searx.engines.html#searx.engines.load_engine">[docs]</a><span class="k">def</span> <span class="nf">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Engine</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Load engine from ``engine_data``.</span>
|
||||
|
||||
<span class="sd"> :param dict engine_data: Attributes from YAML ``settings:engines/<engine>``</span>
|
||||
<span class="sd"> :return: initialized namespace of the ``<engine>``.</span>
|
||||
|
||||
<span class="sd"> 1. create a namespace and load module of the ``<engine>``</span>
|
||||
<span class="sd"> 2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`</span>
|
||||
<span class="sd"> 3. update namespace with values from ``engine_data``</span>
|
||||
|
||||
<span class="sd"> If engine *is active*, return namespace of the engine, otherwise return</span>
|
||||
<span class="sd"> ``None``.</span>
|
||||
|
||||
<span class="sd"> This function also returns ``None`` if initialization of the namespace fails</span>
|
||||
<span class="sd"> for one of the following reasons:</span>
|
||||
|
||||
<span class="sd"> - engine name contains underscore</span>
|
||||
<span class="sd"> - engine name is not lowercase</span>
|
||||
<span class="sd"> - required attribute is not set :py:func:`is_missing_required_attributes`</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># pylint: disable=too-many-return-statements</span>
|
||||
|
||||
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'An engine does not have a "name" field'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s1">'_'</span> <span class="ow">in</span> <span class="n">engine_name</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine name contains underscore: "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="n">engine_name</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'Engine name is not lowercase: "</span><span class="si">{}</span><span class="s1">", converting to lowercase'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">engine_data</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_name</span>
|
||||
|
||||
<span class="c1"># load_module</span>
|
||||
<span class="n">engine_module</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'engine'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine_module</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'The "engine" field is missing for the engine named "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_module</span><span class="p">(</span><span class="n">engine_module</span> <span class="o">+</span> <span class="s1">'.py'</span><span class="p">,</span> <span class="n">ENGINE_DIR</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="ne">SyntaxError</span><span class="p">,</span> <span class="ne">KeyboardInterrupt</span><span class="p">,</span> <span class="ne">SystemExit</span><span class="p">,</span> <span class="ne">SystemError</span><span class="p">,</span> <span class="ne">ImportError</span><span class="p">,</span> <span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Fatal exception in engine "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_module</span><span class="p">))</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">BaseException</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Cannot load engine "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_module</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">)</span>
|
||||
<span class="n">set_language_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||
<span class="n">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">cat</span> <span class="ow">in</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'categories_as_tabs'</span><span class="p">]</span> <span class="k">for</span> <span class="n">cat</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">):</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">OTHER_CATEGORY</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">engine</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">):</span>
|
||||
<span class="c1"># set the logger for engine</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||
<span class="c1"># the engine may have load some other engines</span>
|
||||
<span class="c1"># may sure the logger is initialized</span>
|
||||
<span class="c1"># use sys.modules.copy() to avoid "RuntimeError: dictionary changed size during iteration"</span>
|
||||
<span class="c1"># see https://github.com/python/cpython/issues/89516</span>
|
||||
<span class="c1"># and https://docs.python.org/3.10/library/sys.html#sys.modules</span>
|
||||
<span class="n">modules</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="k">for</span> <span class="n">module_name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="p">(</span>
|
||||
<span class="n">module_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"searx.engines"</span><span class="p">)</span>
|
||||
<span class="ow">and</span> <span class="n">module_name</span> <span class="o">!=</span> <span class="s2">"searx.engines.__init__"</span>
|
||||
<span class="ow">and</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="s2">"logger"</span><span class="p">)</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">module_engine_name</span> <span class="o">=</span> <span class="n">module_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="n">module</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">module_engine_name</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">):</span>
|
||||
<span class="c1"># set engine attributes from engine_data</span>
|
||||
<span class="k">for</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span> <span class="ow">in</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'categories'</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="n">param_value</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">strip</span><span class="p">,</span> <span class="n">param_value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)))</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="o">=</span> <span class="n">param_value</span>
|
||||
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'about'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'about'</span><span class="p">:</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">about</span> <span class="o">=</span> <span class="p">{</span><span class="o">**</span><span class="n">engine</span><span class="o">.</span><span class="n">about</span><span class="p">,</span> <span class="o">**</span><span class="n">engine_data</span><span class="p">[</span><span class="s1">'about'</span><span class="p">]}</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># set default attributes</span>
|
||||
<span class="k">for</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">arg_value</span> <span class="ow">in</span> <span class="n">ENGINE_DEFAULT_ARGS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">):</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">arg_value</span><span class="p">))</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">set_language_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">):</span>
|
||||
<span class="c1"># assign supported languages from json file</span>
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="n">ENGINES_LANGUAGES</span><span class="p">:</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span> <span class="o">=</span> <span class="n">ENGINES_LANGUAGES</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">engine</span><span class="o">.</span><span class="n">engine</span> <span class="ow">in</span> <span class="n">ENGINES_LANGUAGES</span><span class="p">:</span>
|
||||
<span class="c1"># The key of the dictionary ENGINES_LANGUAGES is the *engine name*</span>
|
||||
<span class="c1"># configured in settings.xml. When multiple engines are configured in</span>
|
||||
<span class="c1"># settings.yml to use the same origin engine (python module) these</span>
|
||||
<span class="c1"># additional engines can use the languages from the origin engine.</span>
|
||||
<span class="c1"># For this use the configured ``engine: ...`` from settings.yml</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span> <span class="o">=</span> <span class="n">ENGINES_LANGUAGES</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">engine</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'language'</span><span class="p">):</span>
|
||||
<span class="c1"># For an engine, when there is `language: ...` in the YAML settings, the</span>
|
||||
<span class="c1"># engine supports only one language, in this case</span>
|
||||
<span class="c1"># engine.supported_languages should contains this value defined in</span>
|
||||
<span class="c1"># settings.yml</span>
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="s2">"settings.yml - engine: '</span><span class="si">%s</span><span class="s2">' / language: '</span><span class="si">%s</span><span class="s2">' not supported"</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span> <span class="o">=</span> <span class="p">{</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">:</span> <span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">]}</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span> <span class="o">=</span> <span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># find custom aliases for non standard language codes</span>
|
||||
<span class="k">for</span> <span class="n">engine_lang</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span><span class="p">:</span>
|
||||
<span class="n">iso_lang</span> <span class="o">=</span> <span class="n">match_language</span><span class="p">(</span><span class="n">engine_lang</span><span class="p">,</span> <span class="n">BABEL_LANGS</span><span class="p">,</span> <span class="n">fallback</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="p">(</span>
|
||||
<span class="n">iso_lang</span>
|
||||
<span class="ow">and</span> <span class="n">iso_lang</span> <span class="o">!=</span> <span class="n">engine_lang</span>
|
||||
<span class="ow">and</span> <span class="ow">not</span> <span class="n">engine_lang</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">iso_lang</span><span class="p">)</span>
|
||||
<span class="ow">and</span> <span class="n">iso_lang</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">language_aliases</span><span class="p">[</span><span class="n">iso_lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_lang</span>
|
||||
|
||||
<span class="c1"># language_support</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">language_support</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">supported_languages</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span>
|
||||
|
||||
<span class="c1"># assign language fetching method if auxiliary method exists</span>
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'_fetch_supported_languages'</span><span class="p">):</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'User-Agent'</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
|
||||
<span class="s1">'Accept-Language'</span><span class="p">:</span> <span class="s2">"en-US,en;q=0.5"</span><span class="p">,</span> <span class="c1"># bing needs to set the English language</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">fetch_supported_languages</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="c1"># pylint: disable=protected-access</span>
|
||||
<span class="k">lambda</span><span class="p">:</span> <span class="n">engine</span><span class="o">.</span><span class="n">_fetch_supported_languages</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">supported_languages_url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">))</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'onion_url'</span><span class="p">):</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">search_url</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">onion_url</span> <span class="o">+</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'search_path'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="n">engine</span><span class="o">.</span><span class="n">timeout</span> <span class="o">+=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'extra_proxy_timeout'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_missing_required_attributes"><a class="viewcode-back" href="../../src/searx.engines.html#searx.engines.is_missing_required_attributes">[docs]</a><span class="k">def</span> <span class="nf">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""An attribute is required when its name doesn't start with ``_`` (underline).</span>
|
||||
<span class="sd"> Required attributes must not be ``None``.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">missing</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">for</span> <span class="n">engine_attr</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_attr</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Missing engine config attribute: "</span><span class="si">{0}</span><span class="s1">.</span><span class="si">{1}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">))</span>
|
||||
<span class="n">missing</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="n">missing</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="using_tor_proxy"><a class="viewcode-back" href="../../src/searx.engines.html#searx.engines.using_tor_proxy">[docs]</a><span class="k">def</span> <span class="nf">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Return True if the engine configuration declares to use Tor."""</span>
|
||||
<span class="k">return</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'using_tor_proxy'</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'using_tor_proxy'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">):</span>
|
||||
<span class="c1"># check if engine is inactive</span>
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">inactive</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
|
||||
<span class="c1"># exclude onion engines if not using tor</span>
|
||||
<span class="k">if</span> <span class="s1">'onions'</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="n">engines</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine config error: ambiguous name: </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">engines</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span> <span class="ow">in</span> <span class="n">engine_shortcuts</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine config error: ambiguous shortcut: </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">))</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">engine_shortcuts</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">category_name</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">:</span>
|
||||
<span class="n">categories</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">category_name</span><span class="p">,</span> <span class="p">[])</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="load_engines"><a class="viewcode-back" href="../../src/searx.engines.html#searx.engines.load_engines">[docs]</a><span class="k">def</span> <span class="nf">load_engines</span><span class="p">(</span><span class="n">engine_list</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""usage: ``engine_list = settings['engines']``"""</span>
|
||||
<span class="n">engines</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||
<span class="n">engine_shortcuts</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||
<span class="n">categories</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||
<span class="n">categories</span><span class="p">[</span><span class="s1">'general'</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">engine_data</span> <span class="ow">in</span> <span class="n">engine_list</span><span class="p">:</span>
|
||||
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine</span><span class="p">:</span>
|
||||
<span class="n">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">engines</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,192 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.demo_offline — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_offline</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.demo_offline</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""Within this module we implement a *demo offline engine*. Do not look to</span>
|
||||
<span class="sd">close to the implementation, its just a simple example. To get in use of this</span>
|
||||
<span class="sd">*demo* engine add the following entry to your engines list in ``settings.yml``:</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> - name: my offline engine</span>
|
||||
<span class="sd"> engine: demo_offline</span>
|
||||
<span class="sd"> shortcut: demo</span>
|
||||
<span class="sd"> disabled: false</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
|
||||
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">'offline'</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">]</span>
|
||||
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
|
||||
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># if there is a need for globals, use a leading underline</span>
|
||||
<span class="n">_my_offline_engine</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="init"><a class="viewcode-back" href="../../../src/searx.engines.demo_offline.html#searx.engines.demo_offline.init">[docs]</a><span class="k">def</span> <span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Initialization of the (offline) engine. The origin of this demo engine is a</span>
|
||||
<span class="sd"> simple json string which is loaded in this example while the engine is</span>
|
||||
<span class="sd"> initialized.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">global</span> <span class="n">_my_offline_engine</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||
|
||||
<span class="n">_my_offline_engine</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'[ {"value": "</span><span class="si">%s</span><span class="s1">"}'</span>
|
||||
<span class="s1">', {"value":"first item"}'</span>
|
||||
<span class="s1">', {"value":"second item"}'</span>
|
||||
<span class="s1">', {"value":"third item"}'</span>
|
||||
<span class="s1">']'</span> <span class="o">%</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="search"><a class="viewcode-back" href="../../../src/searx.engines.demo_offline.html#searx.engines.demo_offline.search">[docs]</a><span class="k">def</span> <span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">request_params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Query (offline) engine and return results. Assemble the list of results from</span>
|
||||
<span class="sd"> your local engine. In this demo engine we ignore the 'query' term, usual</span>
|
||||
<span class="sd"> you would pass the 'query' term to your local engine to filter out the</span>
|
||||
<span class="sd"> results.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">ret_val</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">result_list</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">_my_offline_engine</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">result_list</span><span class="p">:</span>
|
||||
<span class="n">entry</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||
<span class="s1">'language'</span><span class="p">:</span> <span class="n">request_params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">],</span>
|
||||
<span class="s1">'value'</span><span class="p">:</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">),</span>
|
||||
<span class="c1"># choose a result template or comment out to use the *default*</span>
|
||||
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'key-value.html'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">ret_val</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">entry</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,219 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.demo_online — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_online</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.demo_online</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""Within this module we implement a *demo online engine*. Do not look to</span>
|
||||
<span class="sd">close to the implementation, its just a simple example which queries `The Art</span>
|
||||
<span class="sd">Institute of Chicago <https://www.artic.edu>`_</span>
|
||||
|
||||
<span class="sd">To get in use of this *demo* engine add the following entry to your engines</span>
|
||||
<span class="sd">list in ``settings.yml``:</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> - name: my online engine</span>
|
||||
<span class="sd"> engine: demo_online</span>
|
||||
<span class="sd"> shortcut: demo</span>
|
||||
<span class="sd"> disabled: false</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">json</span> <span class="kn">import</span> <span class="n">loads</span>
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
|
||||
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">'online'</span>
|
||||
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">]</span>
|
||||
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'images'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">20</span>
|
||||
|
||||
<span class="n">search_api</span> <span class="o">=</span> <span class="s1">'https://api.artic.edu/api/v1/artworks/search?'</span>
|
||||
<span class="n">image_api</span> <span class="o">=</span> <span class="s1">'https://www.artic.edu/iiif/2/'</span>
|
||||
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.artic.edu'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q239303'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'http://api.artic.edu/docs/'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
|
||||
<span class="c1"># if there is a need for globals, use a leading underline</span>
|
||||
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="init"><a class="viewcode-back" href="../../../src/searx.engines.demo_online.html#searx.engines.demo_online.init">[docs]</a><span class="k">def</span> <span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Initialization of the (online) engine. If no initialization is needed, drop</span>
|
||||
<span class="sd"> this init function.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">global</span> <span class="n">_my_online_engine</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.demo_online.html#searx.engines.demo_online.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Build up the ``params`` for the online request. In this example we build a</span>
|
||||
<span class="sd"> URL to fetch images from `artic.edu <https://artic.edu>`__</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||
<span class="s1">'page'</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">],</span>
|
||||
<span class="s1">'fields'</span><span class="p">:</span> <span class="s1">'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles'</span><span class="p">,</span>
|
||||
<span class="s1">'limit'</span><span class="p">:</span> <span class="n">page_size</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_api</span> <span class="o">+</span> <span class="n">args</span>
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.demo_online.html#searx.engines.demo_online.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Parse out the result items from the response. In this example we parse the</span>
|
||||
<span class="sd"> response from `api.artic.edu <https://artic.edu>`__ and filter out all</span>
|
||||
<span class="sd"> images.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">json_data</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]:</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="p">[</span><span class="s1">'image_id'</span><span class="p">]:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="s1">'https://artic.edu/artworks/</span><span class="si">%(id)s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span> <span class="o">+</span> <span class="s2">" (</span><span class="si">%(date_display)s</span><span class="s2">) // </span><span class="si">%(artist_display)s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||
<span class="s1">'content'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'medium_display'</span><span class="p">],</span>
|
||||
<span class="s1">'author'</span><span class="p">:</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'artist_titles'</span><span class="p">]),</span>
|
||||
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">image_api</span> <span class="o">+</span> <span class="s1">'/</span><span class="si">%(image_id)s</span><span class="s1">/full/843,/0/default.jpg'</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||
<span class="s1">'img_format'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'dimensions'</span><span class="p">],</span>
|
||||
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,494 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.google — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.google</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.google</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This is the implementation of the google WEB engine. Some of this</span>
|
||||
<span class="sd">implementations are shared by other engines:</span>
|
||||
|
||||
<span class="sd">- :ref:`google images engine`</span>
|
||||
<span class="sd">- :ref:`google news engine`</span>
|
||||
<span class="sd">- :ref:`google videos engine`</span>
|
||||
|
||||
<span class="sd">The google WEB engine itself has a special setup option:</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> - name: google</span>
|
||||
<span class="sd"> ...</span>
|
||||
<span class="sd"> use_mobile_ui: false</span>
|
||||
|
||||
<span class="sd">``use_mobile_ui``: (default: ``false``)</span>
|
||||
<span class="sd"> Enables to use *mobile endpoint* to bypass the google blocking (see</span>
|
||||
<span class="sd"> :issue:`159`). On the mobile UI of Google Search, the button :guilabel:`More</span>
|
||||
<span class="sd"> results` is not affected by Google rate limiting and we can still do requests</span>
|
||||
<span class="sd"> while actively blocked by the original Google search. By activate</span>
|
||||
<span class="sd"> ``use_mobile_ui`` this behavior is simulated by adding the parameter</span>
|
||||
<span class="sd"> ``async=use_ac:true,_fmt:pc`` to the :py:func:`request`.</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">match_language</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.exceptions</span> <span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
|
||||
|
||||
<span class="c1"># about</span>
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.google.com'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q9366'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search/'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># engine dependent config</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">use_mobile_ui</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">supported_languages_url</span> <span class="o">=</span> <span class="s1">'https://www.google.com/preferences?#languages'</span>
|
||||
|
||||
<span class="c1"># based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests</span>
|
||||
<span class="n">google_domains</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'BG'</span><span class="p">:</span> <span class="s1">'google.bg'</span><span class="p">,</span> <span class="c1"># Bulgaria</span>
|
||||
<span class="s1">'CZ'</span><span class="p">:</span> <span class="s1">'google.cz'</span><span class="p">,</span> <span class="c1"># Czech Republic</span>
|
||||
<span class="s1">'DE'</span><span class="p">:</span> <span class="s1">'google.de'</span><span class="p">,</span> <span class="c1"># Germany</span>
|
||||
<span class="s1">'DK'</span><span class="p">:</span> <span class="s1">'google.dk'</span><span class="p">,</span> <span class="c1"># Denmark</span>
|
||||
<span class="s1">'AT'</span><span class="p">:</span> <span class="s1">'google.at'</span><span class="p">,</span> <span class="c1"># Austria</span>
|
||||
<span class="s1">'CH'</span><span class="p">:</span> <span class="s1">'google.ch'</span><span class="p">,</span> <span class="c1"># Switzerland</span>
|
||||
<span class="s1">'GR'</span><span class="p">:</span> <span class="s1">'google.gr'</span><span class="p">,</span> <span class="c1"># Greece</span>
|
||||
<span class="s1">'AU'</span><span class="p">:</span> <span class="s1">'google.com.au'</span><span class="p">,</span> <span class="c1"># Australia</span>
|
||||
<span class="s1">'CA'</span><span class="p">:</span> <span class="s1">'google.ca'</span><span class="p">,</span> <span class="c1"># Canada</span>
|
||||
<span class="s1">'GB'</span><span class="p">:</span> <span class="s1">'google.co.uk'</span><span class="p">,</span> <span class="c1"># United Kingdom</span>
|
||||
<span class="s1">'ID'</span><span class="p">:</span> <span class="s1">'google.co.id'</span><span class="p">,</span> <span class="c1"># Indonesia</span>
|
||||
<span class="s1">'IE'</span><span class="p">:</span> <span class="s1">'google.ie'</span><span class="p">,</span> <span class="c1"># Ireland</span>
|
||||
<span class="s1">'IN'</span><span class="p">:</span> <span class="s1">'google.co.in'</span><span class="p">,</span> <span class="c1"># India</span>
|
||||
<span class="s1">'MY'</span><span class="p">:</span> <span class="s1">'google.com.my'</span><span class="p">,</span> <span class="c1"># Malaysia</span>
|
||||
<span class="s1">'NZ'</span><span class="p">:</span> <span class="s1">'google.co.nz'</span><span class="p">,</span> <span class="c1"># New Zealand</span>
|
||||
<span class="s1">'PH'</span><span class="p">:</span> <span class="s1">'google.com.ph'</span><span class="p">,</span> <span class="c1"># Philippines</span>
|
||||
<span class="s1">'SG'</span><span class="p">:</span> <span class="s1">'google.com.sg'</span><span class="p">,</span> <span class="c1"># Singapore</span>
|
||||
<span class="s1">'US'</span><span class="p">:</span> <span class="s1">'google.com'</span><span class="p">,</span> <span class="c1"># United States (google.us) redirects to .com</span>
|
||||
<span class="s1">'ZA'</span><span class="p">:</span> <span class="s1">'google.co.za'</span><span class="p">,</span> <span class="c1"># South Africa</span>
|
||||
<span class="s1">'AR'</span><span class="p">:</span> <span class="s1">'google.com.ar'</span><span class="p">,</span> <span class="c1"># Argentina</span>
|
||||
<span class="s1">'CL'</span><span class="p">:</span> <span class="s1">'google.cl'</span><span class="p">,</span> <span class="c1"># Chile</span>
|
||||
<span class="s1">'ES'</span><span class="p">:</span> <span class="s1">'google.es'</span><span class="p">,</span> <span class="c1"># Spain</span>
|
||||
<span class="s1">'MX'</span><span class="p">:</span> <span class="s1">'google.com.mx'</span><span class="p">,</span> <span class="c1"># Mexico</span>
|
||||
<span class="s1">'EE'</span><span class="p">:</span> <span class="s1">'google.ee'</span><span class="p">,</span> <span class="c1"># Estonia</span>
|
||||
<span class="s1">'FI'</span><span class="p">:</span> <span class="s1">'google.fi'</span><span class="p">,</span> <span class="c1"># Finland</span>
|
||||
<span class="s1">'BE'</span><span class="p">:</span> <span class="s1">'google.be'</span><span class="p">,</span> <span class="c1"># Belgium</span>
|
||||
<span class="s1">'FR'</span><span class="p">:</span> <span class="s1">'google.fr'</span><span class="p">,</span> <span class="c1"># France</span>
|
||||
<span class="s1">'IL'</span><span class="p">:</span> <span class="s1">'google.co.il'</span><span class="p">,</span> <span class="c1"># Israel</span>
|
||||
<span class="s1">'HR'</span><span class="p">:</span> <span class="s1">'google.hr'</span><span class="p">,</span> <span class="c1"># Croatia</span>
|
||||
<span class="s1">'HU'</span><span class="p">:</span> <span class="s1">'google.hu'</span><span class="p">,</span> <span class="c1"># Hungary</span>
|
||||
<span class="s1">'IT'</span><span class="p">:</span> <span class="s1">'google.it'</span><span class="p">,</span> <span class="c1"># Italy</span>
|
||||
<span class="s1">'JP'</span><span class="p">:</span> <span class="s1">'google.co.jp'</span><span class="p">,</span> <span class="c1"># Japan</span>
|
||||
<span class="s1">'KR'</span><span class="p">:</span> <span class="s1">'google.co.kr'</span><span class="p">,</span> <span class="c1"># South Korea</span>
|
||||
<span class="s1">'LT'</span><span class="p">:</span> <span class="s1">'google.lt'</span><span class="p">,</span> <span class="c1"># Lithuania</span>
|
||||
<span class="s1">'LV'</span><span class="p">:</span> <span class="s1">'google.lv'</span><span class="p">,</span> <span class="c1"># Latvia</span>
|
||||
<span class="s1">'NO'</span><span class="p">:</span> <span class="s1">'google.no'</span><span class="p">,</span> <span class="c1"># Norway</span>
|
||||
<span class="s1">'NL'</span><span class="p">:</span> <span class="s1">'google.nl'</span><span class="p">,</span> <span class="c1"># Netherlands</span>
|
||||
<span class="s1">'PL'</span><span class="p">:</span> <span class="s1">'google.pl'</span><span class="p">,</span> <span class="c1"># Poland</span>
|
||||
<span class="s1">'BR'</span><span class="p">:</span> <span class="s1">'google.com.br'</span><span class="p">,</span> <span class="c1"># Brazil</span>
|
||||
<span class="s1">'PT'</span><span class="p">:</span> <span class="s1">'google.pt'</span><span class="p">,</span> <span class="c1"># Portugal</span>
|
||||
<span class="s1">'RO'</span><span class="p">:</span> <span class="s1">'google.ro'</span><span class="p">,</span> <span class="c1"># Romania</span>
|
||||
<span class="s1">'RU'</span><span class="p">:</span> <span class="s1">'google.ru'</span><span class="p">,</span> <span class="c1"># Russia</span>
|
||||
<span class="s1">'SK'</span><span class="p">:</span> <span class="s1">'google.sk'</span><span class="p">,</span> <span class="c1"># Slovakia</span>
|
||||
<span class="s1">'SI'</span><span class="p">:</span> <span class="s1">'google.si'</span><span class="p">,</span> <span class="c1"># Slovenia</span>
|
||||
<span class="s1">'SE'</span><span class="p">:</span> <span class="s1">'google.se'</span><span class="p">,</span> <span class="c1"># Sweden</span>
|
||||
<span class="s1">'TH'</span><span class="p">:</span> <span class="s1">'google.co.th'</span><span class="p">,</span> <span class="c1"># Thailand</span>
|
||||
<span class="s1">'TR'</span><span class="p">:</span> <span class="s1">'google.com.tr'</span><span class="p">,</span> <span class="c1"># Turkey</span>
|
||||
<span class="s1">'UA'</span><span class="p">:</span> <span class="s1">'google.com.ua'</span><span class="p">,</span> <span class="c1"># Ukraine</span>
|
||||
<span class="s1">'CN'</span><span class="p">:</span> <span class="s1">'google.com.hk'</span><span class="p">,</span> <span class="c1"># There is no google.cn, we use .com.hk for zh-CN</span>
|
||||
<span class="s1">'HK'</span><span class="p">:</span> <span class="s1">'google.com.hk'</span><span class="p">,</span> <span class="c1"># Hong Kong</span>
|
||||
<span class="s1">'TW'</span><span class="p">:</span> <span class="s1">'google.com.tw'</span><span class="p">,</span> <span class="c1"># Taiwan</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'w'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'m'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">:</span> <span class="s1">'y'</span><span class="p">}</span>
|
||||
|
||||
<span class="c1"># Filter results. 0: None, 1: Moderate, 2: Strict</span>
|
||||
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'off'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'medium'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'high'</span><span class="p">}</span>
|
||||
|
||||
<span class="c1"># specific xpath variables</span>
|
||||
<span class="c1"># ------------------------</span>
|
||||
|
||||
<span class="n">results_xpath</span> <span class="o">=</span> <span class="s1">'.//div[@data-sokoban-container]'</span>
|
||||
<span class="n">title_xpath</span> <span class="o">=</span> <span class="s1">'.//a/h3[1]'</span>
|
||||
<span class="n">href_xpath</span> <span class="o">=</span> <span class="s1">'.//a[h3]/@href'</span>
|
||||
<span class="n">content_xpath</span> <span class="o">=</span> <span class="s1">'.//div[@data-content-feature=1]'</span>
|
||||
|
||||
<span class="c1"># google *sections* are no usual *results*, we ignore them</span>
|
||||
<span class="n">g_section_with_header</span> <span class="o">=</span> <span class="s1">'./g-section-with-header'</span>
|
||||
|
||||
|
||||
<span class="c1"># Suggestions are links placed in a *card-section*, we extract only the text</span>
|
||||
<span class="c1"># from the links not the links itself.</span>
|
||||
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">'//div[contains(@class, "EIaa9b")]//a'</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_lang_info"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google.get_lang_info">[docs]</a><span class="k">def</span> <span class="nf">get_lang_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">,</span> <span class="n">supported_any_language</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Composing various language properties for the google engines.</span>
|
||||
|
||||
<span class="sd"> This function is called by the various google engines (:ref:`google web</span>
|
||||
<span class="sd"> engine`, :ref:`google images engine`, :ref:`google news engine` and</span>
|
||||
<span class="sd"> :ref:`google videos engine`).</span>
|
||||
|
||||
<span class="sd"> :param dict param: request parameters of the engine</span>
|
||||
|
||||
<span class="sd"> :param list lang_list: list of supported languages of the engine</span>
|
||||
<span class="sd"> :py:obj:`ENGINES_LANGUAGES[engine-name] <searx.data.ENGINES_LANGUAGES>`</span>
|
||||
|
||||
<span class="sd"> :param dict lang_list: custom aliases for non standard language codes</span>
|
||||
<span class="sd"> (used when calling :py:func:`searx.utils.match_language`)</span>
|
||||
|
||||
<span class="sd"> :param bool supported_any_language: When a language is not specified, the</span>
|
||||
<span class="sd"> language interpretation is left up to Google to decide how the search</span>
|
||||
<span class="sd"> results should be delivered. This argument is ``True`` for the google</span>
|
||||
<span class="sd"> engine and ``False`` for the other engines (google-images, -news,</span>
|
||||
<span class="sd"> -scholar, -videos).</span>
|
||||
|
||||
<span class="sd"> :rtype: dict</span>
|
||||
<span class="sd"> :returns:</span>
|
||||
<span class="sd"> Py-Dictionary with the key/value pairs:</span>
|
||||
|
||||
<span class="sd"> language:</span>
|
||||
<span class="sd"> Return value from :py:func:`searx.utils.match_language`</span>
|
||||
|
||||
<span class="sd"> country:</span>
|
||||
<span class="sd"> The country code (e.g. US, AT, CA, FR, DE ..)</span>
|
||||
|
||||
<span class="sd"> subdomain:</span>
|
||||
<span class="sd"> Google subdomain :py:obj:`google_domains` that fits to the country</span>
|
||||
<span class="sd"> code.</span>
|
||||
|
||||
<span class="sd"> params:</span>
|
||||
<span class="sd"> Py-Dictionary with additional request arguments (can be passed to</span>
|
||||
<span class="sd"> :py:func:`urllib.parse.urlencode`).</span>
|
||||
|
||||
<span class="sd"> headers:</span>
|
||||
<span class="sd"> Py-Dictionary with additional HTTP headers (can be passed to</span>
|
||||
<span class="sd"> request's headers)</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">ret_val</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'language'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s1">'country'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s1">'subdomain'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s1">'params'</span><span class="p">:</span> <span class="p">{},</span>
|
||||
<span class="s1">'headers'</span><span class="p">:</span> <span class="p">{},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># language ...</span>
|
||||
|
||||
<span class="n">_lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span>
|
||||
<span class="n">_any_language</span> <span class="o">=</span> <span class="n">_lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'all'</span>
|
||||
<span class="k">if</span> <span class="n">_any_language</span><span class="p">:</span>
|
||||
<span class="n">_lang</span> <span class="o">=</span> <span class="s1">'en-US'</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">match_language</span><span class="p">(</span><span class="n">_lang</span><span class="p">,</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">language</span>
|
||||
|
||||
<span class="c1"># country ...</span>
|
||||
|
||||
<span class="n">_l</span> <span class="o">=</span> <span class="n">_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">_l</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
|
||||
<span class="n">country</span> <span class="o">=</span> <span class="n">_l</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">country</span> <span class="o">=</span> <span class="n">_l</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">country</span> <span class="o">==</span> <span class="s1">'EN'</span><span class="p">:</span>
|
||||
<span class="n">country</span> <span class="o">=</span> <span class="s1">'US'</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'country'</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
|
||||
|
||||
<span class="c1"># subdomain ...</span>
|
||||
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'www.'</span> <span class="o">+</span> <span class="n">google_domains</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">country</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="s1">'google.com'</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># params & headers</span>
|
||||
|
||||
<span class="n">lang_country</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">-</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">language</span><span class="p">,</span> <span class="n">country</span><span class="p">)</span> <span class="c1"># (en-US, en-EN, de-DE, de-AU, fr-FR ..)</span>
|
||||
|
||||
<span class="c1"># hl parameter:</span>
|
||||
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#hlsp The</span>
|
||||
<span class="c1"># Interface Language:</span>
|
||||
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages</span>
|
||||
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_list</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_country</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># lr parameter:</span>
|
||||
<span class="c1"># The lr (language restrict) parameter restricts search results to</span>
|
||||
<span class="c1"># documents written in a particular language.</span>
|
||||
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#lrsp</span>
|
||||
<span class="c1"># Language Collection Values:</span>
|
||||
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">_any_language</span> <span class="ow">and</span> <span class="n">supported_any_language</span><span class="p">:</span>
|
||||
|
||||
<span class="c1"># interpretation is left up to Google (based on whoogle)</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># - add parameter ``source=lnt``</span>
|
||||
<span class="c1"># - don't use parameter ``lr``</span>
|
||||
<span class="c1"># - don't add a ``Accept-Language`` HTTP header.</span>
|
||||
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'lnt'</span>
|
||||
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
|
||||
<span class="c1"># restricts search results to documents written in a particular</span>
|
||||
<span class="c1"># language.</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'lr'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"lang_"</span> <span class="o">+</span> <span class="n">lang_list</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_country</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">host</span> <span class="o">==</span> <span class="s1">'sorry.google.com'</span> <span class="ow">or</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'/sorry'</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Google search request"""</span>
|
||||
|
||||
<span class="n">offset</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
|
||||
|
||||
<span class="n">lang_info</span> <span class="o">=</span> <span class="n">get_lang_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">supported_languages</span><span class="p">,</span> <span class="n">language_aliases</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="n">additional_parameters</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">if</span> <span class="n">use_mobile_ui</span><span class="p">:</span>
|
||||
<span class="n">additional_parameters</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'asearch'</span><span class="p">:</span> <span class="s1">'arc'</span><span class="p">,</span>
|
||||
<span class="s1">'async'</span><span class="p">:</span> <span class="s1">'use_ac:true,_fmt:prog'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium</span>
|
||||
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'https://'</span>
|
||||
<span class="o">+</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||
<span class="o">+</span> <span class="s2">"?"</span>
|
||||
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||
<span class="o">**</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span>
|
||||
<span class="s1">'ie'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span>
|
||||
<span class="s1">'oe'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span>
|
||||
<span class="s1">'start'</span><span class="p">:</span> <span class="n">offset</span><span class="p">,</span>
|
||||
<span class="s1">'filter'</span><span class="p">:</span> <span class="s1">'0'</span><span class="p">,</span>
|
||||
<span class="o">**</span><span class="n">additional_parameters</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'CONSENT'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"YES+"</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">use_mobile_ui</span><span class="p">:</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'*/*'</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||
|
||||
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="c1"># convert the text to dom</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="c1"># results --> answer</span>
|
||||
<span class="n">answer_list</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "LGOjhe")]'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">answer_list</span><span class="p">:</span>
|
||||
<span class="n">answer_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">_</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s2">"normalize-space()"</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">answer_list</span><span class="p">]</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'answer'</span><span class="p">:</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">answer_list</span><span class="p">)})</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"did not find 'answer'"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># results --> number_of_results</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">use_mobile_ui</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">_txt</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[@id="result-stats"]//text()'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">_digit</span> <span class="o">=</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">n</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">_txt</span> <span class="k">if</span> <span class="n">n</span><span class="o">.</span><span class="n">isdigit</span><span class="p">()])</span>
|
||||
<span class="n">number_of_results</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">_digit</span><span class="p">)</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'number_of_results'</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">})</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"did not 'number_of_results'"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">exc_info</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># parse results</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">results_xpath</span><span class="p">):</span>
|
||||
|
||||
<span class="c1"># google *sections*</span>
|
||||
<span class="k">if</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">g_section_with_header</span><span class="p">)):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"ignoring <g-section-with-header>"</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">title_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">title_tag</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># this not one of the common google results *section*</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'ignoring item from the result_xpath list: missing title'</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title_tag</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">href_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">content</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'ignoring item from the result_xpath list: missing content of title "</span><span class="si">%s</span><span class="s1">"'</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'add link to results: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">exc_info</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="c1"># parse suggestion</span>
|
||||
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||
<span class="c1"># append suggestion</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||
|
||||
<span class="c1"># return results</span>
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<span class="c1"># get supported languages from their site</span>
|
||||
<span class="k">def</span> <span class="nf">_fetch_supported_languages</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="n">ret_val</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
|
||||
<span class="n">radio_buttons</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//*[@id="langSec"]//input[@name="lr"]'</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">radio_buttons</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"data-name"</span><span class="p">)</span>
|
||||
<span class="n">code</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="n">code</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="n">name</span><span class="p">}</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">ret_val</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,241 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.google_images — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.google_images</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.google_images</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This is the implementation of the google images engine using the google</span>
|
||||
<span class="sd">internal API used the Google Go Android app.</span>
|
||||
|
||||
<span class="sd">This internal API offer results in</span>
|
||||
|
||||
<span class="sd">- JSON (_fmt:json)</span>
|
||||
<span class="sd">- Protobuf (_fmt:pb)</span>
|
||||
<span class="sd">- Protobuf compressed? (_fmt:pc)</span>
|
||||
<span class="sd">- HTML (_fmt:html)</span>
|
||||
<span class="sd">- Protobuf encoded in JSON (_fmt:jspb).</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
<span class="kn">from</span> <span class="nn">json</span> <span class="kn">import</span> <span class="n">loads</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">get_lang_info</span><span class="p">,</span>
|
||||
<span class="n">time_range_dict</span><span class="p">,</span>
|
||||
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># pylint: disable=unused-import</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="n">supported_languages_url</span><span class="p">,</span> <span class="n">_fetch_supported_languages</span>
|
||||
|
||||
<span class="c1"># pylint: enable=unused-import</span>
|
||||
|
||||
<span class="c1"># about</span>
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://images.google.com'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q521550'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># engine dependent config</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'images'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">use_locale_domain</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'images'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'active'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'active'</span><span class="p">}</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_images.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Google-Image search request"""</span>
|
||||
|
||||
<span class="n">lang_info</span> <span class="o">=</span> <span class="n">get_lang_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">supported_languages</span><span class="p">,</span> <span class="n">language_aliases</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'https://'</span>
|
||||
<span class="o">+</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||
<span class="o">+</span> <span class="s2">"?"</span>
|
||||
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||
<span class="s1">'tbm'</span><span class="p">:</span> <span class="s2">"isch"</span><span class="p">,</span>
|
||||
<span class="o">**</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span>
|
||||
<span class="s1">'ie'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span>
|
||||
<span class="s1">'oe'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span>
|
||||
<span class="s1">'asearch'</span><span class="p">:</span> <span class="s1">'isch'</span><span class="p">,</span>
|
||||
<span class="s1">'async'</span><span class="p">:</span> <span class="s1">'_fmt:json,p:1,ijn:'</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]),</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'User-Agent'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12; US) gzip'</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'*/*'</span>
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_images.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||
|
||||
<span class="n">json_start</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'{"ischj":'</span><span class="p">)</span>
|
||||
<span class="n">json_data</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">json_start</span><span class="p">:])</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"ischj"</span><span class="p">][</span><span class="s2">"metadata"</span><span class="p">]:</span>
|
||||
|
||||
<span class="n">result_item</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"referrer_url"</span><span class="p">],</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"page_title"</span><span class="p">],</span>
|
||||
<span class="s1">'content'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"text_in_grid"</span><span class="p">][</span><span class="s2">"snippet"</span><span class="p">],</span>
|
||||
<span class="s1">'source'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"site_title"</span><span class="p">],</span>
|
||||
<span class="s1">'img_format'</span><span class="p">:</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"width"</span><span class="p">]</span><span class="si">}</span><span class="s1"> x </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"height"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span>
|
||||
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"url"</span><span class="p">],</span>
|
||||
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">][</span><span class="s2">"url"</span><span class="p">],</span>
|
||||
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">author</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'iptc'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'creator'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">author</span><span class="p">:</span>
|
||||
<span class="n">result_item</span><span class="p">[</span><span class="s1">'author'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">author</span><span class="p">)</span>
|
||||
|
||||
<span class="n">copyright_notice</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'iptc'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'copyright_notice'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">copyright_notice</span><span class="p">:</span>
|
||||
<span class="n">result_item</span><span class="p">[</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">' / '</span> <span class="o">+</span> <span class="n">copyright_notice</span>
|
||||
|
||||
<span class="n">file_size</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'gsa'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'file_size'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">file_size</span><span class="p">:</span>
|
||||
<span class="n">result_item</span><span class="p">[</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">' (</span><span class="si">%s</span><span class="s1">)'</span> <span class="o">%</span> <span class="n">file_size</span>
|
||||
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result_item</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,295 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.google_news — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.google_news</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.google_news</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This is the implementation of the google news engine. The google news API</span>
|
||||
<span class="sd">ignores some parameters from the common :ref:`google API`:</span>
|
||||
|
||||
<span class="sd">- num_ : the number of search results is ignored</span>
|
||||
<span class="sd">- save_ : is ignored / Google-News results are always *SafeSearch*</span>
|
||||
|
||||
<span class="sd">.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp</span>
|
||||
<span class="sd">.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="c1"># pylint: disable=invalid-name</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">binascii</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
<span class="kn">from</span> <span class="nn">base64</span> <span class="kn">import</span> <span class="n">b64decode</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||
<span class="n">extract_text</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># pylint: disable=unused-import</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">supported_languages_url</span><span class="p">,</span>
|
||||
<span class="n">_fetch_supported_languages</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># pylint: enable=unused-import</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">get_lang_info</span><span class="p">,</span>
|
||||
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># about</span>
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://news.google.com'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q12020'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># compared to other google engines google-news has a different time range</span>
|
||||
<span class="c1"># support. The time range is included in the search term.</span>
|
||||
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'day'</span><span class="p">:</span> <span class="s1">'when:1d'</span><span class="p">,</span>
|
||||
<span class="s1">'week'</span><span class="p">:</span> <span class="s1">'when:7d'</span><span class="p">,</span>
|
||||
<span class="s1">'month'</span><span class="p">:</span> <span class="s1">'when:1m'</span><span class="p">,</span>
|
||||
<span class="s1">'year'</span><span class="p">:</span> <span class="s1">'when:1y'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># engine dependent config</span>
|
||||
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'news'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">use_locale_domain</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="c1"># Google-News results are always *SafeSearch*. Option 'safesearch' is set to</span>
|
||||
<span class="c1"># False here, otherwise checker will report safesearch-errors::</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># safesearch : results are identitical for safesearch=0 and safesearch=2</span>
|
||||
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_news.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Google-News search request"""</span>
|
||||
|
||||
<span class="n">lang_info</span> <span class="o">=</span> <span class="n">get_lang_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">supported_languages</span><span class="p">,</span> <span class="n">language_aliases</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># google news has only one domain</span>
|
||||
<span class="n">lang_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'news.google.com'</span>
|
||||
|
||||
<span class="n">ceid</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'country'</span><span class="p">],</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'language'</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># google news redirects en to en-US</span>
|
||||
<span class="k">if</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'en'</span><span class="p">:</span>
|
||||
<span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'en-US'</span>
|
||||
|
||||
<span class="c1"># Very special to google-news compared to other google engines, the time</span>
|
||||
<span class="c1"># range is included in the search term.</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]:</span>
|
||||
<span class="n">query</span> <span class="o">+=</span> <span class="s1">' '</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'https://'</span>
|
||||
<span class="o">+</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||
<span class="o">+</span> <span class="s2">"?"</span>
|
||||
<span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="o">**</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span> <span class="s1">'ie'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span> <span class="s1">'oe'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span> <span class="s1">'gl'</span><span class="p">:</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'country'</span><span class="p">]})</span>
|
||||
<span class="o">+</span> <span class="p">(</span><span class="s1">'&ceid=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">ceid</span><span class="p">)</span>
|
||||
<span class="p">)</span> <span class="c1"># ceid includes a ':' character which must not be urlencoded</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'CONSENT'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"YES+"</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_news.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># convert the text to dom</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[@class="xrnccd"]'</span><span class="p">):</span>
|
||||
|
||||
<span class="c1"># The first <a> tag in the <article> contains the link to the</span>
|
||||
<span class="c1"># article The href attribute of the <a> is a google internal link,</span>
|
||||
<span class="c1"># we can't use. The real link is hidden in the jslog attribute:</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># <a ...</span>
|
||||
<span class="c1"># jslog="95014; 4:https://www.cnn.com/.../index.html; track:click"</span>
|
||||
<span class="c1"># href="./articles/CAIiENu3nGS...?hl=en-US&amp;gl=US&amp;ceid=US%3Aen"</span>
|
||||
<span class="c1"># ... /></span>
|
||||
|
||||
<span class="n">jslog</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/a/@jslog'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="s1">'http[^;]*'</span><span class="p">,</span> <span class="n">jslog</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">url</span><span class="p">:</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">url</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="c1"># The real URL is base64 encoded in the json attribute:</span>
|
||||
<span class="c1"># jslog="95014; 5:W251bGwsbnVsbCxudW...giXQ==; track:click"</span>
|
||||
<span class="n">jslog</span> <span class="o">=</span> <span class="n">jslog</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">";"</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">':'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">padding</span> <span class="o">=</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">jslog</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span><span class="p">))</span> <span class="o">*</span> <span class="s2">"="</span>
|
||||
<span class="n">jslog</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">jslog</span> <span class="o">+</span> <span class="n">padding</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">:</span>
|
||||
<span class="c1"># URL can't be read, skip this result</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="c1"># now we have : b'[null, ... null,"https://www.cnn.com/.../index.html"]'</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="s1">'http[^;"]*'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">jslog</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># the first <h3> tag in the <article> contains the title of the link</span>
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/h3[1]'</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># The pub_date is mostly a string like 'yesertday', not a real</span>
|
||||
<span class="c1"># timezone date or time. Therefore we can't use publishedDate.</span>
|
||||
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/div[1]/div[1]/time'</span><span class="p">))</span>
|
||||
<span class="n">pub_origin</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/div[1]/div[1]/a'</span><span class="p">))</span>
|
||||
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="s1">' / '</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">pub_origin</span><span class="p">,</span> <span class="n">pub_date</span><span class="p">]</span> <span class="k">if</span> <span class="n">x</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># The image URL is located in a preceding sibling <img> tag, e.g.:</span>
|
||||
<span class="c1"># "https://lh3.googleusercontent.com/DjhQh7DMszk.....z=-p-h100-w100"</span>
|
||||
<span class="c1"># These URL are long but not personalized (double checked via tor).</span>
|
||||
|
||||
<span class="n">img_src</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'preceding-sibling::a/figure/img/@src'</span><span class="p">))</span>
|
||||
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># return results</span>
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,309 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.google_videos — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.google_videos</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.google_videos</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This is the implementation of the google videos engine.</span>
|
||||
|
||||
<span class="sd">.. admonition:: Content-Security-Policy (CSP)</span>
|
||||
|
||||
<span class="sd"> This engine needs to allow images from the `data URLs`_ (prefixed with the</span>
|
||||
<span class="sd"> ``data:`` scheme)::</span>
|
||||
|
||||
<span class="sd"> Header set Content-Security-Policy "img-src 'self' data: ;"</span>
|
||||
|
||||
<span class="sd">.. _data URLs:</span>
|
||||
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="c1"># pylint: disable=invalid-name</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||
<span class="n">extract_text</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">get_lang_info</span><span class="p">,</span>
|
||||
<span class="n">time_range_dict</span><span class="p">,</span>
|
||||
<span class="n">filter_mapping</span><span class="p">,</span>
|
||||
<span class="n">g_section_with_header</span><span class="p">,</span>
|
||||
<span class="n">title_xpath</span><span class="p">,</span>
|
||||
<span class="n">suggestion_xpath</span><span class="p">,</span>
|
||||
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># pylint: disable=unused-import</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.engines.google</span> <span class="kn">import</span> <span class="n">supported_languages_url</span><span class="p">,</span> <span class="n">_fetch_supported_languages</span>
|
||||
|
||||
<span class="c1"># pylint: enable=unused-import</span>
|
||||
|
||||
<span class="c1"># about</span>
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.google.com'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q219885'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># engine dependent config</span>
|
||||
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">language_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">use_locale_domain</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="n">RE_CACHE</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_re</span><span class="p">(</span><span class="n">regexpr</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""returns compiled regular expression"""</span>
|
||||
<span class="n">RE_CACHE</span><span class="p">[</span><span class="n">regexpr</span><span class="p">]</span> <span class="o">=</span> <span class="n">RE_CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">regexpr</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">regexpr</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">RE_CACHE</span><span class="p">[</span><span class="n">regexpr</span><span class="p">]</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">scrap_out_thumbs_src</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
|
||||
<span class="n">ret_val</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">thumb_name</span> <span class="o">=</span> <span class="s1">'dimg_'</span>
|
||||
<span class="k">for</span> <span class="n">script</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//script[contains(., "google.ldi={")]'</span><span class="p">):</span>
|
||||
<span class="n">_script</span> <span class="o">=</span> <span class="n">script</span><span class="o">.</span><span class="n">text</span>
|
||||
<span class="c1"># "dimg_35":"https://i.ytimg.c....",</span>
|
||||
<span class="n">_dimurl</span> <span class="o">=</span> <span class="n">_re</span><span class="p">(</span><span class="s2">"s='([^']*)"</span><span class="p">)</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">_script</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">_re</span><span class="p">(</span><span class="s1">'('</span> <span class="o">+</span> <span class="n">thumb_name</span> <span class="o">+</span> <span class="s1">'[0-9]*)":"(http[^"]*)'</span><span class="p">)</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">_script</span><span class="p">):</span>
|
||||
<span class="n">v</span> <span class="o">=</span> <span class="n">v</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\u003d'</span><span class="p">,</span> <span class="s1">'='</span><span class="p">)</span>
|
||||
<span class="n">v</span> <span class="o">=</span> <span class="n">v</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\u0026'</span><span class="p">,</span> <span class="s1">'&'</span><span class="p">)</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"found </span><span class="si">%s</span><span class="s2"> imgdata for: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">thumb_name</span><span class="p">,</span> <span class="n">ret_val</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="k">return</span> <span class="n">ret_val</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="scrap_out_thumbs"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_videos.scrap_out_thumbs">[docs]</a><span class="k">def</span> <span class="nf">scrap_out_thumbs</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Scrap out thumbnail data from <script> tags."""</span>
|
||||
<span class="n">ret_val</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">thumb_name</span> <span class="o">=</span> <span class="s1">'dimg_'</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">script</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//script[contains(., "_setImagesSrc")]'</span><span class="p">):</span>
|
||||
<span class="n">_script</span> <span class="o">=</span> <span class="n">script</span><span class="o">.</span><span class="n">text</span>
|
||||
|
||||
<span class="c1"># var s='data:image/jpeg;base64, ...'</span>
|
||||
<span class="n">_imgdata</span> <span class="o">=</span> <span class="n">_re</span><span class="p">(</span><span class="s2">"s='([^']*)"</span><span class="p">)</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">_script</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">_imgdata</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="c1"># var ii=['dimg_17']</span>
|
||||
<span class="k">for</span> <span class="n">_vidthumb</span> <span class="ow">in</span> <span class="n">_re</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(</span><span class="si">%s</span><span class="s2">\d+)"</span> <span class="o">%</span> <span class="n">thumb_name</span><span class="p">)</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">_script</span><span class="p">):</span>
|
||||
<span class="c1"># At least the equal sign in the URL needs to be decoded</span>
|
||||
<span class="n">ret_val</span><span class="p">[</span><span class="n">_vidthumb</span><span class="p">]</span> <span class="o">=</span> <span class="n">_imgdata</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\x3d"</span><span class="p">,</span> <span class="s2">"="</span><span class="p">)</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"found </span><span class="si">%s</span><span class="s2"> imgdata for: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">thumb_name</span><span class="p">,</span> <span class="n">ret_val</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_videos.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Google-Video search request"""</span>
|
||||
|
||||
<span class="n">lang_info</span> <span class="o">=</span> <span class="n">get_lang_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">supported_languages</span><span class="p">,</span> <span class="n">language_aliases</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'https://'</span>
|
||||
<span class="o">+</span> <span class="n">lang_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||
<span class="o">+</span> <span class="s2">"?"</span>
|
||||
<span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">'tbm'</span><span class="p">:</span> <span class="s2">"vid"</span><span class="p">,</span> <span class="o">**</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span> <span class="s1">'ie'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span> <span class="s1">'oe'</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">})</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'CONSENT'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"YES+"</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">lang_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'</span>
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.google.html#searx.engines.google_videos.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># convert the text to dom</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">vidthumb_imgdata</span> <span class="o">=</span> <span class="n">scrap_out_thumbs</span><span class="p">(</span><span class="n">dom</span><span class="p">)</span>
|
||||
<span class="n">thumbs_src</span> <span class="o">=</span> <span class="n">scrap_out_thumbs_src</span><span class="p">(</span><span class="n">dom</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">thumbs_src</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># parse results</span>
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "g ")]'</span><span class="p">):</span>
|
||||
|
||||
<span class="c1"># ignore google *sections*</span>
|
||||
<span class="k">if</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">g_section_with_header</span><span class="p">)):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"ignoring <g-section-with-header>"</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="c1"># ingnore articles without an image id / e.g. news articles</span>
|
||||
<span class="n">img_id</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//g-img/img/@id'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">img_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"no img_id found in item </span><span class="si">%s</span><span class="s2"> (news article?)"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">results</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">img_src</span> <span class="o">=</span> <span class="n">vidthumb_imgdata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">img_id</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">img_src</span><span class="p">:</span>
|
||||
<span class="n">img_src</span> <span class="o">=</span> <span class="n">thumbs_src</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">img_id</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[@class="dXiKIc"]//a/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">length</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "P7xzyf")]/span/span'</span><span class="p">))</span>
|
||||
<span class="n">c_node</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[@class="Uroaid"]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">c_node</span><span class="p">)</span>
|
||||
<span class="n">pub_info</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[@class="Zg1NU"]'</span><span class="p">))</span>
|
||||
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||
<span class="s1">'length'</span><span class="p">:</span> <span class="n">length</span><span class="p">,</span>
|
||||
<span class="s1">'author'</span><span class="p">:</span> <span class="n">pub_info</span><span class="p">,</span>
|
||||
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
|
||||
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># parse suggestion</span>
|
||||
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||
<span class="c1"># append suggestion</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,344 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.tineye — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.tineye</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.tineye</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""This engine implements *Tineye - reverse image search*</span>
|
||||
|
||||
<span class="sd">Using TinEye, you can search by image or perform what we call a reverse image</span>
|
||||
<span class="sd">search. You can do that by uploading an image or searching by URL. You can also</span>
|
||||
<span class="sd">simply drag and drop your images to start your search. TinEye constantly crawls</span>
|
||||
<span class="sd">the web and adds images to its index. Today, the TinEye index is over 50.2</span>
|
||||
<span class="sd">billion images `[tineye.com] <https://tineye.com/how>`_.</span>
|
||||
|
||||
<span class="sd">.. hint::</span>
|
||||
|
||||
<span class="sd"> This SearXNG engine only supports *'searching by URL'* and it does not use</span>
|
||||
<span class="sd"> the official API `[api.tineye.com] <https://api.tineye.com/python/docs/>`_.</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">flask_babel</span> <span class="kn">import</span> <span class="n">gettext</span>
|
||||
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://tineye.com'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q2382535'</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://api.tineye.com/python/docs/'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">'online_url_search'</span>
|
||||
<span class="sd">""":py:obj:`searx.search.processors.online_url_search`"""</span>
|
||||
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://tineye.com'</span>
|
||||
<span class="n">search_string</span> <span class="o">=</span> <span class="s1">'/result_json/?page=</span><span class="si">{page}</span><span class="s1">&</span><span class="si">{query}</span><span class="s1">'</span>
|
||||
|
||||
<span class="n">FORMAT_NOT_SUPPORTED</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
|
||||
<span class="s2">"Could not read that image url. This may be due to an unsupported file"</span>
|
||||
<span class="s2">" format. TinEye only supports images that are JPEG, PNG, GIF, BMP, TIFF or WebP."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sd">"""TinEye error message"""</span>
|
||||
|
||||
<span class="n">NO_SIGNATURE_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
|
||||
<span class="s2">"The image is too simple to find matches. TinEye requires a basic level of"</span>
|
||||
<span class="s2">" visual detail to successfully identify matches."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sd">"""TinEye error message"""</span>
|
||||
|
||||
<span class="n">DOWNLOAD_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"The image could not be downloaded."</span><span class="p">)</span>
|
||||
<span class="sd">"""TinEye error message"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.tineye.html#searx.engines.tineye.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Build TinEye HTTP request using ``search_urls`` of a :py:obj:`engine_type`."""</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'data:image'</span><span class="p">]:</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'data:image'</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'http'</span><span class="p">]:</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'http'</span><span class="p">]</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"query URL: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">query</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">query</span><span class="p">})</span>
|
||||
|
||||
<span class="c1"># see https://github.com/TinEye/pytineye/blob/main/pytineye/api.py</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">search_string</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">,</span> <span class="n">page</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">])</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'Connection'</span><span class="p">:</span> <span class="s1">'keep-alive'</span><span class="p">,</span>
|
||||
<span class="s1">'Accept-Encoding'</span><span class="p">:</span> <span class="s1">'gzip, defalte, br'</span><span class="p">,</span>
|
||||
<span class="s1">'Host'</span><span class="p">:</span> <span class="s1">'tineye.com'</span><span class="p">,</span>
|
||||
<span class="s1">'DNT'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||
<span class="s1">'TE'</span><span class="p">:</span> <span class="s1">'trailers'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_tineye_match"><a class="viewcode-back" href="../../../src/searx.engines.tineye.html#searx.engines.tineye.parse_tineye_match">[docs]</a><span class="k">def</span> <span class="nf">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Takes parsed JSON from the API server and turns it into a :py:obj:`dict`</span>
|
||||
<span class="sd"> object.</span>
|
||||
|
||||
<span class="sd"> Attributes `(class Match) <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__</span>
|
||||
|
||||
<span class="sd"> - `image_url`, link to the result image.</span>
|
||||
<span class="sd"> - `domain`, domain this result was found on.</span>
|
||||
<span class="sd"> - `score`, a number (0 to 100) that indicates how closely the images match.</span>
|
||||
<span class="sd"> - `width`, image width in pixels.</span>
|
||||
<span class="sd"> - `height`, image height in pixels.</span>
|
||||
<span class="sd"> - `size`, image area in pixels.</span>
|
||||
<span class="sd"> - `format`, image format.</span>
|
||||
<span class="sd"> - `filesize`, image size in bytes.</span>
|
||||
<span class="sd"> - `overlay`, overlay URL.</span>
|
||||
<span class="sd"> - `tags`, whether this match belongs to a collection or stock domain.</span>
|
||||
|
||||
<span class="sd"> - `backlinks`, a list of Backlink objects pointing to the original websites</span>
|
||||
<span class="sd"> and image URLs. List items are instances of :py:obj:`dict`, (`Backlink</span>
|
||||
<span class="sd"> <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__):</span>
|
||||
|
||||
<span class="sd"> - `url`, the image URL to the image.</span>
|
||||
<span class="sd"> - `backlink`, the original website URL.</span>
|
||||
<span class="sd"> - `crawl_date`, the date the image was crawled.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="c1"># HINT: there exists an alternative backlink dict in the domains list / e.g.::</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># match_json['domains'][0]['backlinks']</span>
|
||||
|
||||
<span class="n">backlinks</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="s2">"backlinks"</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">:</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">backlink_json</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">[</span><span class="s2">"backlinks"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">backlink_json</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"crawl_date"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">crawl_date</span><span class="p">:</span>
|
||||
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">crawl_date</span><span class="p">[:</span><span class="o">-</span><span class="mi">3</span><span class="p">])</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">min</span>
|
||||
|
||||
<span class="n">backlinks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">),</span>
|
||||
<span class="s1">'backlink'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"backlink"</span><span class="p">),</span>
|
||||
<span class="s1">'crawl_date'</span><span class="p">:</span> <span class="n">crawl_date</span><span class="p">,</span>
|
||||
<span class="s1">'image_name'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"image_name"</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="p">{</span>
|
||||
<span class="s1">'image_url'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"image_url"</span><span class="p">),</span>
|
||||
<span class="s1">'domain'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"domain"</span><span class="p">),</span>
|
||||
<span class="s1">'score'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"score"</span><span class="p">),</span>
|
||||
<span class="s1">'width'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"width"</span><span class="p">),</span>
|
||||
<span class="s1">'height'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"height"</span><span class="p">),</span>
|
||||
<span class="s1">'size'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"size"</span><span class="p">),</span>
|
||||
<span class="s1">'image_format'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"format"</span><span class="p">),</span>
|
||||
<span class="s1">'filesize'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filesize"</span><span class="p">),</span>
|
||||
<span class="s1">'overlay'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"overlay"</span><span class="p">),</span>
|
||||
<span class="s1">'tags'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"tags"</span><span class="p">),</span>
|
||||
<span class="s1">'backlinks'</span><span class="p">:</span> <span class="n">backlinks</span><span class="p">,</span>
|
||||
<span class="p">}</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.tineye.html#searx.engines.tineye.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Parse HTTP response from TinEye."""</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||
<span class="n">msg</span> <span class="o">=</span> <span class="s2">"can't parse JSON response // </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">exc</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||
<span class="n">json_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'error'</span><span class="p">:</span> <span class="n">msg</span><span class="p">}</span>
|
||||
|
||||
<span class="c1"># handle error codes from Tineye</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">is_error</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">400</span><span class="p">,</span> <span class="mi">422</span><span class="p">):</span>
|
||||
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s1">'HTTP status: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'error'</span><span class="p">)</span>
|
||||
<span class="n">s_key</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'suggestions'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'key'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">error</span> <span class="ow">and</span> <span class="n">s_key</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">error</span><span class="p">,</span> <span class="n">s_key</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="n">error</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s2">"Invalid image URL"</span><span class="p">:</span>
|
||||
<span class="c1"># test https://docs.searxng.org/_static/searxng-wordmark.svg</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="n">FORMAT_NOT_SUPPORTED</span>
|
||||
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">'NO_SIGNATURE_ERROR'</span><span class="p">:</span>
|
||||
<span class="c1"># test https://pngimg.com/uploads/dot/dot_PNG4.png</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="n">NO_SIGNATURE_ERROR</span>
|
||||
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">'Download Error'</span><span class="p">:</span>
|
||||
<span class="c1"># test https://notexists</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="n">DOWNLOAD_ERROR</span>
|
||||
|
||||
<span class="c1"># see https://github.com/searxng/searxng/pull/1456#issuecomment-1193105023</span>
|
||||
<span class="c1"># results.append({'answer': message})</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span>
|
||||
|
||||
<span class="n">resp</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># append results from matches</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">match_json</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'matches'</span><span class="p">]:</span>
|
||||
|
||||
<span class="n">tineye_match</span> <span class="o">=</span> <span class="n">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'backlinks'</span><span class="p">]:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">backlink</span> <span class="o">=</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'backlinks'</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'backlink'</span><span class="p">],</span>
|
||||
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'image_url'</span><span class="p">],</span>
|
||||
<span class="s1">'source'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'image_name'</span><span class="p">],</span>
|
||||
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
|
||||
<span class="s1">'format'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'image_format'</span><span class="p">],</span>
|
||||
<span class="s1">'widht'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'width'</span><span class="p">],</span>
|
||||
<span class="s1">'height'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'height'</span><span class="p">],</span>
|
||||
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'crawl_date'</span><span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># append number of results</span>
|
||||
|
||||
<span class="n">number_of_results</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'num_matches'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">number_of_results</span><span class="p">:</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'number_of_results'</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">})</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,377 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.xpath — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.xpath</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.xpath</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""The XPath engine is a *generic* engine with which it is possible to configure</span>
|
||||
<span class="sd">engines in the settings.</span>
|
||||
|
||||
<span class="sd">Here is a simple example of a XPath engine configured in the</span>
|
||||
<span class="sd">:ref:`settings engine` section, further read :ref:`engines-dev`.</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> - name : bitbucket</span>
|
||||
<span class="sd"> engine : xpath</span>
|
||||
<span class="sd"> paging : True</span>
|
||||
<span class="sd"> search_url : https://bitbucket.org/repo/all/{pageno}?name={query}</span>
|
||||
<span class="sd"> url_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]/@href</span>
|
||||
<span class="sd"> title_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]</span>
|
||||
<span class="sd"> content_xpath : //article[@class="repo-summary"]/p</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlencode</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">extract_url</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.network</span> <span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||
|
||||
<span class="n">search_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd">Search URL of the engine. Example::</span>
|
||||
|
||||
<span class="sd"> https://example.org/?search={query}&page={pageno}{time_range}{safe_search}</span>
|
||||
|
||||
<span class="sd">Replacements are:</span>
|
||||
|
||||
<span class="sd">``{query}``:</span>
|
||||
<span class="sd"> Search terms from user.</span>
|
||||
|
||||
<span class="sd">``{pageno}``:</span>
|
||||
<span class="sd"> Page number if engine supports pagging :py:obj:`paging`</span>
|
||||
|
||||
<span class="sd">``{lang}``:</span>
|
||||
<span class="sd"> ISO 639-1 language code (en, de, fr ..)</span>
|
||||
|
||||
<span class="sd">``{time_range}``:</span>
|
||||
<span class="sd"> :py:obj:`URL parameter <time_range_url>` if engine :py:obj:`supports time</span>
|
||||
<span class="sd"> range <time_range_support>`. The value for the parameter is taken from</span>
|
||||
<span class="sd"> :py:obj:`time_range_map`.</span>
|
||||
|
||||
<span class="sd">``{safe_search}``:</span>
|
||||
<span class="sd"> Safe-search :py:obj:`URL parameter <safe_search_map>` if engine</span>
|
||||
<span class="sd"> :py:obj:`supports safe-search <safe_search_support>`. The ``{safe_search}``</span>
|
||||
<span class="sd"> replacement is taken from the :py:obj:`safes_search_map`. Filter results::</span>
|
||||
|
||||
<span class="sd"> 0: none, 1: moderate, 2:strict</span>
|
||||
|
||||
<span class="sd"> If not supported, the URL parameter is an empty string.</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="n">lang_all</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||
<span class="sd">'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is</span>
|
||||
<span class="sd">selected.</span>
|
||||
<span class="sd">'''</span>
|
||||
|
||||
<span class="n">no_result_for_http_status</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="sd">'''Return empty result for these HTTP status codes instead of throwing an error.</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> no_result_for_http_status: []</span>
|
||||
<span class="sd">'''</span>
|
||||
|
||||
<span class="n">soft_max_redirects</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="sd">'''Maximum redirects, soft limit. Record an error but don't stop the engine'''</span>
|
||||
|
||||
<span class="n">results_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="sd">'''XPath selector for the list of result items'''</span>
|
||||
|
||||
<span class="n">url_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="sd">'''XPath selector of result's ``url``.'''</span>
|
||||
|
||||
<span class="n">content_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="sd">'''XPath selector of result's ``content``.'''</span>
|
||||
|
||||
<span class="n">title_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="sd">'''XPath selector of result's ``title``.'''</span>
|
||||
|
||||
<span class="n">thumbnail_xpath</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="sd">'''XPath selector of result's ``img_src``.'''</span>
|
||||
|
||||
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="sd">'''XPath selector of result's ``suggestion``.'''</span>
|
||||
|
||||
<span class="n">cached_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="n">cached_url</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
|
||||
<span class="n">cookies</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="sd">'''Some engines might offer different result based on cookies or headers.</span>
|
||||
<span class="sd">Possible use-case: To set safesearch cookie or header to moderate.'''</span>
|
||||
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="sd">'''Engine supports paging [True or False].'''</span>
|
||||
|
||||
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
<span class="sd">'''Number of results on each page. Only needed if the site requires not a page</span>
|
||||
<span class="sd">number, but an offset.'''</span>
|
||||
|
||||
<span class="n">first_page_num</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
<span class="sd">'''Number of the first page (usually 0 or 1).'''</span>
|
||||
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="sd">'''Engine supports search time range.'''</span>
|
||||
|
||||
<span class="n">time_range_url</span> <span class="o">=</span> <span class="s1">'&hours=</span><span class="si">{time_range_val}</span><span class="s1">'</span>
|
||||
<span class="sd">'''Time range URL parameter in the in :py:obj:`search_url`. If no time range is</span>
|
||||
<span class="sd">requested by the user, the URL parameter is an empty string. The</span>
|
||||
<span class="sd">``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> time_range_url : '&days={time_range_val}'</span>
|
||||
<span class="sd">'''</span>
|
||||
|
||||
<span class="n">time_range_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'day'</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span>
|
||||
<span class="s1">'week'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
|
||||
<span class="s1">'month'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span><span class="p">,</span>
|
||||
<span class="s1">'year'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="sd">'''Maps time range value from user to ``{time_range_val}`` in</span>
|
||||
<span class="sd">:py:obj:`time_range_url`.</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> time_range_map:</span>
|
||||
<span class="sd"> day: 1</span>
|
||||
<span class="sd"> week: 7</span>
|
||||
<span class="sd"> month: 30</span>
|
||||
<span class="sd"> year: 365</span>
|
||||
<span class="sd">'''</span>
|
||||
|
||||
<span class="n">safe_search_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="sd">'''Engine supports safe-search.'''</span>
|
||||
|
||||
<span class="n">safe_search_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'&filter=none'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'&filter=moderate'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'&filter=strict'</span><span class="p">}</span>
|
||||
<span class="sd">'''Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.</span>
|
||||
|
||||
<span class="sd">.. code:: yaml</span>
|
||||
|
||||
<span class="sd"> safesearch: true</span>
|
||||
<span class="sd"> safes_search_map:</span>
|
||||
<span class="sd"> 0: '&filter=none'</span>
|
||||
<span class="sd"> 1: '&filter=moderate'</span>
|
||||
<span class="sd"> 2: '&filter=strict'</span>
|
||||
|
||||
<span class="sd">'''</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../admin/engines/searx.engines.xpath.html#searx.engines.xpath.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">'''Build request parameters (see :ref:`engine request`).'''</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang_all</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span>
|
||||
|
||||
<span class="n">time_range</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">):</span>
|
||||
<span class="n">time_range_val</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">))</span>
|
||||
<span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_range_val</span><span class="o">=</span><span class="n">time_range_val</span><span class="p">)</span>
|
||||
|
||||
<span class="n">safe_search</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||
<span class="n">safe_search</span> <span class="o">=</span> <span class="n">safe_search_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">fargs</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'query'</span><span class="p">:</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">})[</span><span class="mi">2</span><span class="p">:],</span>
|
||||
<span class="s1">'lang'</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
|
||||
<span class="s1">'pageno'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">page_size</span> <span class="o">+</span> <span class="n">first_page_num</span><span class="p">,</span>
|
||||
<span class="s1">'time_range'</span><span class="p">:</span> <span class="n">time_range</span><span class="p">,</span>
|
||||
<span class="s1">'safe_search'</span><span class="p">:</span> <span class="n">safe_search</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">cookies</span><span class="p">)</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fargs</span><span class="p">)</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'soft_max_redirects'</span><span class="p">]</span> <span class="o">=</span> <span class="n">soft_max_redirects</span>
|
||||
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../admin/engines/searx.engines.xpath.html#searx.engines.xpath.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span> <span class="c1"># pylint: disable=too-many-branches</span>
|
||||
<span class="w"> </span><span class="sd">'''Scrap *results* from the response (see :ref:`engine results`).'''</span>
|
||||
<span class="k">if</span> <span class="n">no_result_for_http_status</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="n">no_result_for_http_status</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">is_onion</span> <span class="o">=</span> <span class="s1">'onions'</span> <span class="ow">in</span> <span class="n">categories</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">results_xpath</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">results_xpath</span><span class="p">):</span>
|
||||
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">search_url</span><span class="p">)</span>
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">))</span>
|
||||
<span class="n">tmp_result</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">}</span>
|
||||
|
||||
<span class="c1"># add thumbnail if available</span>
|
||||
<span class="k">if</span> <span class="n">thumbnail_xpath</span><span class="p">:</span>
|
||||
<span class="n">thumbnail_xpath_result</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">thumbnail_xpath</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'img_src'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># add alternative cached url if available</span>
|
||||
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
|
||||
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'cached_url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">is_onion</span><span class="p">:</span>
|
||||
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'is_onion'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp_result</span><span class="p">)</span>
|
||||
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">cached</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
|
||||
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">)),</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||
<span class="s1">'cached_url'</span><span class="p">:</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">cached</span><span class="p">,</span>
|
||||
<span class="s1">'is_onion'</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
|
||||
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span> <span class="s1">'is_onion'</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">})</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">suggestion_xpath</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"found </span><span class="si">%s</span><span class="s2"> results"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">results</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,285 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.engines.yahoo — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.engines.yahoo</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.engines.yahoo</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""Yahoo Search (Web)</span>
|
||||
|
||||
<span class="sd">Languages are supported by mapping the language to a domain. If domain is not</span>
|
||||
<span class="sd">found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">unquote</span><span class="p">,</span>
|
||||
<span class="n">urlencode</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||
<span class="n">extract_text</span><span class="p">,</span>
|
||||
<span class="n">match_language</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># about</span>
|
||||
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://search.yahoo.com/'</span><span class="p">,</span>
|
||||
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developer.yahoo.com/api/'</span><span class="p">,</span>
|
||||
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># engine dependent config</span>
|
||||
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">supported_languages_url</span> <span class="o">=</span> <span class="s1">'https://search.yahoo.com/preferences/languages'</span>
|
||||
<span class="sd">"""Supported languages are read from Yahoo preference page."""</span>
|
||||
|
||||
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'day'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'1d'</span><span class="p">,</span> <span class="s1">'d'</span><span class="p">),</span>
|
||||
<span class="s1">'week'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'1w'</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">),</span>
|
||||
<span class="s1">'month'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'1m'</span><span class="p">,</span> <span class="s1">'m'</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">language_aliases</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'zh-HK'</span><span class="p">:</span> <span class="s1">'zh_chs'</span><span class="p">,</span>
|
||||
<span class="s1">'zh-CN'</span><span class="p">:</span> <span class="s1">'zh_chs'</span><span class="p">,</span> <span class="c1"># dead since 2015 / routed to hk.search.yahoo.com</span>
|
||||
<span class="s1">'zh-TW'</span><span class="p">:</span> <span class="s1">'zh_cht'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">lang2domain</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'zh_chs'</span><span class="p">:</span> <span class="s1">'hk.search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'zh_cht'</span><span class="p">:</span> <span class="s1">'tw.search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'en'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'bg'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'cs'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'da'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'el'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'et'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'he'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'hr'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'ja'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'ko'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'sk'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="s1">'sl'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="sd">"""Map language to domain"""</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_get_language</span><span class="p">(</span><span class="n">params</span><span class="p">):</span>
|
||||
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">language_aliases</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">match_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">],</span> <span class="n">supported_languages</span><span class="p">,</span> <span class="n">language_aliases</span><span class="p">)</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"params['language']: </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">],</span> <span class="n">lang</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">lang</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="request"><a class="viewcode-back" href="../../../src/searx.engines.yahoo.html#searx.engines.yahoo.request">[docs]</a><span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""build request"""</span>
|
||||
<span class="n">offset</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">+</span> <span class="mi">1</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">_get_language</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
|
||||
<span class="n">age</span><span class="p">,</span> <span class="n">btf</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">],</span> <span class="p">(</span><span class="s1">''</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
|
||||
|
||||
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s1">'p'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||
<span class="s1">'ei'</span><span class="p">:</span> <span class="s1">'UTF-8'</span><span class="p">,</span>
|
||||
<span class="s1">'fl'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s1">'vl'</span><span class="p">:</span> <span class="s1">'lang_'</span> <span class="o">+</span> <span class="n">lang</span><span class="p">,</span>
|
||||
<span class="s1">'btf'</span><span class="p">:</span> <span class="n">btf</span><span class="p">,</span>
|
||||
<span class="s1">'fr2'</span><span class="p">:</span> <span class="s1">'time'</span><span class="p">,</span>
|
||||
<span class="s1">'age'</span><span class="p">:</span> <span class="n">age</span><span class="p">,</span>
|
||||
<span class="s1">'b'</span><span class="p">:</span> <span class="n">offset</span><span class="p">,</span>
|
||||
<span class="s1">'xargs'</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">lang2domain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">.search.yahoo.com'</span> <span class="o">%</span> <span class="n">lang</span><span class="p">)</span>
|
||||
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'https://</span><span class="si">%s</span><span class="s1">/search?</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">params</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_url"><a class="viewcode-back" href="../../../src/searx.engines.yahoo.html#searx.engines.yahoo.parse_url">[docs]</a><span class="k">def</span> <span class="nf">parse_url</span><span class="p">(</span><span class="n">url_string</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""remove yahoo-specific tracking-url"""</span>
|
||||
|
||||
<span class="n">endings</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'/RS'</span><span class="p">,</span> <span class="s1">'/RK'</span><span class="p">]</span>
|
||||
<span class="n">endpositions</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">start</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'http'</span><span class="p">,</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'/RU='</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">ending</span> <span class="ow">in</span> <span class="n">endings</span><span class="p">:</span>
|
||||
<span class="n">endpos</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="n">ending</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">endpos</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
|
||||
<span class="n">endpositions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">endpos</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">url_string</span>
|
||||
|
||||
<span class="n">end</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">unquote</span><span class="p">(</span><span class="n">url_string</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">])</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="response"><a class="viewcode-back" href="../../../src/searx.engines.yahoo.html#searx.engines.yahoo.response">[docs]</a><span class="k">def</span> <span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""parse response"""</span>
|
||||
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># parse results</span>
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class,"algo-sr")]'</span><span class="p">):</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//h3/a/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">parse_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//h3/a'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">title</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">offset</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'span'</span><span class="p">)))</span>
|
||||
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">)[</span><span class="n">offset</span><span class="p">:]</span>
|
||||
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "compText")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span>
|
||||
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># append result</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "AlsoTry")]//table//a'</span><span class="p">):</span>
|
||||
<span class="c1"># append suggestion</span>
|
||||
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<span class="c1"># get supported languages from their site</span>
|
||||
<span class="k">def</span> <span class="nf">_fetch_supported_languages</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||
<span class="n">supported_languages</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">offset</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="s1">'lang_'</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "lang-item")]/input/@value'</span><span class="p">):</span>
|
||||
<span class="n">supported_languages</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">[</span><span class="n">offset</span><span class="p">:])</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">supported_languages</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../engines.html">searx.engines</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,302 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.infopage — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.infopage</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.infopage</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="c1"># pyright: basic</span>
|
||||
<span class="sd">"""Render SearXNG instance documentation.</span>
|
||||
|
||||
<span class="sd">Usage in a Flask app route:</span>
|
||||
|
||||
<span class="sd">.. code:: python</span>
|
||||
|
||||
<span class="sd"> from searx import infopage</span>
|
||||
|
||||
<span class="sd"> _INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage)</span>
|
||||
|
||||
<span class="sd"> @app.route('/info/<pagename>', methods=['GET'])</span>
|
||||
<span class="sd"> def info(pagename):</span>
|
||||
|
||||
<span class="sd"> locale = request.preferences.get_value('locale')</span>
|
||||
<span class="sd"> page = _INFO_PAGES.get_page(pagename, locale)</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'InfoPage'</span><span class="p">,</span> <span class="s1">'InfoPageSet'</span><span class="p">]</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">os.path</span>
|
||||
<span class="kn">import</span> <span class="nn">logging</span>
|
||||
<span class="kn">import</span> <span class="nn">typing</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">urllib.parse</span>
|
||||
<span class="kn">import</span> <span class="nn">jinja2</span>
|
||||
<span class="kn">from</span> <span class="nn">flask.helpers</span> <span class="kn">import</span> <span class="n">url_for</span>
|
||||
<span class="kn">from</span> <span class="nn">markdown_it</span> <span class="kn">import</span> <span class="n">MarkdownIt</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">..</span> <span class="kn">import</span> <span class="n">get_setting</span>
|
||||
<span class="kn">from</span> <span class="nn">..compat</span> <span class="kn">import</span> <span class="n">cached_property</span>
|
||||
<span class="kn">from</span> <span class="nn">..version</span> <span class="kn">import</span> <span class="n">GIT_URL</span>
|
||||
<span class="kn">from</span> <span class="nn">..locales</span> <span class="kn">import</span> <span class="n">LOCALE_NAMES</span>
|
||||
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">'searx.infopage'</span><span class="p">)</span>
|
||||
<span class="n">_INFO_FOLDER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InfoPage"><a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage">[docs]</a><span class="k">class</span> <span class="nc">InfoPage</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""A page of the :py:obj:`online documentation <InfoPageSet>`."""</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fname</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">fname</span> <span class="o">=</span> <span class="n">fname</span>
|
||||
|
||||
<span class="nd">@cached_property</span>
|
||||
<span class="k">def</span> <span class="nf">raw_content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raw content of the page (without any jinja rendering)"""</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||
|
||||
<span class="nd">@cached_property</span>
|
||||
<span class="k">def</span> <span class="nf">content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Content of the page (rendered in a Jinja conntext)"""</span>
|
||||
<span class="n">ctx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_ctx</span><span class="p">()</span>
|
||||
<span class="n">template</span> <span class="o">=</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">()</span><span class="o">.</span><span class="n">from_string</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">template</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">ctx</span><span class="p">)</span>
|
||||
|
||||
<span class="nd">@cached_property</span>
|
||||
<span class="k">def</span> <span class="nf">title</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Title of the content (without any markup)"""</span>
|
||||
<span class="n">t</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'# '</span><span class="p">):</span>
|
||||
<span class="n">t</span> <span class="o">=</span> <span class="n">l</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'# '</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">t</span>
|
||||
|
||||
<span class="nd">@cached_property</span>
|
||||
<span class="k">def</span> <span class="nf">html</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Render Markdown (CommonMark_) to HTML by using markdown-it-py_.</span>
|
||||
|
||||
<span class="sd"> .. _CommonMark: https://commonmark.org/</span>
|
||||
<span class="sd"> .. _markdown-it-py: https://github.com/executablebooks/markdown-it-py</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="n">MarkdownIt</span><span class="p">(</span><span class="s2">"commonmark"</span><span class="p">,</span> <span class="p">{</span><span class="s2">"typographer"</span><span class="p">:</span> <span class="kc">True</span><span class="p">})</span><span class="o">.</span><span class="n">enable</span><span class="p">([</span><span class="s2">"replacements"</span><span class="p">,</span> <span class="s2">"smartquotes"</span><span class="p">])</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<div class="viewcode-block" id="InfoPage.get_ctx"><a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.get_ctx">[docs]</a> <span class="k">def</span> <span class="nf">get_ctx</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Jinja context to render :py:obj:`InfoPage.content`"""</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">_md_link</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">url_for</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="s2">"[</span><span class="si">%s</span><span class="s2">](</span><span class="si">%s</span><span class="s2">)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">_md_search</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">?q=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'search'</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="s1">'[</span><span class="si">%s</span><span class="s1">](</span><span class="si">%s</span><span class="s1">)'</span> <span class="o">%</span> <span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="n">ctx</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">ctx</span><span class="p">[</span><span class="s1">'GIT_URL'</span><span class="p">]</span> <span class="o">=</span> <span class="n">GIT_URL</span>
|
||||
<span class="n">ctx</span><span class="p">[</span><span class="s1">'get_setting'</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_setting</span>
|
||||
<span class="n">ctx</span><span class="p">[</span><span class="s1">'link'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_link</span>
|
||||
<span class="n">ctx</span><span class="p">[</span><span class="s1">'search'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_search</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">ctx</span></div>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s1">'<</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1"> fname=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="si">!r}</span><span class="s1">>'</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InfoPageSet"><a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet">[docs]</a><span class="k">class</span> <span class="nc">InfoPageSet</span><span class="p">:</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||
<span class="w"> </span><span class="sd">"""Cached rendering of the online documentation a SearXNG instance has.</span>
|
||||
|
||||
<span class="sd"> :param page_class: render online documentation by :py:obj:`InfoPage` parser.</span>
|
||||
<span class="sd"> :type page_class: :py:obj:`InfoPage`</span>
|
||||
|
||||
<span class="sd"> :param info_folder: information directory</span>
|
||||
<span class="sd"> :type info_folder: str</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span> <span class="n">page_class</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="n">typing</span><span class="o">.</span><span class="n">Type</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">info_folder</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">page_class</span> <span class="o">=</span> <span class="n">page_class</span> <span class="ow">or</span> <span class="n">InfoPage</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">info_folder</span> <span class="ow">or</span> <span class="n">_INFO_FOLDER</span>
|
||||
<span class="w"> </span><span class="sd">"""location of the Markdwon files"""</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Dict</span><span class="p">[</span><span class="nb">tuple</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||
<span class="w"> </span><span class="sd">"""default language"""</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span> <span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">_INFO_FOLDER</span><span class="p">)</span> <span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span> <span class="ow">in</span> <span class="n">LOCALE_NAMES</span>
|
||||
<span class="p">]</span>
|
||||
<span class="w"> </span><span class="sd">"""list of supported languages (aka locales)"""</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s1">'search-syntax'</span><span class="p">,</span>
|
||||
<span class="s1">'about'</span><span class="p">,</span>
|
||||
<span class="s1">'donate'</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
<span class="w"> </span><span class="sd">"""list of articles in the online documentation"""</span>
|
||||
|
||||
<div class="viewcode-block" id="InfoPageSet.get_page"><a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.get_page">[docs]</a> <span class="k">def</span> <span class="nf">get_page</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pagename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Return ``pagename`` instance of :py:obj:`InfoPage`</span>
|
||||
|
||||
<span class="sd"> :param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc`</span>
|
||||
<span class="sd"> :type pagename: str</span>
|
||||
|
||||
<span class="sd"> :param locale: language of the page, e.g. ``en``, ``zh_Hans_CN``</span>
|
||||
<span class="sd"> (default: :py:obj:`InfoPageSet.i18n_origin`)</span>
|
||||
<span class="sd"> :type locale: str</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">pagename</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">locale</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">cache_key</span> <span class="o">=</span> <span class="p">(</span><span class="n">pagename</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">cache_key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># not yet instantiated</span>
|
||||
|
||||
<span class="n">fname</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">,</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">),</span> <span class="n">pagename</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'.md'</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">fname</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'file </span><span class="si">%s</span><span class="s1"> does not exists'</span><span class="p">,</span> <span class="n">fname</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_class</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">page</span>
|
||||
<span class="k">return</span> <span class="n">page</span></div>
|
||||
|
||||
<div class="viewcode-block" id="InfoPageSet.iter_pages"><a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.iter_pages">[docs]</a> <span class="k">def</span> <span class="nf">iter_pages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">fallback_to_default</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Iterate over all pages of the TOC"""</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||
<span class="k">for</span> <span class="n">page_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
|
||||
<span class="n">page_locale</span> <span class="o">=</span> <span class="n">locale</span>
|
||||
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">fallback_to_default</span> <span class="ow">and</span> <span class="n">page</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">page_locale</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">page</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># page is None if the page was deleted by the administrator</span>
|
||||
<span class="k">yield</span> <span class="n">page_name</span><span class="p">,</span> <span class="n">page_locale</span><span class="p">,</span> <span class="n">page</span></div></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,420 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.locales — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.locales</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.locales</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
||||
<span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""Initialize :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`.</span>
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Set</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">pathlib</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">babel</span> <span class="kn">import</span> <span class="n">Locale</span>
|
||||
<span class="kn">from</span> <span class="nn">babel.support</span> <span class="kn">import</span> <span class="n">Translations</span>
|
||||
<span class="kn">import</span> <span class="nn">babel.languages</span>
|
||||
<span class="kn">import</span> <span class="nn">babel.core</span>
|
||||
<span class="kn">import</span> <span class="nn">flask_babel</span>
|
||||
<span class="kn">import</span> <span class="nn">flask</span>
|
||||
<span class="kn">from</span> <span class="nn">flask.ctx</span> <span class="kn">import</span> <span class="n">has_request_context</span>
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'locales'</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="c1"># safe before monkey patching flask_babel.get_translations</span>
|
||||
<span class="n">_flask_babel_get_translations</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span>
|
||||
|
||||
<span class="n">LOCALE_NAMES</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="sd">"""Mapping of locales and their description. Locales e.g. 'fr' or 'pt-BR' (see</span>
|
||||
<span class="sd">:py:obj:`locales_initialize`).</span>
|
||||
|
||||
<span class="sd">:meta hide-value:</span>
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="n">RTL_LOCALES</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||
<span class="sd">"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see</span>
|
||||
<span class="sd">:py:obj:`locales_initialize`)."""</span>
|
||||
|
||||
<span class="n">ADDITIONAL_TRANSLATIONS</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"dv"</span><span class="p">:</span> <span class="s2">"ދިވެހި (Dhivehi)"</span><span class="p">,</span>
|
||||
<span class="s2">"oc"</span><span class="p">:</span> <span class="s2">"Occitan"</span><span class="p">,</span>
|
||||
<span class="s2">"szl"</span><span class="p">:</span> <span class="s2">"Ślōnski (Silesian)"</span><span class="p">,</span>
|
||||
<span class="s2">"pap"</span><span class="p">:</span> <span class="s2">"Papiamento"</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="sd">"""Additional languages SearXNG has translations for but not supported by</span>
|
||||
<span class="sd">python-babel (see :py:obj:`locales_initialize`)."""</span>
|
||||
|
||||
<span class="n">LOCALE_BEST_MATCH</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"dv"</span><span class="p">:</span> <span class="s2">"si"</span><span class="p">,</span>
|
||||
<span class="s2">"oc"</span><span class="p">:</span> <span class="s1">'fr-FR'</span><span class="p">,</span>
|
||||
<span class="s2">"szl"</span><span class="p">:</span> <span class="s2">"pl"</span><span class="p">,</span>
|
||||
<span class="s2">"nl-BE"</span><span class="p">:</span> <span class="s2">"nl"</span><span class="p">,</span>
|
||||
<span class="s2">"zh-HK"</span><span class="p">:</span> <span class="s2">"zh-Hant-TW"</span><span class="p">,</span>
|
||||
<span class="s2">"pap"</span><span class="p">:</span> <span class="s2">"pt-BR"</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="sd">"""Map a locale we do not have a translations for to a locale we have a</span>
|
||||
<span class="sd">translation for. By example: use Taiwan version of the translation for Hong</span>
|
||||
<span class="sd">Kong."""</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">localeselector</span><span class="p">():</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
|
||||
<span class="n">value</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'locale'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">value</span>
|
||||
|
||||
<span class="c1"># first, set the language that is not supported by babel</span>
|
||||
<span class="k">if</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
|
||||
<span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">form</span><span class="p">[</span><span class="s1">'use-translation'</span><span class="p">]</span> <span class="o">=</span> <span class="n">locale</span>
|
||||
|
||||
<span class="c1"># second, map locale to a value python-babel supports</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">LOCALE_BEST_MATCH</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">locale</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
|
||||
<span class="c1"># if there is an error loading the preferences</span>
|
||||
<span class="c1"># the locale is going to be ''</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||
|
||||
<span class="c1"># babel uses underscore instead of hyphen.</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">locale</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_translations"><a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_translations">[docs]</a><span class="k">def</span> <span class="nf">get_translations</span><span class="p">():</span>
|
||||
<span class="w"> </span><span class="sd">"""Monkey patch of :py:obj:`flask_babel.get_translations`"""</span>
|
||||
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
|
||||
<span class="n">use_translation</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">form</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'use-translation'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">use_translation</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
|
||||
<span class="n">babel_ext</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">current_app</span><span class="o">.</span><span class="n">extensions</span><span class="p">[</span><span class="s1">'babel'</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">Translations</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">next</span><span class="p">(</span><span class="n">babel_ext</span><span class="o">.</span><span class="n">translation_directories</span><span class="p">),</span> <span class="n">use_translation</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">_flask_babel_get_translations</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_locale_descr"><a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_locale_descr">[docs]</a><span class="k">def</span> <span class="nf">get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">locale_name</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR'</span>
|
||||
|
||||
<span class="sd"> :param locale: instance of :py:class:`Locale`</span>
|
||||
<span class="sd"> :param locale_name: name e.g. 'fr' or 'pt_BR' (delimiter is *underscore*)</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="n">native_language</span><span class="p">,</span> <span class="n">native_territory</span> <span class="o">=</span> <span class="n">_get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">locale_name</span><span class="p">)</span>
|
||||
<span class="n">english_language</span><span class="p">,</span> <span class="n">english_territory</span> <span class="o">=</span> <span class="n">_get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">native_territory</span> <span class="o">==</span> <span class="n">english_territory</span><span class="p">:</span>
|
||||
<span class="n">english_territory</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">native_territory</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">english_territory</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">native_language</span> <span class="o">==</span> <span class="n">english_language</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">native_language</span>
|
||||
<span class="k">return</span> <span class="n">native_language</span> <span class="o">+</span> <span class="s1">' ('</span> <span class="o">+</span> <span class="n">english_language</span> <span class="o">+</span> <span class="s1">')'</span>
|
||||
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">native_language</span> <span class="o">+</span> <span class="s1">', '</span> <span class="o">+</span> <span class="n">native_territory</span> <span class="o">+</span> <span class="s1">' ('</span> <span class="o">+</span> <span class="n">english_language</span>
|
||||
<span class="k">if</span> <span class="n">english_territory</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">result</span> <span class="o">+</span> <span class="s1">', '</span> <span class="o">+</span> <span class="n">english_territory</span> <span class="o">+</span> <span class="s1">')'</span>
|
||||
<span class="k">return</span> <span class="n">result</span> <span class="o">+</span> <span class="s1">')'</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">language_code</span><span class="p">):</span>
|
||||
<span class="n">language_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">get_language_name</span><span class="p">(</span><span class="n">language_code</span><span class="p">)</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">language_name</span> <span class="ow">and</span> <span class="p">(</span><span class="s1">'a'</span> <span class="o"><=</span> <span class="n">language_name</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o"><=</span> <span class="s1">'z'</span><span class="p">):</span>
|
||||
<span class="n">language_name</span> <span class="o">=</span> <span class="n">language_name</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
|
||||
<span class="n">terrirtory_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">get_territory_name</span><span class="p">(</span><span class="n">language_code</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">language_name</span><span class="p">,</span> <span class="n">terrirtory_name</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="locales_initialize"><a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.locales_initialize">[docs]</a><span class="k">def</span> <span class="nf">locales_initialize</span><span class="p">(</span><span class="n">directory</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Initialize locales environment of the SearXNG session.</span>
|
||||
|
||||
<span class="sd"> - monkey patch :py:obj:`flask_babel.get_translations` by :py:obj:`get_translations`</span>
|
||||
<span class="sd"> - init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="n">directory</span> <span class="o">=</span> <span class="n">directory</span> <span class="ow">or</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s1">'translations'</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"locales_initialize: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
||||
<span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span> <span class="o">=</span> <span class="n">get_translations</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">tag</span><span class="p">,</span> <span class="n">descr</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">LOCALE_BEST_MATCH</span><span class="p">[</span><span class="n">tag</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="n">LOCALE_NAMES</span><span class="p">[</span><span class="n">tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">descr</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">text_direction</span> <span class="o">==</span> <span class="s1">'rtl'</span><span class="p">:</span>
|
||||
<span class="n">RTL_LOCALES</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">LOCALE_BEST_MATCH</span><span class="p">:</span>
|
||||
<span class="n">descr</span> <span class="o">=</span> <span class="n">LOCALE_NAMES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">descr</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="n">LOCALE_NAMES</span><span class="p">[</span><span class="n">tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">tag</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">text_direction</span> <span class="o">==</span> <span class="s1">'rtl'</span><span class="p">:</span>
|
||||
<span class="n">RTL_LOCALES</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">dirname</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">directory</span><span class="p">)):</span>
|
||||
<span class="c1"># Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">directory</span><span class="p">,</span> <span class="n">dirname</span><span class="p">,</span> <span class="s1">'LC_MESSAGES'</span><span class="p">)):</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">tag</span> <span class="o">=</span> <span class="n">dirname</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="n">descr</span> <span class="o">=</span> <span class="n">LOCALE_NAMES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">descr</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">dirname</span><span class="p">)</span>
|
||||
<span class="n">LOCALE_NAMES</span><span class="p">[</span><span class="n">tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_locale_descr</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">dirname</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">text_direction</span> <span class="o">==</span> <span class="s1">'rtl'</span><span class="p">:</span>
|
||||
<span class="n">RTL_LOCALES</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_engine_locale"><a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_engine_locale">[docs]</a><span class="k">def</span> <span class="nf">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">engine_locales</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Return engine's language (aka locale) string that best fits to argument</span>
|
||||
<span class="sd"> ``searxng_locale``.</span>
|
||||
|
||||
<span class="sd"> Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to</span>
|
||||
<span class="sd"> corresponding *engine locales*::</span>
|
||||
|
||||
<span class="sd"> <engine>: {</span>
|
||||
<span class="sd"> # SearXNG string : engine-string</span>
|
||||
<span class="sd"> 'ca-ES' : 'ca_ES',</span>
|
||||
<span class="sd"> 'fr-BE' : 'fr_BE',</span>
|
||||
<span class="sd"> 'fr-CA' : 'fr_CA',</span>
|
||||
<span class="sd"> 'fr-CH' : 'fr_CH',</span>
|
||||
<span class="sd"> 'fr' : 'fr_FR',</span>
|
||||
<span class="sd"> ...</span>
|
||||
<span class="sd"> 'pl-PL' : 'pl_PL',</span>
|
||||
<span class="sd"> 'pt-PT' : 'pt_PT'</span>
|
||||
<span class="sd"> }</span>
|
||||
|
||||
<span class="sd"> .. hint::</span>
|
||||
|
||||
<span class="sd"> The *SearXNG locale* string has to be known by babel!</span>
|
||||
|
||||
<span class="sd"> If there is no direct 1:1 mapping, this functions tries to narrow down</span>
|
||||
<span class="sd"> engine's language (locale). If no value can be determined by these</span>
|
||||
<span class="sd"> approximation attempts the ``default`` value is returned.</span>
|
||||
|
||||
<span class="sd"> Assumptions:</span>
|
||||
|
||||
<span class="sd"> A. When user select a language the results should be optimized according to</span>
|
||||
<span class="sd"> the selected language.</span>
|
||||
|
||||
<span class="sd"> B. When user select a language and a territory the results should be</span>
|
||||
<span class="sd"> optimized with first priority on terrirtory and second on language.</span>
|
||||
|
||||
<span class="sd"> First approximation rule (*by territory*):</span>
|
||||
|
||||
<span class="sd"> When the user selects a locale with terrirtory (and a language), the</span>
|
||||
<span class="sd"> territory has priority over the language. If any of the offical languages</span>
|
||||
<span class="sd"> in the terrirtory is supported by the engine (``engine_locales``) it will</span>
|
||||
<span class="sd"> be used.</span>
|
||||
|
||||
<span class="sd"> Second approximation rule (*by language*):</span>
|
||||
|
||||
<span class="sd"> If "First approximation rule" brings no result or the user selects only a</span>
|
||||
<span class="sd"> language without a terrirtory. Check in which territories the language</span>
|
||||
<span class="sd"> has an offical status and if one of these territories is supported by the</span>
|
||||
<span class="sd"> engine.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||
|
||||
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># There was a 1:1 mapping (e.g. "fr-BE --> fr_BE" or "fr --> fr_FR"), no</span>
|
||||
<span class="c1"># need to narrow language nor territory.</span>
|
||||
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">default</span>
|
||||
|
||||
<span class="c1"># SearXNG's selected locale is not supported by the engine ..</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||
<span class="c1"># Try to narrow by *offical* languages in the territory (??-XX).</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">official_language</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">official_language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span>
|
||||
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||
|
||||
<span class="c1"># Engine does not support one of the offical languages in the territory or</span>
|
||||
<span class="c1"># there is only a language selected without a territory.</span>
|
||||
|
||||
<span class="c1"># Now lets have a look if the searxng_lang (the language selected by the</span>
|
||||
<span class="c1"># user) is a offical language in other territories. If so, check if</span>
|
||||
<span class="c1"># engine does support the searxng_lang in this other territory.</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">:</span>
|
||||
|
||||
<span class="n">searxng_lang</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span><span class="p">:</span>
|
||||
<span class="n">searxng_lang</span> <span class="o">+=</span> <span class="s1">'_'</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span>
|
||||
|
||||
<span class="n">terr_lang_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">langs</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">get_global</span><span class="p">(</span><span class="s2">"territory_languages"</span><span class="p">)</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">langs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'official_status'</span><span class="p">):</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">terr_lang_dict</span><span class="p">[</span><span class="n">territory</span><span class="p">]</span> <span class="o">=</span> <span class="n">langs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># first: check fr-FR, de-DE .. is supported by the engine</span>
|
||||
<span class="c1"># exception: 'en' --> 'en-US'</span>
|
||||
|
||||
<span class="n">territory</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">territory</span> <span class="o">==</span> <span class="s1">'EN'</span><span class="p">:</span>
|
||||
<span class="n">territory</span> <span class="o">=</span> <span class="s1">'US'</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">territory</span><span class="p">):</span>
|
||||
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">territory</span>
|
||||
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||
|
||||
<span class="c1"># second: sort by population_percent and take first match</span>
|
||||
|
||||
<span class="c1"># drawback of "population percent": if there is a terrirtory with a</span>
|
||||
<span class="c1"># small number of people (e.g 100) but the majority speaks the</span>
|
||||
<span class="c1"># language, then the percentage migth be 100% (--> 100 people) but in</span>
|
||||
<span class="c1"># a different terrirtory with more people (e.g. 10.000) where only 10%</span>
|
||||
<span class="c1"># speak the language the total amount of speaker is higher (--> 200</span>
|
||||
<span class="c1"># people).</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># By example: The population of Saint-Martin is 33.000, of which 100%</span>
|
||||
<span class="c1"># speak French, but this is less than the 30% of the approximately 2.5</span>
|
||||
<span class="c1"># million Belgian citizens</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'</span>
|
||||
<span class="c1"># - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'</span>
|
||||
|
||||
<span class="n">terr_lang_list</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">terr_lang_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">_lang</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">terr_lang_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="s1">'population_percent'</span><span class="p">],</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">territory</span>
|
||||
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||
|
||||
<span class="c1"># No luck: narrow by "language from territory" and "territory from language"</span>
|
||||
<span class="c1"># does not fit to a locale supported by the engine.</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">default</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">default</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,356 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.redislib — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.redislib</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.redislib</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="sd">"""A collection of convenient functions and redis/lua scripts.</span>
|
||||
|
||||
<span class="sd">This code was partial inspired by the `Bullet-Proofing Lua Scripts in RedisPy`_</span>
|
||||
<span class="sd">article.</span>
|
||||
|
||||
<span class="sd">.. _Bullet-Proofing Lua Scripts in RedisPy:</span>
|
||||
<span class="sd"> https://redis.com/blog/bullet-proofing-lua-scripts-in-redispy/</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">hmac</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">get_setting</span>
|
||||
|
||||
<span class="n">LUA_SCRIPT_STORAGE</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="sd">"""A global dictionary to cache client's ``Script`` objects, used by</span>
|
||||
<span class="sd">:py:obj:`lua_script_storage`"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="lua_script_storage"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.lua_script_storage">[docs]</a><span class="k">def</span> <span class="nf">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">script</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Returns a redis :py:obj:`Script</span>
|
||||
<span class="sd"> <redis.commands.core.CoreCommands.register_script>` instance.</span>
|
||||
|
||||
<span class="sd"> Due to performance reason the ``Script`` object is instantiated only once</span>
|
||||
<span class="sd"> for a client (``client.register_script(..)``) and is cached in</span>
|
||||
<span class="sd"> :py:obj:`LUA_SCRIPT_STORAGE`.</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="c1"># redis connection can be closed, lets use the id() of the redis connector</span>
|
||||
<span class="c1"># as key in the script-storage:</span>
|
||||
<span class="n">client_id</span> <span class="o">=</span> <span class="nb">id</span><span class="p">(</span><span class="n">client</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">client_id</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">script</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">][</span><span class="n">script</span><span class="p">]</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">register_script</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">][</span><span class="n">script</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
<span class="n">PURGE_BY_PREFIX</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||
<span class="s2">local prefix = tostring(ARGV[1])</span>
|
||||
<span class="s2">for i, name in ipairs(redis.call('KEYS', prefix .. '*')) do</span>
|
||||
<span class="s2"> redis.call('EXPIRE', name, 0)</span>
|
||||
<span class="s2">end</span>
|
||||
<span class="s2">"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="purge_by_prefix"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.purge_by_prefix">[docs]</a><span class="k">def</span> <span class="nf">purge_by_prefix</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"SearXNG_"</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Purge all keys with ``prefix`` from database.</span>
|
||||
|
||||
<span class="sd"> Queries all keys in the database by the given prefix and set expire time to</span>
|
||||
<span class="sd"> zero. The default prefix will drop all keys which has been set by SearXNG</span>
|
||||
<span class="sd"> (drops SearXNG schema entirely from database).</span>
|
||||
|
||||
<span class="sd"> The implementation is the lua script from string :py:obj:`PURGE_BY_PREFIX`.</span>
|
||||
<span class="sd"> The lua script uses EXPIRE_ instead of DEL_: if there are a lot keys to</span>
|
||||
<span class="sd"> delete and/or their values are big, `DEL` could take more time and blocks</span>
|
||||
<span class="sd"> the command loop while `EXPIRE` turns back immediate.</span>
|
||||
|
||||
<span class="sd"> :param prefix: prefix of the key to delete (default: ``SearXNG_``)</span>
|
||||
<span class="sd"> :type name: str</span>
|
||||
|
||||
<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span>
|
||||
<span class="sd"> .. _DEL: https://redis.io/commands/del/</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">PURGE_BY_PREFIX</span><span class="p">)</span>
|
||||
<span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">prefix</span><span class="p">])</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="secret_hash"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.secret_hash">[docs]</a><span class="k">def</span> <span class="nf">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Creates a hash of the ``name``.</span>
|
||||
|
||||
<span class="sd"> Combines argument ``name`` with the ``secret_key`` from :ref:`settings</span>
|
||||
<span class="sd"> server`. This function can be used to get a more anonymised name of a Redis</span>
|
||||
<span class="sd"> KEY.</span>
|
||||
|
||||
<span class="sd"> :param name: the name to create a secret hash for</span>
|
||||
<span class="sd"> :type name: str</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">m</span> <span class="o">=</span> <span class="n">hmac</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="n">digestmod</span><span class="o">=</span><span class="s1">'sha256'</span><span class="p">)</span>
|
||||
<span class="n">m</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">get_setting</span><span class="p">(</span><span class="s1">'server.secret_key'</span><span class="p">),</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<span class="n">INCR_COUNTER</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||
<span class="s2">local limit = tonumber(ARGV[1])</span>
|
||||
<span class="s2">local expire = tonumber(ARGV[2])</span>
|
||||
<span class="s2">local c_name = KEYS[1]</span>
|
||||
|
||||
<span class="s2">local c = redis.call('GET', c_name)</span>
|
||||
|
||||
<span class="s2">if not c then</span>
|
||||
<span class="s2"> c = redis.call('INCR', c_name)</span>
|
||||
<span class="s2"> if expire > 0 then</span>
|
||||
<span class="s2"> redis.call('EXPIRE', c_name, expire)</span>
|
||||
<span class="s2"> end</span>
|
||||
<span class="s2">else</span>
|
||||
<span class="s2"> c = tonumber(c)</span>
|
||||
<span class="s2"> if limit == 0 or c < limit then</span>
|
||||
<span class="s2"> c = redis.call('INCR', c_name)</span>
|
||||
<span class="s2"> end</span>
|
||||
<span class="s2">end</span>
|
||||
<span class="s2">return c</span>
|
||||
<span class="s2">"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="incr_counter"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.incr_counter">[docs]</a><span class="k">def</span> <span class="nf">incr_counter</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Increment a counter and return the new value.</span>
|
||||
|
||||
<span class="sd"> If counter with redis key ``SearXNG_counter_<name>`` does not exists it is</span>
|
||||
<span class="sd"> created with initial value 1 returned. The replacement ``<name>`` is a</span>
|
||||
<span class="sd"> *secret hash* of the value from argument ``name`` (see</span>
|
||||
<span class="sd"> :py:func:`secret_hash`).</span>
|
||||
|
||||
<span class="sd"> The implementation of the redis counter is the lua script from string</span>
|
||||
<span class="sd"> :py:obj:`INCR_COUNTER`.</span>
|
||||
|
||||
<span class="sd"> :param name: name of the counter</span>
|
||||
<span class="sd"> :type name: str</span>
|
||||
|
||||
<span class="sd"> :param expire: live-time of the counter in seconds (default ``None`` means</span>
|
||||
<span class="sd"> infinite).</span>
|
||||
<span class="sd"> :type expire: int / see EXPIRE_</span>
|
||||
|
||||
<span class="sd"> :param limit: limit where the counter stops to increment (default ``None``)</span>
|
||||
<span class="sd"> :type limit: int / limit is 2^64 see INCR_</span>
|
||||
|
||||
<span class="sd"> :return: value of the incremented counter</span>
|
||||
<span class="sd"> :type return: int</span>
|
||||
|
||||
<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span>
|
||||
<span class="sd"> .. _INCR: https://redis.io/commands/incr/</span>
|
||||
|
||||
<span class="sd"> A simple demo of a counter with expire time and limit::</span>
|
||||
|
||||
<span class="sd"> >>> for i in range(6):</span>
|
||||
<span class="sd"> ... i, incr_counter(client, "foo", 3, 5) # max 3, duration 5 sec</span>
|
||||
<span class="sd"> ... time.sleep(1) # from the third call on max has been reached</span>
|
||||
<span class="sd"> ...</span>
|
||||
<span class="sd"> (0, 1)</span>
|
||||
<span class="sd"> (1, 2)</span>
|
||||
<span class="sd"> (2, 3)</span>
|
||||
<span class="sd"> (3, 3)</span>
|
||||
<span class="sd"> (4, 3)</span>
|
||||
<span class="sd"> (5, 1)</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">INCR_COUNTER</span><span class="p">)</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="n">c</span> <span class="o">=</span> <span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">limit</span><span class="p">,</span> <span class="n">expire</span><span class="p">],</span> <span class="n">keys</span><span class="o">=</span><span class="p">[</span><span class="n">name</span><span class="p">])</span>
|
||||
<span class="k">return</span> <span class="n">c</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="drop_counter"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.drop_counter">[docs]</a><span class="k">def</span> <span class="nf">drop_counter</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Drop counter with redis key ``SearXNG_counter_<name>``</span>
|
||||
|
||||
<span class="sd"> The replacement ``<name>`` is a *secret hash* of the value from argument</span>
|
||||
<span class="sd"> ``name`` (see :py:func:`incr_counter` and :py:func:`incr_sliding_window`).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="n">client</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">name</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="n">INCR_SLIDING_WINDOW</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||
<span class="s2">local expire = tonumber(ARGV[1])</span>
|
||||
<span class="s2">local name = KEYS[1]</span>
|
||||
<span class="s2">local current_time = redis.call('TIME')</span>
|
||||
|
||||
<span class="s2">redis.call('ZREMRANGEBYSCORE', name, 0, current_time[1] - expire)</span>
|
||||
<span class="s2">redis.call('ZADD', name, current_time[1], current_time[1] .. current_time[2])</span>
|
||||
<span class="s2">local result = redis.call('ZCOUNT', name, 0, current_time[1] + 1)</span>
|
||||
<span class="s2">redis.call('EXPIRE', name, expire)</span>
|
||||
<span class="s2">return result</span>
|
||||
<span class="s2">"""</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="incr_sliding_window"><a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.incr_sliding_window">[docs]</a><span class="k">def</span> <span class="nf">incr_sliding_window</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">duration</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Increment a sliding-window counter and return the new value.</span>
|
||||
|
||||
<span class="sd"> If counter with redis key ``SearXNG_counter_<name>`` does not exists it is</span>
|
||||
<span class="sd"> created with initial value 1 returned. The replacement ``<name>`` is a</span>
|
||||
<span class="sd"> *secret hash* of the value from argument ``name`` (see</span>
|
||||
<span class="sd"> :py:func:`secret_hash`).</span>
|
||||
|
||||
<span class="sd"> :param name: name of the counter</span>
|
||||
<span class="sd"> :type name: str</span>
|
||||
|
||||
<span class="sd"> :param duration: live-time of the sliding window in seconds</span>
|
||||
<span class="sd"> :typeduration: int</span>
|
||||
|
||||
<span class="sd"> :return: value of the incremented counter</span>
|
||||
<span class="sd"> :type return: int</span>
|
||||
|
||||
<span class="sd"> The implementation of the redis counter is the lua script from string</span>
|
||||
<span class="sd"> :py:obj:`INCR_SLIDING_WINDOW`. The lua script uses `sorted sets in Redis`_</span>
|
||||
<span class="sd"> to implement a sliding window for the redis key ``SearXNG_counter_<name>``</span>
|
||||
<span class="sd"> (ZADD_). The current TIME_ is used to score the items in the sorted set and</span>
|
||||
<span class="sd"> the time window is moved by removing items with a score lower current time</span>
|
||||
<span class="sd"> minus *duration* time (ZREMRANGEBYSCORE_).</span>
|
||||
|
||||
<span class="sd"> The EXPIRE_ time (the duration of the sliding window) is refreshed on each</span>
|
||||
<span class="sd"> call (incrementation) and if there is no call in this duration, the sorted</span>
|
||||
<span class="sd"> set expires from the redis DB.</span>
|
||||
|
||||
<span class="sd"> The return value is the amount of items in the sorted set (ZCOUNT_), what</span>
|
||||
<span class="sd"> means the number of calls in the sliding window.</span>
|
||||
|
||||
<span class="sd"> .. _Sorted sets in Redis:</span>
|
||||
<span class="sd"> https://redis.com/ebook/part-1-getting-started/chapter-1-getting-to-know-redis/1-2-what-redis-data-structures-look-like/1-2-5-sorted-sets-in-redis/</span>
|
||||
<span class="sd"> .. _TIME: https://redis.io/commands/time/</span>
|
||||
<span class="sd"> .. _ZADD: https://redis.io/commands/zadd/</span>
|
||||
<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span>
|
||||
<span class="sd"> .. _ZREMRANGEBYSCORE: https://redis.io/commands/zremrangebyscore/</span>
|
||||
<span class="sd"> .. _ZCOUNT: https://redis.io/commands/zcount/</span>
|
||||
|
||||
<span class="sd"> A simple demo of the sliding window::</span>
|
||||
|
||||
<span class="sd"> >>> for i in range(5):</span>
|
||||
<span class="sd"> ... incr_sliding_window(client, "foo", 3) # duration 3 sec</span>
|
||||
<span class="sd"> ... time.sleep(1) # from the third call (second) on the window is moved</span>
|
||||
<span class="sd"> ...</span>
|
||||
<span class="sd"> 1</span>
|
||||
<span class="sd"> 2</span>
|
||||
<span class="sd"> 3</span>
|
||||
<span class="sd"> 3</span>
|
||||
<span class="sd"> 3</span>
|
||||
<span class="sd"> >>> time.sleep(3) # wait until expire</span>
|
||||
<span class="sd"> >>> incr_sliding_window(client, "foo", 3)</span>
|
||||
<span class="sd"> 1</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">INCR_SLIDING_WINDOW</span><span class="p">)</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="n">c</span> <span class="o">=</span> <span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">duration</span><span class="p">],</span> <span class="n">keys</span><span class="o">=</span><span class="p">[</span><span class="n">name</span><span class="p">])</span>
|
||||
<span class="k">return</span> <span class="n">c</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,324 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.search — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.search</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.search</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="c1"># pylint: disable=missing-module-docstring, too-few-public-methods</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">threading</span>
|
||||
<span class="kn">from</span> <span class="nn">timeit</span> <span class="kn">import</span> <span class="n">default_timer</span>
|
||||
<span class="kn">from</span> <span class="nn">uuid</span> <span class="kn">import</span> <span class="n">uuid4</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">flask</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">settings</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.answerers</span> <span class="kn">import</span> <span class="n">ask</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.external_bang</span> <span class="kn">import</span> <span class="n">get_bang_url</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.results</span> <span class="kn">import</span> <span class="n">ResultContainer</span>
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.plugins</span> <span class="kn">import</span> <span class="n">plugins</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.search.models</span> <span class="kn">import</span> <span class="n">EngineRef</span><span class="p">,</span> <span class="n">SearchQuery</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.engines</span> <span class="kn">import</span> <span class="n">load_engines</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.network</span> <span class="kn">import</span> <span class="n">initialize</span> <span class="k">as</span> <span class="n">initialize_network</span><span class="p">,</span> <span class="n">check_network_configuration</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.metrics</span> <span class="kn">import</span> <span class="n">initialize</span> <span class="k">as</span> <span class="n">initialize_metrics</span><span class="p">,</span> <span class="n">counter_inc</span><span class="p">,</span> <span class="n">histogram_observe_time</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.search.processors</span> <span class="kn">import</span> <span class="n">PROCESSORS</span><span class="p">,</span> <span class="n">initialize</span> <span class="k">as</span> <span class="n">initialize_processors</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.search.checker</span> <span class="kn">import</span> <span class="n">initialize</span> <span class="k">as</span> <span class="n">initialize_checker</span>
|
||||
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'search'</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">settings_engines</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">enable_checker</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">check_network</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">enable_metrics</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||
<span class="n">settings_engines</span> <span class="o">=</span> <span class="n">settings_engines</span> <span class="ow">or</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'engines'</span><span class="p">]</span>
|
||||
<span class="n">load_engines</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">)</span>
|
||||
<span class="n">initialize_network</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">,</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">check_network</span><span class="p">:</span>
|
||||
<span class="n">check_network_configuration</span><span class="p">()</span>
|
||||
<span class="n">initialize_metrics</span><span class="p">([</span><span class="n">engine</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="k">for</span> <span class="n">engine</span> <span class="ow">in</span> <span class="n">settings_engines</span><span class="p">],</span> <span class="n">enable_metrics</span><span class="p">)</span>
|
||||
<span class="n">initialize_processors</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">enable_checker</span><span class="p">:</span>
|
||||
<span class="n">initialize_checker</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="Search"><a class="viewcode-back" href="../../src/searx.search.html#searx.search.Search">[docs]</a><span class="k">class</span> <span class="nc">Search</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Search information container"""</span>
|
||||
|
||||
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s2">"search_query"</span><span class="p">,</span> <span class="s2">"result_container"</span><span class="p">,</span> <span class="s2">"start_time"</span><span class="p">,</span> <span class="s2">"actual_timeout"</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">search_query</span><span class="p">:</span> <span class="n">SearchQuery</span><span class="p">):</span>
|
||||
<span class="c1"># init vars</span>
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">search_query</span> <span class="o">=</span> <span class="n">search_query</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span> <span class="o">=</span> <span class="n">ResultContainer</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">start_time</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">actual_timeout</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">search_external_bang</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Check if there is a external bang.</span>
|
||||
<span class="sd"> If yes, update self.result_container and return True</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">external_bang</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">redirect_url</span> <span class="o">=</span> <span class="n">get_bang_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># This means there was a valid bang and the</span>
|
||||
<span class="c1"># rest of the search does not need to be continued</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">redirect_url</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">search_answerers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Check if an answer return a result.</span>
|
||||
<span class="sd"> If yes, update self.result_container and return True</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">answerers_results</span> <span class="o">=</span> <span class="n">ask</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">answerers_results</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">results</span> <span class="ow">in</span> <span class="n">answerers_results</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="s1">'answer'</span><span class="p">,</span> <span class="n">results</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
|
||||
<span class="c1"># do search-request</span>
|
||||
<span class="k">def</span> <span class="nf">_get_requests</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># init vars</span>
|
||||
<span class="n">requests</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="c1"># max of all selected engine timeout</span>
|
||||
<span class="n">default_timeout</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
|
||||
<span class="c1"># start search-reqest for all selected engines</span>
|
||||
<span class="k">for</span> <span class="n">engineref</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">engineref_list</span><span class="p">:</span>
|
||||
<span class="n">processor</span> <span class="o">=</span> <span class="n">PROCESSORS</span><span class="p">[</span><span class="n">engineref</span><span class="o">.</span><span class="n">name</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># stop the request now if the engine is suspend</span>
|
||||
<span class="k">if</span> <span class="n">processor</span><span class="o">.</span><span class="n">extend_container_if_suspended</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="p">):</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="c1"># set default request parameters</span>
|
||||
<span class="n">request_params</span> <span class="o">=</span> <span class="n">processor</span><span class="o">.</span><span class="n">get_params</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="p">,</span> <span class="n">engineref</span><span class="o">.</span><span class="n">category</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">request_params</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">counter_inc</span><span class="p">(</span><span class="s1">'engine'</span><span class="p">,</span> <span class="n">engineref</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'search'</span><span class="p">,</span> <span class="s1">'count'</span><span class="p">,</span> <span class="s1">'sent'</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># append request to list</span>
|
||||
<span class="n">requests</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">engineref</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">,</span> <span class="n">request_params</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># update default_timeout</span>
|
||||
<span class="n">default_timeout</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">default_timeout</span><span class="p">,</span> <span class="n">processor</span><span class="o">.</span><span class="n">engine</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># adjust timeout</span>
|
||||
<span class="n">max_request_timeout</span> <span class="o">=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">][</span><span class="s1">'max_request_timeout'</span><span class="p">]</span>
|
||||
<span class="n">actual_timeout</span> <span class="o">=</span> <span class="n">default_timeout</span>
|
||||
<span class="n">query_timeout</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">timeout_limit</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">max_request_timeout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">query_timeout</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># No max, no user query: default_timeout</span>
|
||||
<span class="k">pass</span>
|
||||
<span class="k">elif</span> <span class="n">max_request_timeout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">query_timeout</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># No max, but user query: From user query except if above default</span>
|
||||
<span class="n">actual_timeout</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">default_timeout</span><span class="p">,</span> <span class="n">query_timeout</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">max_request_timeout</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">query_timeout</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># Max, no user query: Default except if above max</span>
|
||||
<span class="n">actual_timeout</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">default_timeout</span><span class="p">,</span> <span class="n">max_request_timeout</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">max_request_timeout</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">query_timeout</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># Max & user query: From user query except if above max</span>
|
||||
<span class="n">actual_timeout</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">query_timeout</span><span class="p">,</span> <span class="n">max_request_timeout</span><span class="p">)</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"actual_timeout=</span><span class="si">{0}</span><span class="s2"> (default_timeout=</span><span class="si">{1}</span><span class="s2">, ?timeout_limit=</span><span class="si">{2}</span><span class="s2">, max_request_timeout=</span><span class="si">{3}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">actual_timeout</span><span class="p">,</span> <span class="n">default_timeout</span><span class="p">,</span> <span class="n">query_timeout</span><span class="p">,</span> <span class="n">max_request_timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">requests</span><span class="p">,</span> <span class="n">actual_timeout</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">search_multiple_requests</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">requests</span><span class="p">):</span>
|
||||
<span class="c1"># pylint: disable=protected-access</span>
|
||||
<span class="n">search_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid4</span><span class="p">())</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">engine_name</span><span class="p">,</span> <span class="n">query</span><span class="p">,</span> <span class="n">request_params</span> <span class="ow">in</span> <span class="n">requests</span><span class="p">:</span>
|
||||
<span class="n">th</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||
<span class="n">target</span><span class="o">=</span><span class="n">PROCESSORS</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">search</span><span class="p">,</span>
|
||||
<span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">request_params</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_time</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">actual_timeout</span><span class="p">),</span>
|
||||
<span class="n">name</span><span class="o">=</span><span class="n">search_id</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">th</span><span class="o">.</span><span class="n">_timeout</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">th</span><span class="o">.</span><span class="n">_engine_name</span> <span class="o">=</span> <span class="n">engine_name</span>
|
||||
<span class="n">th</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">th</span> <span class="ow">in</span> <span class="n">threading</span><span class="o">.</span><span class="n">enumerate</span><span class="p">():</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||
<span class="k">if</span> <span class="n">th</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">search_id</span><span class="p">:</span>
|
||||
<span class="n">remaining_time</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">actual_timeout</span> <span class="o">-</span> <span class="p">(</span><span class="n">default_timer</span><span class="p">()</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_time</span><span class="p">))</span>
|
||||
<span class="n">th</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">remaining_time</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">th</span><span class="o">.</span><span class="n">is_alive</span><span class="p">():</span>
|
||||
<span class="n">th</span><span class="o">.</span><span class="n">_timeout</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">add_unresponsive_engine</span><span class="p">(</span><span class="n">th</span><span class="o">.</span><span class="n">_engine_name</span><span class="p">,</span> <span class="s1">'timeout'</span><span class="p">)</span>
|
||||
<span class="n">PROCESSORS</span><span class="p">[</span><span class="n">th</span><span class="o">.</span><span class="n">_engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'engine timeout'</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">search_standard</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Update self.result_container, self.actual_timeout</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">requests</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">actual_timeout</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_requests</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># send all search-request</span>
|
||||
<span class="k">if</span> <span class="n">requests</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">search_multiple_requests</span><span class="p">(</span><span class="n">requests</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># return results, suggestions, answers and infoboxes</span>
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
|
||||
<span class="c1"># do search-request</span>
|
||||
<div class="viewcode-block" id="Search.search"><a class="viewcode-back" href="../../src/searx.search.html#searx.search.Search.search">[docs]</a> <span class="k">def</span> <span class="nf">search</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ResultContainer</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">start_time</span> <span class="o">=</span> <span class="n">default_timer</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_external_bang</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">search_answerers</span><span class="p">():</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">search_standard</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_container</span></div></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SearchWithPlugins"><a class="viewcode-back" href="../../src/searx.search.html#searx.search.SearchWithPlugins">[docs]</a><span class="k">class</span> <span class="nc">SearchWithPlugins</span><span class="p">(</span><span class="n">Search</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Inherit from the Search class, add calls to the plugins."""</span>
|
||||
|
||||
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">'ordered_plugin_list'</span><span class="p">,</span> <span class="s1">'request'</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">search_query</span><span class="p">:</span> <span class="n">SearchQuery</span><span class="p">,</span> <span class="n">ordered_plugin_list</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">):</span>
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">search_query</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">ordered_plugin_list</span> <span class="o">=</span> <span class="n">ordered_plugin_list</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">on_result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_on_result</span>
|
||||
<span class="c1"># pylint: disable=line-too-long</span>
|
||||
<span class="c1"># get the "real" request to use it outside the Flask context.</span>
|
||||
<span class="c1"># see</span>
|
||||
<span class="c1"># * https://github.com/pallets/flask/blob/d01d26e5210e3ee4cbbdef12f05c886e08e92852/src/flask/globals.py#L55</span>
|
||||
<span class="c1"># * https://github.com/pallets/werkzeug/blob/3c5d3c9bd0d9ce64590f0af8997a38f3823b368d/src/werkzeug/local.py#L548-L559</span>
|
||||
<span class="c1"># * https://werkzeug.palletsprojects.com/en/2.0.x/local/#werkzeug.local.LocalProxy._get_current_object</span>
|
||||
<span class="c1"># pylint: enable=line-too-long</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">request</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">_get_current_object</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">_on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">plugins</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ordered_plugin_list</span><span class="p">,</span> <span class="s1">'on_result'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="p">,</span> <span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
|
||||
<div class="viewcode-block" id="SearchWithPlugins.search"><a class="viewcode-back" href="../../src/searx.search.html#searx.search.SearchWithPlugins.search">[docs]</a> <span class="k">def</span> <span class="nf">search</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ResultContainer</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">plugins</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ordered_plugin_list</span><span class="p">,</span> <span class="s1">'pre_search'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="p">,</span> <span class="bp">self</span><span class="p">):</span>
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
|
||||
|
||||
<span class="n">plugins</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ordered_plugin_list</span><span class="p">,</span> <span class="s1">'post_search'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_container</span></div></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,230 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.search.models — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||
<li class="nav-item nav-item-2"><a href="../search.html" accesskey="U">searx.search</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.search.models</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.search.models</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">typing</span>
|
||||
<span class="kn">import</span> <span class="nn">babel</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="EngineRef"><a class="viewcode-back" href="../../../src/searx.search.html#searx.search.EngineRef">[docs]</a><span class="k">class</span> <span class="nc">EngineRef</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Reference by names to an engine and category"""</span>
|
||||
|
||||
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'category'</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">category</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">category</span> <span class="o">=</span> <span class="n">category</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"EngineRef(</span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">category</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">name</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">category</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">category</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">category</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SearchQuery"><a class="viewcode-back" href="../../../src/searx.search.html#searx.search.SearchQuery">[docs]</a><span class="k">class</span> <span class="nc">SearchQuery</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""container for all the search parameters (query, language, etc...)"""</span>
|
||||
|
||||
<span class="vm">__slots__</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'query'</span><span class="p">,</span>
|
||||
<span class="s1">'engineref_list'</span><span class="p">,</span>
|
||||
<span class="s1">'lang'</span><span class="p">,</span>
|
||||
<span class="s1">'locale'</span><span class="p">,</span>
|
||||
<span class="s1">'safesearch'</span><span class="p">,</span>
|
||||
<span class="s1">'pageno'</span><span class="p">,</span>
|
||||
<span class="s1">'time_range'</span><span class="p">,</span>
|
||||
<span class="s1">'timeout_limit'</span><span class="p">,</span>
|
||||
<span class="s1">'external_bang'</span><span class="p">,</span>
|
||||
<span class="s1">'engine_data'</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">engineref_list</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">List</span><span class="p">[</span><span class="n">EngineRef</span><span class="p">],</span>
|
||||
<span class="n">lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'all'</span><span class="p">,</span>
|
||||
<span class="n">safesearch</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="n">pageno</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">time_range</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout_limit</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">external_bang</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">engine_data</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Optional</span><span class="p">[</span><span class="n">typing</span><span class="o">.</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">engineref_list</span> <span class="o">=</span> <span class="n">engineref_list</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">safesearch</span> <span class="o">=</span> <span class="n">safesearch</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">pageno</span> <span class="o">=</span> <span class="n">pageno</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">timeout_limit</span> <span class="o">=</span> <span class="n">timeout_limit</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">external_bang</span> <span class="o">=</span> <span class="n">external_bang</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">engine_data</span> <span class="o">=</span> <span class="n">engine_data</span> <span class="ow">or</span> <span class="p">{}</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">locale</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
<span class="nd">@property</span>
|
||||
<span class="k">def</span> <span class="nf">categories</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">engineref</span><span class="p">:</span> <span class="n">engineref</span><span class="o">.</span><span class="n">category</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">engineref_list</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"SearchQuery(</span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">, </span><span class="si">{!r}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">engineref_list</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">safesearch</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">pageno</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">time_range</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">timeout_limit</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">external_bang</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">query</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">query</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">engineref_list</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">engineref_list</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">lang</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">lang</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">safesearch</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">safesearch</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">pageno</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">pageno</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">time_range</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">time_range</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout_limit</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">timeout_limit</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_bang</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">external_bang</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
|
||||
<span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">query</span><span class="p">,</span>
|
||||
<span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">engineref_list</span><span class="p">),</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">safesearch</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">pageno</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">time_range</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">timeout_limit</span><span class="p">,</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">external_bang</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
<ul>
|
||||
<li><a href="../search.html">searx.search</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,762 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searx.utils — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searx.utils</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searx.utils</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="c1"># pyright: basic</span>
|
||||
<span class="sd">"""Utility functions for the engines</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib.util</span>
|
||||
<span class="kn">import</span> <span class="nn">types</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Set</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">MutableMapping</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Callable</span>
|
||||
<span class="kn">from</span> <span class="nn">numbers</span> <span class="kn">import</span> <span class="n">Number</span>
|
||||
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">splitext</span><span class="p">,</span> <span class="n">join</span>
|
||||
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">choice</span>
|
||||
<span class="kn">from</span> <span class="nn">html.parser</span> <span class="kn">import</span> <span class="n">HTMLParser</span>
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urljoin</span><span class="p">,</span> <span class="n">urlparse</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">html</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml.etree</span> <span class="kn">import</span> <span class="n">ElementBase</span><span class="p">,</span> <span class="n">XPath</span><span class="p">,</span> <span class="n">XPathError</span><span class="p">,</span> <span class="n">XPathSyntaxError</span><span class="p">,</span> <span class="n">_ElementStringResult</span><span class="p">,</span> <span class="n">_ElementUnicodeResult</span>
|
||||
<span class="kn">from</span> <span class="nn">babel.core</span> <span class="kn">import</span> <span class="n">get_global</span>
|
||||
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">settings</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.data</span> <span class="kn">import</span> <span class="n">USER_AGENTS</span><span class="p">,</span> <span class="n">data_dir</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.version</span> <span class="kn">import</span> <span class="n">VERSION_TAG</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.languages</span> <span class="kn">import</span> <span class="n">language_codes</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.exceptions</span> <span class="kn">import</span> <span class="n">SearxXPathSyntaxException</span><span class="p">,</span> <span class="n">SearxEngineXPathException</span>
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'utils'</span><span class="p">)</span>
|
||||
|
||||
<span class="n">XPathSpecType</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">XPath</span><span class="p">]</span>
|
||||
|
||||
<span class="n">_BLOCKED_TAGS</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'script'</span><span class="p">,</span> <span class="s1">'style'</span><span class="p">)</span>
|
||||
|
||||
<span class="n">_ECMA_UNESCAPE4_RE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'</span><span class="si">%u</span><span class="s1">([0-9a-fA-F]</span><span class="si">{4}</span><span class="s1">)'</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">UNICODE</span><span class="p">)</span>
|
||||
<span class="n">_ECMA_UNESCAPE2_RE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'%([0-9a-fA-F]</span><span class="si">{2}</span><span class="s1">)'</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">UNICODE</span><span class="p">)</span>
|
||||
|
||||
<span class="n">_STORAGE_UNIT_VALUE</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'TB'</span><span class="p">:</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">,</span>
|
||||
<span class="s1">'GB'</span><span class="p">:</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">,</span>
|
||||
<span class="s1">'MB'</span><span class="p">:</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">,</span>
|
||||
<span class="s1">'TiB'</span><span class="p">:</span> <span class="mi">1000</span> <span class="o">*</span> <span class="mi">1000</span> <span class="o">*</span> <span class="mi">1000</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">,</span>
|
||||
<span class="s1">'MiB'</span><span class="p">:</span> <span class="mi">1000</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">,</span>
|
||||
<span class="s1">'KiB'</span><span class="p">:</span> <span class="mi">1000</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">_XPATH_CACHE</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">XPath</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">_LANG_TO_LC_CACHE</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="n">_FASTTEXT_MODEL</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"fasttext.FastText._FastText"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="sd">"""fasttext model to predict laguage of a search term"""</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_NotSetClass</span><span class="p">:</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||
<span class="w"> </span><span class="sd">"""Internal class for this module, do not create instance of this class.</span>
|
||||
<span class="sd"> Replace the None value, allow explicitly pass None as a function argument"""</span>
|
||||
|
||||
|
||||
<span class="n">_NOTSET</span> <span class="o">=</span> <span class="n">_NotSetClass</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="searx_useragent"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.searx_useragent">[docs]</a><span class="k">def</span> <span class="nf">searx_useragent</span><span class="p">()</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Return the searx User Agent"""</span>
|
||||
<span class="k">return</span> <span class="s1">'searx/</span><span class="si">{searx_version}</span><span class="s1"> </span><span class="si">{suffix}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">searx_version</span><span class="o">=</span><span class="n">VERSION_TAG</span><span class="p">,</span> <span class="n">suffix</span><span class="o">=</span><span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">][</span><span class="s1">'useragent_suffix'</span><span class="p">]</span>
|
||||
<span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="gen_useragent"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.gen_useragent">[docs]</a><span class="k">def</span> <span class="nf">gen_useragent</span><span class="p">(</span><span class="n">os_string</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Return a random browser User Agent</span>
|
||||
|
||||
<span class="sd"> See searx/data/useragents.json</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="n">USER_AGENTS</span><span class="p">[</span><span class="s1">'ua'</span><span class="p">]</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">os</span><span class="o">=</span><span class="n">os_string</span> <span class="ow">or</span> <span class="n">choice</span><span class="p">(</span><span class="n">USER_AGENTS</span><span class="p">[</span><span class="s1">'os'</span><span class="p">]),</span> <span class="n">version</span><span class="o">=</span><span class="n">choice</span><span class="p">(</span><span class="n">USER_AGENTS</span><span class="p">[</span><span class="s1">'versions'</span><span class="p">]))</span></div>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_HTMLTextExtractorException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Internal exception raised when the HTML is invalid"""</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_HTMLTextExtractor</span><span class="p">(</span><span class="n">HTMLParser</span><span class="p">):</span> <span class="c1"># pylint: disable=W0223 # (see https://bugs.python.org/issue31844)</span>
|
||||
<span class="w"> </span><span class="sd">"""Internal class to extract text from HTML"""</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="n">HTMLParser</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">'br'</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">' '</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">handle_endtag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">tag</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]:</span>
|
||||
<span class="k">raise</span> <span class="n">_HTMLTextExtractorException</span><span class="p">()</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">is_valid_tag</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_BLOCKED_TAGS</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">handle_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_valid_tag</span><span class="p">():</span>
|
||||
<span class="k">return</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">handle_charref</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_valid_tag</span><span class="p">():</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="n">name</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'x'</span><span class="p">,</span> <span class="s1">'X'</span><span class="p">):</span>
|
||||
<span class="n">codepoint</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">name</span><span class="p">[</span><span class="mi">1</span><span class="p">:],</span> <span class="mi">16</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">codepoint</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">chr</span><span class="p">(</span><span class="n">codepoint</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">handle_entityref</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_valid_tag</span><span class="p">():</span>
|
||||
<span class="k">return</span>
|
||||
<span class="c1"># codepoint = htmlentitydefs.name2codepoint[name]</span>
|
||||
<span class="c1"># self.result.append(chr(codepoint))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_text</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">result</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="html_to_text"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.html_to_text">[docs]</a><span class="k">def</span> <span class="nf">html_to_text</span><span class="p">(</span><span class="n">html_str</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Extract text from a HTML string</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * html_str (str): string HTML</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * str: extracted text</span>
|
||||
|
||||
<span class="sd"> Examples:</span>
|
||||
<span class="sd"> >>> html_to_text('Example <span id="42">#2</span>')</span>
|
||||
<span class="sd"> 'Example #2'</span>
|
||||
|
||||
<span class="sd"> >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')</span>
|
||||
<span class="sd"> 'Example'</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">html_str</span> <span class="o">=</span> <span class="n">html_str</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\r</span><span class="s1">'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span>
|
||||
<span class="n">html_str</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_str</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
|
||||
<span class="n">s</span> <span class="o">=</span> <span class="n">_HTMLTextExtractor</span><span class="p">()</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">s</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="n">html_str</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">_HTMLTextExtractorException</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"HTMLTextExtractor: invalid HTML</span><span class="se">\n</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">html_str</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">s</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_text"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.extract_text">[docs]</a><span class="k">def</span> <span class="nf">extract_text</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="n">allow_none</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Extract text from a lxml result</span>
|
||||
|
||||
<span class="sd"> * if xpath_results is list, extract the text from each result and concat the list</span>
|
||||
<span class="sd"> * if xpath_results is a xml element, extract all the text node from it</span>
|
||||
<span class="sd"> ( text_content() method from lxml )</span>
|
||||
<span class="sd"> * if xpath_results is a string element, then it's already done</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="c1"># it's list of result : concat everything using recursive call</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">xpath_results</span><span class="p">:</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">result</span> <span class="o">+</span> <span class="p">(</span><span class="n">extract_text</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="n">ElementBase</span><span class="p">):</span>
|
||||
<span class="c1"># it's a element</span>
|
||||
<span class="n">text</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">tostring</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'unicode'</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">'text'</span><span class="p">,</span> <span class="n">with_tail</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="p">(</span><span class="n">_ElementStringResult</span><span class="p">,</span> <span class="n">_ElementUnicodeResult</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">Number</span><span class="p">,</span> <span class="nb">bool</span><span class="p">)):</span>
|
||||
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">xpath_results</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">allow_none</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">xpath_results</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">allow_none</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'extract_text(None, allow_none=False)'</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'unsupported type'</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="normalize_url"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.normalize_url">[docs]</a><span class="k">def</span> <span class="nf">normalize_url</span><span class="p">(</span><span class="n">url</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * url (str): Relative URL</span>
|
||||
<span class="sd"> * base_url (str): Base URL, it must be an absolute URL.</span>
|
||||
|
||||
<span class="sd"> Example:</span>
|
||||
<span class="sd"> >>> normalize_url('https://example.com', 'http://example.com/')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> normalize_url('//example.com', 'http://example.com/')</span>
|
||||
<span class="sd"> 'http://example.com/'</span>
|
||||
<span class="sd"> >>> normalize_url('//example.com', 'https://example.com/')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> normalize_url('/path?a=1', 'https://example.com')</span>
|
||||
<span class="sd"> 'https://example.com/path?a=1'</span>
|
||||
<span class="sd"> >>> normalize_url('', 'https://example.com')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> normalize_url('/test', '/path')</span>
|
||||
<span class="sd"> raise ValueError</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * lxml.etree.ParserError</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * str: normalized URL</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'//'</span><span class="p">):</span>
|
||||
<span class="c1"># add http or https to this kind of url //example.com/</span>
|
||||
<span class="n">parsed_search_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">base_url</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{0}</span><span class="s1">:</span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">parsed_search_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s1">'http'</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'/'</span><span class="p">):</span>
|
||||
<span class="c1"># fix relative url to the search engine</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">urljoin</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># fix relative urls that fall through the crack</span>
|
||||
<span class="k">if</span> <span class="s1">'://'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">url</span><span class="p">:</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">urljoin</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># add a / at this end of the url if there is no path</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Cannot parse url'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">:</span>
|
||||
<span class="n">url</span> <span class="o">+=</span> <span class="s1">'/'</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">url</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_url"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.extract_url">[docs]</a><span class="k">def</span> <span class="nf">extract_url</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">,</span> <span class="n">base_url</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Extract and normalize URL from lxml Element</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)</span>
|
||||
<span class="sd"> * base_url (str): Base URL</span>
|
||||
|
||||
<span class="sd"> Example:</span>
|
||||
<span class="sd"> >>> def f(s, search_url):</span>
|
||||
<span class="sd"> >>> return searx.utils.extract_url(html.fromstring(s), search_url)</span>
|
||||
<span class="sd"> >>> f('<span id="42">https://example.com</span>', 'http://example.com/')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> f('https://example.com', 'http://example.com/')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> f('//example.com', 'http://example.com/')</span>
|
||||
<span class="sd"> 'http://example.com/'</span>
|
||||
<span class="sd"> >>> f('//example.com', 'https://example.com/')</span>
|
||||
<span class="sd"> 'https://example.com/'</span>
|
||||
<span class="sd"> >>> f('/path?a=1', 'https://example.com')</span>
|
||||
<span class="sd"> 'https://example.com/path?a=1'</span>
|
||||
<span class="sd"> >>> f('', 'https://example.com')</span>
|
||||
<span class="sd"> raise lxml.etree.ParserError</span>
|
||||
<span class="sd"> >>> searx.utils.extract_url([], 'https://example.com')</span>
|
||||
<span class="sd"> raise ValueError</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * ValueError</span>
|
||||
<span class="sd"> * lxml.etree.ParserError</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * str: normalized URL</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">xpath_results</span> <span class="o">==</span> <span class="p">[]:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Empty url resultset'</span><span class="p">)</span>
|
||||
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">xpath_results</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">url</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">normalize_url</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">base_url</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'URL not found'</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="dict_subset"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.dict_subset">[docs]</a><span class="k">def</span> <span class="nf">dict_subset</span><span class="p">(</span><span class="n">dictionary</span><span class="p">:</span> <span class="n">MutableMapping</span><span class="p">,</span> <span class="n">properties</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Extract a subset of a dict</span>
|
||||
|
||||
<span class="sd"> Examples:</span>
|
||||
<span class="sd"> >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])</span>
|
||||
<span class="sd"> {'A': 'a', 'C': 'c'}</span>
|
||||
<span class="sd"> >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])</span>
|
||||
<span class="sd"> {'A': 'a'}</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">dictionary</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">properties</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">dictionary</span><span class="p">}</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_torrent_size"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.get_torrent_size">[docs]</a><span class="k">def</span> <span class="nf">get_torrent_size</span><span class="p">(</span><span class="n">filesize</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">filesize_multiplier</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * filesize (str): size</span>
|
||||
<span class="sd"> * filesize_multiplier (str): TB, GB, .... TiB, GiB...</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * int: number of bytes</span>
|
||||
|
||||
<span class="sd"> Example:</span>
|
||||
<span class="sd"> >>> get_torrent_size('5', 'GB')</span>
|
||||
<span class="sd"> 5368709120</span>
|
||||
<span class="sd"> >>> get_torrent_size('3.14', 'MiB')</span>
|
||||
<span class="sd"> 3140000</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">multiplier</span> <span class="o">=</span> <span class="n">_STORAGE_UNIT_VALUE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">filesize_multiplier</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">filesize</span><span class="p">)</span> <span class="o">*</span> <span class="n">multiplier</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="convert_str_to_int"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.convert_str_to_int">[docs]</a><span class="k">def</span> <span class="nf">convert_str_to_int</span><span class="p">(</span><span class="n">number_str</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Convert number_str to int or 0 if number_str is not a number."""</span>
|
||||
<span class="k">if</span> <span class="n">number_str</span><span class="o">.</span><span class="n">isdigit</span><span class="p">():</span>
|
||||
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">number_str</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="mi">0</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="int_or_zero"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.int_or_zero">[docs]</a><span class="k">def</span> <span class="nf">int_or_zero</span><span class="p">(</span><span class="n">num</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Convert num to int or 0. num can be either a str or a list.</span>
|
||||
<span class="sd"> If num is a list, the first element is converted to int (or return 0 if the list is empty).</span>
|
||||
<span class="sd"> If num is a str, see convert_str_to_int</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">num</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="mi">0</span>
|
||||
<span class="n">num</span> <span class="o">=</span> <span class="n">num</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">convert_str_to_int</span><span class="p">(</span><span class="n">num</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_valid_lang"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.is_valid_lang">[docs]</a><span class="k">def</span> <span class="nf">is_valid_lang</span><span class="p">(</span><span class="n">lang</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Return language code and name if lang describe a language.</span>
|
||||
|
||||
<span class="sd"> Examples:</span>
|
||||
<span class="sd"> >>> is_valid_lang('zz')</span>
|
||||
<span class="sd"> None</span>
|
||||
<span class="sd"> >>> is_valid_lang('uk')</span>
|
||||
<span class="sd"> (True, 'uk', 'ukrainian')</span>
|
||||
<span class="sd"> >>> is_valid_lang(b'uk')</span>
|
||||
<span class="sd"> (True, 'uk', 'ukrainian')</span>
|
||||
<span class="sd"> >>> is_valid_lang('en')</span>
|
||||
<span class="sd"> (True, 'en', 'english')</span>
|
||||
<span class="sd"> >>> searx.utils.is_valid_lang('Español')</span>
|
||||
<span class="sd"> (True, 'es', 'spanish')</span>
|
||||
<span class="sd"> >>> searx.utils.is_valid_lang('Spanish')</span>
|
||||
<span class="sd"> (True, 'es', 'spanish')</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
||||
<span class="n">is_abbr</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">lang</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">is_abbr</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">language_codes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">l</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="n">lang</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">l</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">2</span><span class="p">],</span> <span class="n">l</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">language_codes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">l</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">lang</span> <span class="ow">or</span> <span class="n">l</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">lang</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">l</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">2</span><span class="p">],</span> <span class="n">l</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||
<span class="k">return</span> <span class="kc">None</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_get_lang_to_lc_dict</span><span class="p">(</span><span class="n">lang_list</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="n">key</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">lang_list</span><span class="p">)</span>
|
||||
<span class="n">value</span> <span class="o">=</span> <span class="n">_LANG_TO_LC_CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">value</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">lang_list</span><span class="p">:</span>
|
||||
<span class="n">value</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span> <span class="n">lang</span><span class="p">)</span>
|
||||
<span class="n">_LANG_TO_LC_CACHE</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
|
||||
<span class="k">return</span> <span class="n">value</span>
|
||||
|
||||
|
||||
<span class="c1"># babel's get_global contains all sorts of miscellaneous locale and territory related data</span>
|
||||
<span class="c1"># see get_global in: https://github.com/python-babel/babel/blob/master/babel/core.py</span>
|
||||
<span class="k">def</span> <span class="nf">_get_from_babel</span><span class="p">(</span><span class="n">lang_code</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">key</span><span class="p">):</span>
|
||||
<span class="n">match</span> <span class="o">=</span> <span class="n">get_global</span><span class="p">(</span><span class="n">key</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_code</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">))</span>
|
||||
<span class="c1"># for some keys, such as territory_aliases, match may be a list</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">match</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">match</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">match</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_match_language</span><span class="p">(</span><span class="n">lang_code</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">lang_list</span><span class="o">=</span><span class="p">[],</span> <span class="n">custom_aliases</span><span class="o">=</span><span class="p">{})</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span> <span class="c1"># pylint: disable=W0102</span>
|
||||
<span class="w"> </span><span class="sd">"""auxiliary function to match lang_code in lang_list"""</span>
|
||||
<span class="c1"># replace language code with a custom alias if necessary</span>
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="n">custom_aliases</span><span class="p">:</span>
|
||||
<span class="n">lang_code</span> <span class="o">=</span> <span class="n">custom_aliases</span><span class="p">[</span><span class="n">lang_code</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="n">lang_list</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">lang_code</span>
|
||||
|
||||
<span class="c1"># try to get the most likely country for this language</span>
|
||||
<span class="n">subtags</span> <span class="o">=</span> <span class="n">_get_from_babel</span><span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="s1">'likely_subtags'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">subtags</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">subtags</span> <span class="ow">in</span> <span class="n">lang_list</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">subtags</span>
|
||||
<span class="n">subtag_parts</span> <span class="o">=</span> <span class="n">subtags</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="n">new_code</span> <span class="o">=</span> <span class="n">subtag_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">subtag_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">new_code</span> <span class="ow">in</span> <span class="n">custom_aliases</span><span class="p">:</span>
|
||||
<span class="n">new_code</span> <span class="o">=</span> <span class="n">custom_aliases</span><span class="p">[</span><span class="n">new_code</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">new_code</span> <span class="ow">in</span> <span class="n">lang_list</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">new_code</span>
|
||||
|
||||
<span class="c1"># try to get the any supported country for this language</span>
|
||||
<span class="k">return</span> <span class="n">_get_lang_to_lc_dict</span><span class="p">(</span><span class="n">lang_list</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="match_language"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.match_language">[docs]</a><span class="k">def</span> <span class="nf">match_language</span><span class="p">(</span> <span class="c1"># pylint: disable=W0102</span>
|
||||
<span class="n">locale_code</span><span class="p">,</span> <span class="n">lang_list</span><span class="o">=</span><span class="p">[],</span> <span class="n">custom_aliases</span><span class="o">=</span><span class="p">{},</span> <span class="n">fallback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'en-US'</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""get the language code from lang_list that best matches locale_code"""</span>
|
||||
<span class="c1"># try to get language from given locale_code</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">_match_language</span><span class="p">(</span><span class="n">locale_code</span><span class="p">,</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">language</span>
|
||||
|
||||
<span class="n">locale_parts</span> <span class="o">=</span> <span class="n">locale_code</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="n">lang_code</span> <span class="o">=</span> <span class="n">locale_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># if locale_code has script, try matching without it</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">locale_parts</span><span class="p">)</span> <span class="o">></span> <span class="mi">2</span><span class="p">:</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">_match_language</span><span class="p">(</span><span class="n">lang_code</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">locale_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">language</span>
|
||||
|
||||
<span class="c1"># try to get language using an equivalent country code</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">locale_parts</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="n">country_alias</span> <span class="o">=</span> <span class="n">_get_from_babel</span><span class="p">(</span><span class="n">locale_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="s1">'territory_aliases'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">country_alias</span><span class="p">:</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">_match_language</span><span class="p">(</span><span class="n">lang_code</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">country_alias</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">language</span>
|
||||
|
||||
<span class="c1"># try to get language using an equivalent language code</span>
|
||||
<span class="n">alias</span> <span class="o">=</span> <span class="n">_get_from_babel</span><span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="s1">'language_aliases'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">alias</span><span class="p">:</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">_match_language</span><span class="p">(</span><span class="n">alias</span><span class="p">,</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">language</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="o">!=</span> <span class="n">locale_code</span><span class="p">:</span>
|
||||
<span class="c1"># try to get language from given language without giving the country</span>
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">_match_language</span><span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="n">lang_list</span><span class="p">,</span> <span class="n">custom_aliases</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">language</span> <span class="ow">or</span> <span class="n">fallback</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">load_module</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">module_dir</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">types</span><span class="o">.</span><span class="n">ModuleType</span><span class="p">:</span>
|
||||
<span class="n">modname</span> <span class="o">=</span> <span class="n">splitext</span><span class="p">(</span><span class="n">filename</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">modpath</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">module_dir</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
||||
<span class="c1"># and https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly</span>
|
||||
<span class="n">spec</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">util</span><span class="o">.</span><span class="n">spec_from_file_location</span><span class="p">(</span><span class="n">modname</span><span class="p">,</span> <span class="n">modpath</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">spec</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error loading '</span><span class="si">{</span><span class="n">modpath</span><span class="si">}</span><span class="s2">' module"</span><span class="p">)</span>
|
||||
<span class="n">module</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">util</span><span class="o">.</span><span class="n">module_from_spec</span><span class="p">(</span><span class="n">spec</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">spec</span><span class="o">.</span><span class="n">loader</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error loading '</span><span class="si">{</span><span class="n">modpath</span><span class="si">}</span><span class="s2">' module"</span><span class="p">)</span>
|
||||
<span class="n">spec</span><span class="o">.</span><span class="n">loader</span><span class="o">.</span><span class="n">exec_module</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">module</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="to_string"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.to_string">[docs]</a><span class="k">def</span> <span class="nf">to_string</span><span class="p">(</span><span class="n">obj</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Convert obj to its string representation."""</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">obj</span>
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s1">'__str__'</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">repr</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ecma_unescape"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.ecma_unescape">[docs]</a><span class="k">def</span> <span class="nf">ecma_unescape</span><span class="p">(</span><span class="n">string</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Python implementation of the unescape javascript function</span>
|
||||
|
||||
<span class="sd"> https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string</span>
|
||||
<span class="sd"> https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape</span>
|
||||
|
||||
<span class="sd"> Examples:</span>
|
||||
<span class="sd"> >>> ecma_unescape('%u5409')</span>
|
||||
<span class="sd"> '吉'</span>
|
||||
<span class="sd"> >>> ecma_unescape('%20')</span>
|
||||
<span class="sd"> ' '</span>
|
||||
<span class="sd"> >>> ecma_unescape('%F3')</span>
|
||||
<span class="sd"> 'ó'</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># "%u5409" becomes "吉"</span>
|
||||
<span class="n">string</span> <span class="o">=</span> <span class="n">_ECMA_UNESCAPE4_RE</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="nb">chr</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="mi">16</span><span class="p">)),</span> <span class="n">string</span><span class="p">)</span>
|
||||
<span class="c1"># "%20" becomes " ", "%F3" becomes "ó"</span>
|
||||
<span class="n">string</span> <span class="o">=</span> <span class="n">_ECMA_UNESCAPE2_RE</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="nb">chr</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="mi">16</span><span class="p">)),</span> <span class="n">string</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">string</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_string_replaces_function</span><span class="p">(</span><span class="n">replaces</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="n">rep</span> <span class="o">=</span> <span class="p">{</span><span class="n">re</span><span class="o">.</span><span class="n">escape</span><span class="p">(</span><span class="n">k</span><span class="p">):</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">replaces</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">rep</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">func</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">pattern</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="k">lambda</span> <span class="n">m</span><span class="p">:</span> <span class="n">rep</span><span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">escape</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">))],</span> <span class="n">text</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">func</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_engine_from_settings"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.get_engine_from_settings">[docs]</a><span class="k">def</span> <span class="nf">get_engine_from_settings</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Return engine configuration from settings.yml of a given engine name"""</span>
|
||||
|
||||
<span class="k">if</span> <span class="s1">'engines'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">settings</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="p">{}</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">engine</span> <span class="ow">in</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'engines'</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="s1">'name'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="n">name</span> <span class="o">==</span> <span class="n">engine</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]:</span>
|
||||
<span class="k">return</span> <span class="n">engine</span>
|
||||
|
||||
<span class="k">return</span> <span class="p">{}</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_xpath"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.get_xpath">[docs]</a><span class="k">def</span> <span class="nf">get_xpath</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">:</span> <span class="n">XPathSpecType</span><span class="p">)</span> <span class="o">-></span> <span class="n">XPath</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Return cached compiled XPath</span>
|
||||
|
||||
<span class="sd"> There is no thread lock.</span>
|
||||
<span class="sd"> Worst case scenario, xpath_str is compiled more than one time.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * result (bool, float, list, str): Results.</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath</span>
|
||||
<span class="sd"> * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">_XPATH_CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">XPath</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">XPathSyntaxError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SearxXPathSyntaxException</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">msg</span><span class="p">))</span> <span class="kn">from</span> <span class="nn">e</span>
|
||||
<span class="n">_XPATH_CACHE</span><span class="p">[</span><span class="n">xpath_spec</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span>
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="n">XPath</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">xpath_spec</span>
|
||||
|
||||
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'xpath_spec must be either a str or a lxml.etree.XPath'</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="eval_xpath"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.eval_xpath">[docs]</a><span class="k">def</span> <span class="nf">eval_xpath</span><span class="p">(</span><span class="n">element</span><span class="p">:</span> <span class="n">ElementBase</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="n">XPathSpecType</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.</span>
|
||||
<span class="sd"> See https://lxml.de/xpathxslt.html#xpath-return-values</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * element (ElementBase): [description]</span>
|
||||
<span class="sd"> * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * result (bool, float, list, str): Results.</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath</span>
|
||||
<span class="sd"> * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath</span>
|
||||
<span class="sd"> * SearxEngineXPathException: Raise when the XPath can't be evaluated.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">xpath</span> <span class="o">=</span> <span class="n">get_xpath</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">xpath</span><span class="p">(</span><span class="n">element</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">XPathError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">arg</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="p">])</span>
|
||||
<span class="k">raise</span> <span class="n">SearxEngineXPathException</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="n">arg</span><span class="p">)</span> <span class="kn">from</span> <span class="nn">e</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="eval_xpath_list"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.eval_xpath_list">[docs]</a><span class="k">def</span> <span class="nf">eval_xpath_list</span><span class="p">(</span><span class="n">element</span><span class="p">:</span> <span class="n">ElementBase</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="n">XPathSpecType</span><span class="p">,</span> <span class="n">min_len</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Same as eval_xpath, check if the result is a list</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * element (ElementBase): [description]</span>
|
||||
<span class="sd"> * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath</span>
|
||||
<span class="sd"> * min_len (int, optional): [description]. Defaults to None.</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath</span>
|
||||
<span class="sd"> * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath</span>
|
||||
<span class="sd"> * SearxEngineXPathException: raise if the result is not a list</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * result (bool, float, list, str): Results.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">element</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="n">SearxEngineXPathException</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="s1">'the result is not a list'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">min_len</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">min_len</span> <span class="o">></span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="n">SearxEngineXPathException</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="s1">'len(xpath_str) < '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">min_len</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">result</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="eval_xpath_getindex"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.eval_xpath_getindex">[docs]</a><span class="k">def</span> <span class="nf">eval_xpath_getindex</span><span class="p">(</span><span class="n">elements</span><span class="p">:</span> <span class="n">ElementBase</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="n">XPathSpecType</span><span class="p">,</span> <span class="n">index</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">_NOTSET</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Call eval_xpath_list then get one element using the index parameter.</span>
|
||||
<span class="sd"> If the index does not exist, either aise an exception is default is not set,</span>
|
||||
<span class="sd"> other return the default value (can be None).</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> * elements (ElementBase): lxml element to apply the xpath.</span>
|
||||
<span class="sd"> * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath.</span>
|
||||
<span class="sd"> * index (int): index to get</span>
|
||||
<span class="sd"> * default (Object, optional): Defaults if index doesn't exist.</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath</span>
|
||||
<span class="sd"> * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath</span>
|
||||
<span class="sd"> * SearxEngineXPathException: if the index is not found. Also see eval_xpath.</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> * result (bool, float, list, str): Results.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">elements</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o"><=</span> <span class="n">index</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">result</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">default</span> <span class="o">==</span> <span class="n">_NOTSET</span><span class="p">:</span>
|
||||
<span class="c1"># raise an SearxEngineXPathException instead of IndexError</span>
|
||||
<span class="c1"># to record xpath_spec</span>
|
||||
<span class="k">raise</span> <span class="n">SearxEngineXPathException</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">,</span> <span class="s1">'index '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">index</span><span class="p">)</span> <span class="o">+</span> <span class="s1">' not found'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">default</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_get_fasttext_model</span><span class="p">()</span> <span class="o">-></span> <span class="s2">"fasttext.FastText._FastText"</span><span class="p">:</span>
|
||||
<span class="k">global</span> <span class="n">_FASTTEXT_MODEL</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||
<span class="k">if</span> <span class="n">_FASTTEXT_MODEL</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="kn">import</span> <span class="nn">fasttext</span> <span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||
|
||||
<span class="c1"># Monkey patch: prevent fasttext from showing a (useless) warning when loading a model.</span>
|
||||
<span class="n">fasttext</span><span class="o">.</span><span class="n">FastText</span><span class="o">.</span><span class="n">eprint</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="kc">None</span>
|
||||
<span class="n">_FASTTEXT_MODEL</span> <span class="o">=</span> <span class="n">fasttext</span><span class="o">.</span><span class="n">load_model</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">data_dir</span> <span class="o">/</span> <span class="s1">'lid.176.ftz'</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">_FASTTEXT_MODEL</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="detect_language"><a class="viewcode-back" href="../../src/searx.utils.html#searx.utils.detect_language">[docs]</a><span class="k">def</span> <span class="nf">detect_language</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">threshold</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.3</span><span class="p">,</span> <span class="n">min_probability</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""https://fasttext.cc/docs/en/language-identification.html"""</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'text must a str'</span><span class="p">)</span>
|
||||
<span class="n">r</span> <span class="o">=</span> <span class="n">_get_fasttext_model</span><span class="p">()</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">),</span> <span class="n">k</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="n">threshold</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">r</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="n">min_probability</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'__label__'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="kc">None</span></div>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,329 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searxng_extra.standalone_searx — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/tabs.css" />
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searxng_extra.standalone_searx</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searxng_extra.standalone_searx</h1><div class="highlight"><pre>
|
||||
<span></span><span class="ch">#!/usr/bin/env python</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
|
||||
<span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="c1"># (C) Copyright Contributors to the SearXNG project.</span>
|
||||
<span class="c1"># (C) Copyright Contributors to the searx project (2014 - 2021)</span>
|
||||
|
||||
<span class="sd">"""Script to run SearXNG from terminal.</span>
|
||||
|
||||
<span class="sd">Getting categories without initiate the engine will only return `['general']`</span>
|
||||
|
||||
<span class="sd">>>> import searx.engines</span>
|
||||
<span class="sd">... list(searx.engines.categories.keys())</span>
|
||||
<span class="sd">['general']</span>
|
||||
<span class="sd">>>> import searx.search</span>
|
||||
<span class="sd">... searx.search.initialize()</span>
|
||||
<span class="sd">... list(searx.engines.categories.keys())</span>
|
||||
<span class="sd">['general', 'it', 'science', 'images', 'news', 'videos', 'music', 'files', 'social media', 'map']</span>
|
||||
|
||||
<span class="sd">Example to use this script:</span>
|
||||
|
||||
<span class="sd">.. code:: bash</span>
|
||||
|
||||
<span class="sd"> $ python3 searxng_extra/standalone_searx.py rain</span>
|
||||
|
||||
<span class="sd">.. danger::</span>
|
||||
|
||||
<span class="sd"> Be warned, using the ``standalone_searx.py`` won't give you privacy!</span>
|
||||
|
||||
<span class="sd"> On the contrary, this script behaves like a SearXNG server: your IP is</span>
|
||||
<span class="sd"> exposed and tracked by all active engines (google, bing, qwant, ... ), with</span>
|
||||
<span class="sd"> every query!</span>
|
||||
|
||||
<span class="sd">Example to run it from python:</span>
|
||||
|
||||
<span class="sd">>>> import importlib</span>
|
||||
<span class="sd">... import json</span>
|
||||
<span class="sd">... import sys</span>
|
||||
<span class="sd">... import searx.engines</span>
|
||||
<span class="sd">... import searx.search</span>
|
||||
<span class="sd">... search_query = 'rain'</span>
|
||||
<span class="sd">... # initialize engines</span>
|
||||
<span class="sd">... searx.search.initialize()</span>
|
||||
<span class="sd">... # load engines categories once instead of each time the function called</span>
|
||||
<span class="sd">... engine_cs = list(searx.engines.categories.keys())</span>
|
||||
<span class="sd">... # load module</span>
|
||||
<span class="sd">... spec = importlib.util.spec_from_file_location(</span>
|
||||
<span class="sd">... 'utils.standalone_searx', 'searxng_extra/standalone_searx.py')</span>
|
||||
<span class="sd">... sas = importlib.util.module_from_spec(spec)</span>
|
||||
<span class="sd">... spec.loader.exec_module(sas)</span>
|
||||
<span class="sd">... # use function from module</span>
|
||||
<span class="sd">... prog_args = sas.parse_argument([search_query], category_choices=engine_cs)</span>
|
||||
<span class="sd">... search_q = sas.get_search_query(prog_args, engine_categories=engine_cs)</span>
|
||||
<span class="sd">... res_dict = sas.to_dict(search_q)</span>
|
||||
<span class="sd">... sys.stdout.write(json.dumps(</span>
|
||||
<span class="sd">... res_dict, sort_keys=True, indent=4, ensure_ascii=False,</span>
|
||||
<span class="sd">... default=sas.json_serial))</span>
|
||||
<span class="sd">{</span>
|
||||
<span class="sd"> "answers": [],</span>
|
||||
<span class="sd"> "infoboxes": [ {...} ],</span>
|
||||
<span class="sd"> "paging": true,</span>
|
||||
<span class="sd"> "results": [... ],</span>
|
||||
<span class="sd"> "results_number": 820000000.0,</span>
|
||||
<span class="sd"> "search": {</span>
|
||||
<span class="sd"> "lang": "all",</span>
|
||||
<span class="sd"> "pageno": 1,</span>
|
||||
<span class="sd"> "q": "rain",</span>
|
||||
<span class="sd"> "safesearch": 0,</span>
|
||||
<span class="sd"> "timerange": null</span>
|
||||
<span class="sd"> },</span>
|
||||
<span class="sd"> "suggestions": [...]</span>
|
||||
<span class="sd">}</span>
|
||||
|
||||
<span class="sd">"""</span> <span class="c1"># pylint: disable=line-too-long</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">argparse</span>
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">json</span> <span class="kn">import</span> <span class="n">dumps</span>
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">searx</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.preferences</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.query</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.search</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.webadapter</span>
|
||||
|
||||
<span class="n">EngineCategoriesVar</span> <span class="o">=</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_search_query"><a class="viewcode-back" href="../../dev/searxng_extra/standalone_searx.py.html#searxng_extra.standalone_searx.get_search_query">[docs]</a><span class="k">def</span> <span class="nf">get_search_query</span><span class="p">(</span>
|
||||
<span class="n">args</span><span class="p">:</span> <span class="n">argparse</span><span class="o">.</span><span class="n">Namespace</span><span class="p">,</span> <span class="n">engine_categories</span><span class="p">:</span> <span class="n">EngineCategoriesVar</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">SearchQuery</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Get search results for the query"""</span>
|
||||
<span class="k">if</span> <span class="n">engine_categories</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">engine_categories</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">category</span> <span class="o">=</span> <span class="n">args</span><span class="o">.</span><span class="n">category</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
|
||||
<span class="n">category</span> <span class="o">=</span> <span class="n">args</span><span class="o">.</span><span class="n">category</span>
|
||||
<span class="n">form</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"q"</span><span class="p">:</span> <span class="n">args</span><span class="o">.</span><span class="n">query</span><span class="p">,</span>
|
||||
<span class="s2">"categories"</span><span class="p">:</span> <span class="n">category</span><span class="p">,</span>
|
||||
<span class="s2">"pageno"</span><span class="p">:</span> <span class="nb">str</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">pageno</span><span class="p">),</span>
|
||||
<span class="s2">"language"</span><span class="p">:</span> <span class="n">args</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span>
|
||||
<span class="s2">"time_range"</span><span class="p">:</span> <span class="n">args</span><span class="o">.</span><span class="n">timerange</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">preferences</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">Preferences</span><span class="p">([</span><span class="s1">'simple'</span><span class="p">],</span> <span class="n">engine_categories</span><span class="p">,</span> <span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">,</span> <span class="p">[])</span>
|
||||
<span class="n">preferences</span><span class="o">.</span><span class="n">key_value_settings</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">safesearch</span><span class="p">)</span>
|
||||
|
||||
<span class="n">search_query</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">webadapter</span><span class="o">.</span><span class="n">get_search_query_from_webapp</span><span class="p">(</span><span class="n">preferences</span><span class="p">,</span> <span class="n">form</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">search_query</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="no_parsed_url"><a class="viewcode-back" href="../../dev/searxng_extra/standalone_searx.py.html#searxng_extra.standalone_searx.no_parsed_url">[docs]</a><span class="k">def</span> <span class="nf">no_parsed_url</span><span class="p">(</span><span class="n">results</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]])</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Remove parsed url from dict."""</span>
|
||||
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">results</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="n">result</span><span class="p">[</span><span class="s1">'parsed_url'</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="json_serial"><a class="viewcode-back" href="../../dev/searxng_extra/standalone_searx.py.html#searxng_extra.standalone_searx.json_serial">[docs]</a><span class="k">def</span> <span class="nf">json_serial</span><span class="p">(</span><span class="n">obj</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""JSON serializer for objects not serializable by default json code.</span>
|
||||
|
||||
<span class="sd"> :raise TypeError: raised when **obj** is not serializable</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
|
||||
<span class="n">serial</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="n">serial</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">set</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Type (</span><span class="si">{}</span><span class="s2">) not serializable"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="to_dict"><a class="viewcode-back" href="../../dev/searxng_extra/standalone_searx.py.html#searxng_extra.standalone_searx.to_dict">[docs]</a><span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="n">search_query</span><span class="p">:</span> <span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">SearchQuery</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Get result from parsed arguments."""</span>
|
||||
<span class="n">result_container</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">Search</span><span class="p">(</span><span class="n">search_query</span><span class="p">)</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
|
||||
<span class="n">result_container_json</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"search"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"q"</span><span class="p">:</span> <span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">,</span>
|
||||
<span class="s2">"pageno"</span><span class="p">:</span> <span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="n">search_query</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span>
|
||||
<span class="s2">"safesearch"</span><span class="p">:</span> <span class="n">search_query</span><span class="o">.</span><span class="n">safesearch</span><span class="p">,</span>
|
||||
<span class="s2">"timerange"</span><span class="p">:</span> <span class="n">search_query</span><span class="o">.</span><span class="n">time_range</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="s2">"results"</span><span class="p">:</span> <span class="n">no_parsed_url</span><span class="p">(</span><span class="n">result_container</span><span class="o">.</span><span class="n">get_ordered_results</span><span class="p">()),</span>
|
||||
<span class="s2">"infoboxes"</span><span class="p">:</span> <span class="n">result_container</span><span class="o">.</span><span class="n">infoboxes</span><span class="p">,</span>
|
||||
<span class="s2">"suggestions"</span><span class="p">:</span> <span class="nb">list</span><span class="p">(</span><span class="n">result_container</span><span class="o">.</span><span class="n">suggestions</span><span class="p">),</span>
|
||||
<span class="s2">"answers"</span><span class="p">:</span> <span class="nb">list</span><span class="p">(</span><span class="n">result_container</span><span class="o">.</span><span class="n">answers</span><span class="p">),</span>
|
||||
<span class="s2">"paging"</span><span class="p">:</span> <span class="n">result_container</span><span class="o">.</span><span class="n">paging</span><span class="p">,</span>
|
||||
<span class="s2">"results_number"</span><span class="p">:</span> <span class="n">result_container</span><span class="o">.</span><span class="n">results_number</span><span class="p">(),</span>
|
||||
<span class="p">}</span>
|
||||
<span class="k">return</span> <span class="n">result_container_json</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_argument"><a class="viewcode-back" href="../../dev/searxng_extra/standalone_searx.py.html#searxng_extra.standalone_searx.parse_argument">[docs]</a><span class="k">def</span> <span class="nf">parse_argument</span><span class="p">(</span>
|
||||
<span class="n">args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">category_choices</span><span class="p">:</span> <span class="n">EngineCategoriesVar</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">argparse</span><span class="o">.</span><span class="n">Namespace</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Parse command line.</span>
|
||||
|
||||
<span class="sd"> :raise SystemExit: Query argument required on `args`</span>
|
||||
|
||||
<span class="sd"> Examples:</span>
|
||||
|
||||
<span class="sd"> >>> import importlib</span>
|
||||
<span class="sd"> ... # load module</span>
|
||||
<span class="sd"> ... spec = importlib.util.spec_from_file_location(</span>
|
||||
<span class="sd"> ... 'utils.standalone_searx', 'utils/standalone_searx.py')</span>
|
||||
<span class="sd"> ... sas = importlib.util.module_from_spec(spec)</span>
|
||||
<span class="sd"> ... spec.loader.exec_module(sas)</span>
|
||||
<span class="sd"> ... sas.parse_argument()</span>
|
||||
<span class="sd"> usage: ptipython [-h] [--category [{general}]] [--lang [LANG]] [--pageno [PAGENO]] [--safesearch [{0,1,2}]] [--timerange [{day,week,month,year}]]</span>
|
||||
<span class="sd"> query</span>
|
||||
<span class="sd"> SystemExit: 2</span>
|
||||
<span class="sd"> >>> sas.parse_argument(['rain'])</span>
|
||||
<span class="sd"> Namespace(category='general', lang='all', pageno=1, query='rain', safesearch='0', timerange=None)</span>
|
||||
<span class="sd"> """</span> <span class="c1"># noqa: E501</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">category_choices</span><span class="p">:</span>
|
||||
<span class="n">category_choices</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="n">parser</span> <span class="o">=</span> <span class="n">argparse</span><span class="o">.</span><span class="n">ArgumentParser</span><span class="p">(</span><span class="n">description</span><span class="o">=</span><span class="s1">'Standalone searx.'</span><span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">'query'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s1">'Text query'</span><span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
|
||||
<span class="s1">'--category'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s1">'?'</span><span class="p">,</span> <span class="n">choices</span><span class="o">=</span><span class="n">category_choices</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">'general'</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s1">'Search category'</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">'--lang'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s1">'?'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">'all'</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s1">'Search language'</span><span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">'--pageno'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s1">'?'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s1">'Page number starting from 1'</span><span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
|
||||
<span class="s1">'--safesearch'</span><span class="p">,</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">nargs</span><span class="o">=</span><span class="s1">'?'</span><span class="p">,</span>
|
||||
<span class="n">choices</span><span class="o">=</span><span class="p">[</span><span class="s1">'0'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">,</span> <span class="s1">'2'</span><span class="p">],</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s1">'0'</span><span class="p">,</span>
|
||||
<span class="n">help</span><span class="o">=</span><span class="s1">'Safe content filter from none to strict'</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
|
||||
<span class="s1">'--timerange'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s1">'?'</span><span class="p">,</span> <span class="n">choices</span><span class="o">=</span><span class="p">[</span><span class="s1">'day'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">],</span> <span class="n">help</span><span class="o">=</span><span class="s1">'Filter by time range'</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">(</span><span class="n">args</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||||
<span class="n">settings_engines</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">settings</span><span class="p">[</span><span class="s1">'engines'</span><span class="p">]</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">load_engines</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">)</span>
|
||||
<span class="n">engine_cs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="n">prog_args</span> <span class="o">=</span> <span class="n">parse_argument</span><span class="p">(</span><span class="n">category_choices</span><span class="o">=</span><span class="n">engine_cs</span><span class="p">)</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">initialize_network</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">,</span> <span class="n">searx</span><span class="o">.</span><span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">])</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">check_network_configuration</span><span class="p">()</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">initialize_metrics</span><span class="p">([</span><span class="n">engine</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="k">for</span> <span class="n">engine</span> <span class="ow">in</span> <span class="n">settings_engines</span><span class="p">])</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">initialize_processors</span><span class="p">(</span><span class="n">settings_engines</span><span class="p">)</span>
|
||||
<span class="n">search_q</span> <span class="o">=</span> <span class="n">get_search_query</span><span class="p">(</span><span class="n">prog_args</span><span class="p">,</span> <span class="n">engine_categories</span><span class="o">=</span><span class="n">engine_cs</span><span class="p">)</span>
|
||||
<span class="n">res_dict</span> <span class="o">=</span> <span class="n">to_dict</span><span class="p">(</span><span class="n">search_q</span><span class="p">)</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">dumps</span><span class="p">(</span><span class="n">res_dict</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">json_serial</span><span class="p">))</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../index.html">
|
||||
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,420 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searxng_extra.update.update_engine_descriptions — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searxng_extra.update.update_engine_descriptions</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searxng_extra.update.update_engine_descriptions</h1><div class="highlight"><pre>
|
||||
<span></span><span class="ch">#!/usr/bin/env python</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
|
||||
<span class="sd">"""Fetch website description from websites and from</span>
|
||||
<span class="sd">:origin:`searx/engines/wikidata.py` engine.</span>
|
||||
|
||||
<span class="sd">Output file: :origin:`searx/data/engine_descriptions.json`.</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="c1"># pylint: disable=invalid-name, global-statement</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
|
||||
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">join</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">lxml.html</span> <span class="kn">import</span> <span class="n">fromstring</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx.engines</span> <span class="kn">import</span> <span class="n">wikidata</span><span class="p">,</span> <span class="n">set_loggers</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">match_language</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.locales</span> <span class="kn">import</span> <span class="n">LOCALE_NAMES</span><span class="p">,</span> <span class="n">locales_initialize</span>
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">searx_dir</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">gen_useragent</span><span class="p">,</span> <span class="n">detect_language</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.search</span>
|
||||
<span class="kn">import</span> <span class="nn">searx.network</span>
|
||||
|
||||
<span class="n">set_loggers</span><span class="p">(</span><span class="n">wikidata</span><span class="p">,</span> <span class="s1">'wikidata'</span><span class="p">)</span>
|
||||
<span class="n">locales_initialize</span><span class="p">()</span>
|
||||
|
||||
<span class="n">SPARQL_WIKIPEDIA_ARTICLE</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||
<span class="s2">SELECT DISTINCT ?item ?name</span>
|
||||
<span class="s2">WHERE {</span>
|
||||
<span class="s2"> hint:Query hint:optimizer "None".</span>
|
||||
<span class="s2"> VALUES ?item { %IDS% }</span>
|
||||
<span class="s2"> ?article schema:about ?item ;</span>
|
||||
<span class="s2"> schema:inLanguage ?lang ;</span>
|
||||
<span class="s2"> schema:name ?name ;</span>
|
||||
<span class="s2"> schema:isPartOf [ wikibase:wikiGroup "wikipedia" ] .</span>
|
||||
<span class="s2"> FILTER(?lang in (%LANGUAGES_SPARQL%)) .</span>
|
||||
<span class="s2"> FILTER (!CONTAINS(?name, ':')) .</span>
|
||||
<span class="s2">}</span>
|
||||
<span class="s2">"""</span>
|
||||
|
||||
<span class="n">SPARQL_DESCRIPTION</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||
<span class="s2">SELECT DISTINCT ?item ?itemDescription</span>
|
||||
<span class="s2">WHERE {</span>
|
||||
<span class="s2"> VALUES ?item { %IDS% }</span>
|
||||
<span class="s2"> ?item schema:description ?itemDescription .</span>
|
||||
<span class="s2"> FILTER (lang(?itemDescription) in (%LANGUAGES_SPARQL%))</span>
|
||||
<span class="s2">}</span>
|
||||
<span class="s2">ORDER BY ?itemLang</span>
|
||||
<span class="s2">"""</span>
|
||||
|
||||
<span class="n">NOT_A_DESCRIPTION</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s1">'web site'</span><span class="p">,</span>
|
||||
<span class="s1">'site web'</span><span class="p">,</span>
|
||||
<span class="s1">'komputa serĉilo'</span><span class="p">,</span>
|
||||
<span class="s1">'interreta serĉilo'</span><span class="p">,</span>
|
||||
<span class="s1">'bilaketa motor'</span><span class="p">,</span>
|
||||
<span class="s1">'web search engine'</span><span class="p">,</span>
|
||||
<span class="s1">'wikimedia täpsustuslehekülg'</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="n">SKIP_ENGINE_SOURCE</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="c1"># fmt: off</span>
|
||||
<span class="p">(</span><span class="s1">'gitlab'</span><span class="p">,</span> <span class="s1">'wikidata'</span><span class="p">)</span>
|
||||
<span class="c1"># descriptions are about wikipedia disambiguation pages</span>
|
||||
<span class="c1"># fmt: on</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="n">LANGUAGES</span> <span class="o">=</span> <span class="n">LOCALE_NAMES</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||||
<span class="n">WIKIPEDIA_LANGUAGES</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'language'</span><span class="p">:</span> <span class="s1">'wikipedia_language'</span><span class="p">}</span>
|
||||
<span class="n">LANGUAGES_SPARQL</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="n">IDS</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">descriptions</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">wd_to_engine_name</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">normalize_description</span><span class="p">(</span><span class="n">description</span><span class="p">):</span>
|
||||
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="p">[</span><span class="nb">chr</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">31</span><span class="p">)]:</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">description</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">description</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
|
||||
<span class="k">return</span> <span class="n">description</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">update_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">description</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">normalize_description</span><span class="p">(</span><span class="n">description</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">description</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="n">description</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="n">NOT_A_DESCRIPTION</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">source</span><span class="p">)</span> <span class="ow">in</span> <span class="n">SKIP_ENGINE_SOURCE</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="s1">' '</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">description</span><span class="p">:</span>
|
||||
<span class="c1"># skip unique word description (like "website")</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="n">replace</span> <span class="ow">or</span> <span class="n">lang</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">descriptions</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]:</span>
|
||||
<span class="n">descriptions</span><span class="p">[</span><span class="n">engine_name</span><span class="p">][</span><span class="n">lang</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">description</span><span class="p">,</span> <span class="n">source</span><span class="p">]</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_wikipedia_summary</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="n">pageid</span><span class="p">):</span>
|
||||
<span class="n">params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'language'</span><span class="p">:</span> <span class="n">lang</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">),</span> <span class="s1">'headers'</span><span class="p">:</span> <span class="p">{}}</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">]</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="n">pageid</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span> <span class="n">headers</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">],</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
<span class="n">api_result</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'extract'</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_website_description</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">lang1</span><span class="p">,</span> <span class="n">lang2</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'User-Agent'</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
|
||||
<span class="s1">'Accept'</span><span class="p">:</span> <span class="s1">'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'</span><span class="p">,</span>
|
||||
<span class="s1">'DNT'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||
<span class="s1">'Upgrade-Insecure-Requests'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||
<span class="s1">'Sec-GPC'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||
<span class="s1">'Cache-Control'</span><span class="p">:</span> <span class="s1">'max-age=0'</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">lang1</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">lang_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">lang1</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">lang2</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">lang_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lang2</span><span class="p">)</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s1">'Accept-Language'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">lang_list</span><span class="p">)</span><span class="si">}</span><span class="s1">;q=0.8'</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||
<span class="k">return</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">html</span> <span class="o">=</span> <span class="n">fromstring</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||
<span class="n">html</span> <span class="o">=</span> <span class="n">fromstring</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||||
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">html</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'/html/head/meta[@name="description"]/@content'</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">description</span><span class="p">:</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">html</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'/html/head/meta[@property="og:description"]/@content'</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">description</span><span class="p">:</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">html</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'/html/head/title'</span><span class="p">))</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">html</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'/html/@lang'</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">lang</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">lang1</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang1</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">detect_language</span><span class="p">(</span><span class="n">description</span><span class="p">)</span> <span class="ow">or</span> <span class="n">lang</span> <span class="ow">or</span> <span class="s1">'en'</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="n">description</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">initialize</span><span class="p">():</span>
|
||||
<span class="k">global</span> <span class="n">IDS</span><span class="p">,</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="p">,</span> <span class="n">LANGUAGES_SPARQL</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">search</span><span class="o">.</span><span class="n">initialize</span><span class="p">()</span>
|
||||
<span class="n">wikipedia_engine</span> <span class="o">=</span> <span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">]</span>
|
||||
<span class="n">WIKIPEDIA_LANGUAGES</span> <span class="o">=</span> <span class="p">{</span><span class="n">language</span><span class="p">:</span> <span class="n">wikipedia_engine</span><span class="o">.</span><span class="n">url_lang</span><span class="p">(</span><span class="n">language</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">))</span> <span class="k">for</span> <span class="n">language</span> <span class="ow">in</span> <span class="n">LANGUAGES</span><span class="p">}</span>
|
||||
<span class="n">WIKIPEDIA_LANGUAGES</span><span class="p">[</span><span class="s1">'nb_NO'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'no'</span>
|
||||
<span class="n">LANGUAGES_SPARQL</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="n">l</span><span class="si">}</span><span class="s2">'"</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="nb">set</span><span class="p">(</span><span class="n">WIKIPEDIA_LANGUAGES</span><span class="o">.</span><span class="n">values</span><span class="p">()))</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span><span class="p">,</span> <span class="n">engine</span> <span class="ow">in</span> <span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">descriptions</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">wikidata_id</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"about"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'wikidata_id'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">wikidata_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">wd_to_engine_name</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">wikidata_id</span><span class="p">,</span> <span class="nb">set</span><span class="p">())</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||
|
||||
<span class="n">IDS</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">wd_id</span><span class="p">:</span> <span class="s1">'wd:'</span> <span class="o">+</span> <span class="n">wd_id</span><span class="p">,</span> <span class="n">wd_to_engine_name</span><span class="o">.</span><span class="n">keys</span><span class="p">())))</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">fetch_wikidata_descriptions</span><span class="p">():</span>
|
||||
<span class="n">searx</span><span class="o">.</span><span class="n">network</span><span class="o">.</span><span class="n">set_timeout_for_thread</span><span class="p">(</span><span class="mi">60</span><span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">wikidata</span><span class="o">.</span><span class="n">send_wikidata_query</span><span class="p">(</span>
|
||||
<span class="n">SPARQL_DESCRIPTION</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%IDS%'</span><span class="p">,</span> <span class="n">IDS</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%LANGUAGES_SPARQL%'</span><span class="p">,</span> <span class="n">LANGUAGES_SPARQL</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">binding</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s1">'results'</span><span class="p">][</span><span class="s1">'bindings'</span><span class="p">]:</span>
|
||||
<span class="n">wikidata_id</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'item'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'http://www.wikidata.org/entity/'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="n">wikidata_lang</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'itemDescription'</span><span class="p">][</span><span class="s1">'xml:lang'</span><span class="p">]</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'itemDescription'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">wd_to_engine_name</span><span class="p">[</span><span class="n">wikidata_id</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">LANGUAGES</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="p">[</span><span class="n">lang</span><span class="p">]</span> <span class="o">==</span> <span class="n">wikidata_lang</span><span class="p">:</span>
|
||||
<span class="n">update_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">description</span><span class="p">,</span> <span class="s1">'wikidata'</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">fetch_wikipedia_descriptions</span><span class="p">():</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">wikidata</span><span class="o">.</span><span class="n">send_wikidata_query</span><span class="p">(</span>
|
||||
<span class="n">SPARQL_WIKIPEDIA_ARTICLE</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%IDS%'</span><span class="p">,</span> <span class="n">IDS</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%LANGUAGES_SPARQL%'</span><span class="p">,</span> <span class="n">LANGUAGES_SPARQL</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">binding</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s1">'results'</span><span class="p">][</span><span class="s1">'bindings'</span><span class="p">]:</span>
|
||||
<span class="n">wikidata_id</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'item'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'http://www.wikidata.org/entity/'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="n">wikidata_lang</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'name'</span><span class="p">][</span><span class="s1">'xml:lang'</span><span class="p">]</span>
|
||||
<span class="n">pageid</span> <span class="o">=</span> <span class="n">binding</span><span class="p">[</span><span class="s1">'name'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">wd_to_engine_name</span><span class="p">[</span><span class="n">wikidata_id</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">LANGUAGES</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="p">[</span><span class="n">lang</span><span class="p">]</span> <span class="o">==</span> <span class="n">wikidata_lang</span><span class="p">:</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">get_wikipedia_summary</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="n">pageid</span><span class="p">)</span>
|
||||
<span class="n">update_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">description</span><span class="p">,</span> <span class="s1">'wikipedia'</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">normalize_url</span><span class="p">(</span><span class="n">url</span><span class="p">):</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{language}</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span><span class="n">path</span><span class="o">=</span><span class="s1">'/'</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">query</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">fragment</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'https://api.'</span><span class="p">,</span> <span class="s1">'https://'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">url</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">fetch_website_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">website</span><span class="p">):</span>
|
||||
<span class="n">default_lang</span><span class="p">,</span> <span class="n">default_description</span> <span class="o">=</span> <span class="n">get_website_description</span><span class="p">(</span><span class="n">website</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">default_lang</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">default_description</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="c1"># the front page can't be fetched: skip this engine</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">wikipedia_languages_r</span> <span class="o">=</span> <span class="p">{</span><span class="n">V</span><span class="p">:</span> <span class="n">K</span> <span class="k">for</span> <span class="n">K</span><span class="p">,</span> <span class="n">V</span> <span class="ow">in</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||
<span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'en'</span><span class="p">,</span> <span class="s1">'es'</span><span class="p">,</span> <span class="s1">'pt'</span><span class="p">,</span> <span class="s1">'ru'</span><span class="p">,</span> <span class="s1">'tr'</span><span class="p">,</span> <span class="s1">'fr'</span><span class="p">]</span>
|
||||
<span class="n">languages</span> <span class="o">=</span> <span class="n">languages</span> <span class="o">+</span> <span class="p">[</span><span class="n">l</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">LANGUAGES</span> <span class="k">if</span> <span class="n">l</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">]</span>
|
||||
|
||||
<span class="n">previous_matched_lang</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">previous_count</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">lang</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">descriptions</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]:</span>
|
||||
<span class="n">fetched_lang</span><span class="p">,</span> <span class="n">desc</span> <span class="o">=</span> <span class="n">get_website_description</span><span class="p">(</span><span class="n">website</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="p">[</span><span class="n">lang</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">fetched_lang</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">desc</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">matched_lang</span> <span class="o">=</span> <span class="n">match_language</span><span class="p">(</span><span class="n">fetched_lang</span><span class="p">,</span> <span class="n">LANGUAGES</span><span class="p">,</span> <span class="n">fallback</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">matched_lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">fetched_wikipedia_lang</span> <span class="o">=</span> <span class="n">match_language</span><span class="p">(</span><span class="n">fetched_lang</span><span class="p">,</span> <span class="n">WIKIPEDIA_LANGUAGES</span><span class="o">.</span><span class="n">values</span><span class="p">(),</span> <span class="n">fallback</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">matched_lang</span> <span class="o">=</span> <span class="n">wikipedia_languages_r</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">fetched_wikipedia_lang</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">matched_lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">update_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">matched_lang</span><span class="p">,</span> <span class="n">desc</span><span class="p">,</span> <span class="n">website</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="c1"># check if desc changed with the different lang values</span>
|
||||
<span class="k">if</span> <span class="n">matched_lang</span> <span class="o">==</span> <span class="n">previous_matched_lang</span><span class="p">:</span>
|
||||
<span class="n">previous_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="k">if</span> <span class="n">previous_count</span> <span class="o">==</span> <span class="mi">6</span><span class="p">:</span>
|
||||
<span class="c1"># the website has returned the same description for 6 different languages in Accept-Language header</span>
|
||||
<span class="c1"># stop now</span>
|
||||
<span class="k">break</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">previous_matched_lang</span> <span class="o">=</span> <span class="n">matched_lang</span>
|
||||
<span class="n">previous_count</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">fetch_website_descriptions</span><span class="p">():</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span><span class="p">,</span> <span class="n">engine</span> <span class="ow">in</span> <span class="n">searx</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">website</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"about"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'website'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">website</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"search_url"</span><span class="p">):</span>
|
||||
<span class="n">website</span> <span class="o">=</span> <span class="n">normalize_url</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"search_url"</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">website</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"base_url"</span><span class="p">):</span>
|
||||
<span class="n">website</span> <span class="o">=</span> <span class="n">normalize_url</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"base_url"</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">website</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">fetch_website_description</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">website</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_engine_descriptions_filename</span><span class="p">():</span>
|
||||
<span class="k">return</span> <span class="n">join</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">,</span> <span class="s2">"data"</span><span class="p">),</span> <span class="s2">"engine_descriptions.json"</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_output"><a class="viewcode-back" href="../../../dev/searxng_extra/update.html#searxng_extra.update.update_engine_descriptions.get_output">[docs]</a><span class="k">def</span> <span class="nf">get_output</span><span class="p">():</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> From descriptions[engine][language] = [description, source]</span>
|
||||
<span class="sd"> To</span>
|
||||
|
||||
<span class="sd"> * output[language][engine] = description_and_source</span>
|
||||
<span class="sd"> * description_and_source can be:</span>
|
||||
<span class="sd"> * [description, source]</span>
|
||||
<span class="sd"> * description (if source = "wikipedia")</span>
|
||||
<span class="sd"> * [f"engine:lang", "ref"] (reference to another existing description)</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">output</span> <span class="o">=</span> <span class="p">{</span><span class="n">locale</span><span class="p">:</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">LOCALE_NAMES</span><span class="p">}</span>
|
||||
|
||||
<span class="n">seen_descriptions</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">engine_name</span><span class="p">,</span> <span class="n">lang_descriptions</span> <span class="ow">in</span> <span class="n">descriptions</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">for</span> <span class="n">language</span><span class="p">,</span> <span class="n">description</span> <span class="ow">in</span> <span class="n">lang_descriptions</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="n">description</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">in</span> <span class="n">seen_descriptions</span><span class="p">:</span>
|
||||
<span class="n">ref</span> <span class="o">=</span> <span class="n">seen_descriptions</span><span class="p">[</span><span class="n">description</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="p">[</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">ref</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s1">:</span><span class="si">{</span><span class="n">ref</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'ref'</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">seen_descriptions</span><span class="p">[</span><span class="n">description</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">=</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">description</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'wikipedia'</span><span class="p">:</span>
|
||||
<span class="n">description</span> <span class="o">=</span> <span class="n">description</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">language</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">description</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">output</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||||
<span class="n">initialize</span><span class="p">()</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Fetching wikidata descriptions'</span><span class="p">)</span>
|
||||
<span class="n">fetch_wikidata_descriptions</span><span class="p">()</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Fetching wikipedia descriptions'</span><span class="p">)</span>
|
||||
<span class="n">fetch_wikipedia_descriptions</span><span class="p">()</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Fetching website descriptions'</span><span class="p">)</span>
|
||||
<span class="n">fetch_website_descriptions</span><span class="p">()</span>
|
||||
|
||||
<span class="n">output</span> <span class="o">=</span> <span class="n">get_output</span><span class="p">()</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">get_engine_descriptions_filename</span><span class="p">(),</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">separators</span><span class="o">=</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">':'</span><span class="p">),</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
|
||||
|
||||
|
||||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
||||
<span class="n">main</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,278 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searxng_extra.update.update_external_bangs — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searxng_extra.update.update_external_bangs</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searxng_extra.update.update_external_bangs</h1><div class="highlight"><pre>
|
||||
<span></span><span class="ch">#!/usr/bin/env python</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
<span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="sd">"""Update :origin:`searx/data/external_bangs.json` using the duckduckgo bangs</span>
|
||||
<span class="sd">(:origin:`CI Update data ... <.github/workflows/data-update.yml>`).</span>
|
||||
|
||||
<span class="sd">https://duckduckgo.com/newbang loads:</span>
|
||||
|
||||
<span class="sd">* a javascript which provides the bang version ( https://duckduckgo.com/bv1.js )</span>
|
||||
<span class="sd">* a JSON file which contains the bangs ( https://duckduckgo.com/bang.v260.js for example )</span>
|
||||
|
||||
<span class="sd">This script loads the javascript, then the bangs.</span>
|
||||
|
||||
<span class="sd">The javascript URL may change in the future ( for example</span>
|
||||
<span class="sd">https://duckduckgo.com/bv2.js ), but most probably it will requires to update</span>
|
||||
<span class="sd">RE_BANG_VERSION</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
<span class="c1"># pylint: disable=C0116</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">join</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">httpx</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">searx_dir</span> <span class="c1"># pylint: disable=E0401 C0413</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.external_bang</span> <span class="kn">import</span> <span class="n">LEAF_KEY</span>
|
||||
|
||||
<span class="c1"># from https://duckduckgo.com/newbang</span>
|
||||
<span class="n">URL_BV1</span> <span class="o">=</span> <span class="s1">'https://duckduckgo.com/bv1.js'</span>
|
||||
<span class="n">RE_BANG_VERSION</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\/bang\.v([0-9]+)\.js'</span><span class="p">)</span>
|
||||
<span class="n">HTTPS_COLON</span> <span class="o">=</span> <span class="s1">'https:'</span>
|
||||
<span class="n">HTTP_COLON</span> <span class="o">=</span> <span class="s1">'http:'</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_bang_url</span><span class="p">():</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">URL_BV1</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
|
||||
<span class="n">r</span> <span class="o">=</span> <span class="n">RE_BANG_VERSION</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s1">'https://duckduckgo.com/bang.v</span><span class="si">{</span><span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s1">.js'</span><span class="p">,</span> <span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">fetch_ddg_bangs</span><span class="p">(</span><span class="n">url</span><span class="p">):</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="merge_when_no_leaf"><a class="viewcode-back" href="../../../dev/searxng_extra/update.html#searxng_extra.update.update_external_bangs.merge_when_no_leaf">[docs]</a><span class="k">def</span> <span class="nf">merge_when_no_leaf</span><span class="p">(</span><span class="n">node</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Minimize the number of nodes</span>
|
||||
|
||||
<span class="sd"> ``A -> B -> C``</span>
|
||||
|
||||
<span class="sd"> - ``B`` is child of ``A``</span>
|
||||
<span class="sd"> - ``C`` is child of ``B``</span>
|
||||
|
||||
<span class="sd"> If there are no ``C`` equals to ``<LEAF_KEY>``, then each ``C`` are merged</span>
|
||||
<span class="sd"> into ``A``. For example (5 nodes)::</span>
|
||||
|
||||
<span class="sd"> d -> d -> g -> <LEAF_KEY> (ddg)</span>
|
||||
<span class="sd"> -> i -> g -> <LEAF_KEY> (dig)</span>
|
||||
|
||||
<span class="sd"> becomes (3 noodes)::</span>
|
||||
|
||||
<span class="sd"> d -> dg -> <LEAF_KEY></span>
|
||||
<span class="sd"> -> ig -> <LEAF_KEY></span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">restart</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">node</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="c1"># create a copy of the keys so node can be modified</span>
|
||||
<span class="n">keys</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">node</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">keys</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="n">LEAF_KEY</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">value</span> <span class="o">=</span> <span class="n">node</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="n">value_keys</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">value</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">LEAF_KEY</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">value_keys</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">value_key</span> <span class="ow">in</span> <span class="n">value_keys</span><span class="p">:</span>
|
||||
<span class="n">node</span><span class="p">[</span><span class="n">key</span> <span class="o">+</span> <span class="n">value_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span><span class="p">[</span><span class="n">value_key</span><span class="p">]</span>
|
||||
<span class="n">merge_when_no_leaf</span><span class="p">(</span><span class="n">node</span><span class="p">[</span><span class="n">key</span> <span class="o">+</span> <span class="n">value_key</span><span class="p">])</span>
|
||||
<span class="k">del</span> <span class="n">node</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="n">restart</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">merge_when_no_leaf</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">restart</span><span class="p">:</span>
|
||||
<span class="n">merge_when_no_leaf</span><span class="p">(</span><span class="n">node</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">optimize_leaf</span><span class="p">(</span><span class="n">parent</span><span class="p">,</span> <span class="n">parent_key</span><span class="p">,</span> <span class="n">node</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">node</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">node</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">LEAF_KEY</span> <span class="ow">in</span> <span class="n">node</span> <span class="ow">and</span> <span class="n">parent</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">parent</span><span class="p">[</span><span class="n">parent_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">node</span><span class="p">[</span><span class="n">LEAF_KEY</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">node</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">optimize_leaf</span><span class="p">(</span><span class="n">node</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">parse_ddg_bangs</span><span class="p">(</span><span class="n">ddg_bangs</span><span class="p">):</span>
|
||||
<span class="n">bang_trie</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">bang_urls</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">bang_definition</span> <span class="ow">in</span> <span class="n">ddg_bangs</span><span class="p">:</span>
|
||||
<span class="c1"># bang_list</span>
|
||||
<span class="n">bang_url</span> <span class="o">=</span> <span class="n">bang_definition</span><span class="p">[</span><span class="s1">'u'</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s1">'{{</span><span class="si">{s}</span><span class="s1">}}'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">bang_url</span><span class="p">:</span>
|
||||
<span class="c1"># ignore invalid bang</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">bang_url</span> <span class="o">=</span> <span class="n">bang_url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'{{</span><span class="si">{s}</span><span class="s1">}}'</span><span class="p">,</span> <span class="nb">chr</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># only for the https protocol: "https://example.com" becomes "//example.com"</span>
|
||||
<span class="k">if</span> <span class="n">bang_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">HTTPS_COLON</span> <span class="o">+</span> <span class="s1">'//'</span><span class="p">):</span>
|
||||
<span class="n">bang_url</span> <span class="o">=</span> <span class="n">bang_url</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">HTTPS_COLON</span><span class="p">)</span> <span class="p">:]</span>
|
||||
|
||||
<span class="c1">#</span>
|
||||
<span class="k">if</span> <span class="n">bang_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">HTTP_COLON</span> <span class="o">+</span> <span class="s1">'//'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">bang_url</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">HTTP_COLON</span><span class="p">)</span> <span class="p">:]</span> <span class="ow">in</span> <span class="n">bang_urls</span><span class="p">:</span>
|
||||
<span class="c1"># if the bang_url uses the http:// protocol, and the same URL exists in https://</span>
|
||||
<span class="c1"># then reuse the https:// bang definition. (written //example.com)</span>
|
||||
<span class="n">bang_def_output</span> <span class="o">=</span> <span class="n">bang_urls</span><span class="p">[</span><span class="n">bang_url</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">HTTP_COLON</span><span class="p">)</span> <span class="p">:]]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="c1"># normal use case : new http:// URL or https:// URL (without "https:", see above)</span>
|
||||
<span class="n">bang_rank</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">bang_definition</span><span class="p">[</span><span class="s1">'r'</span><span class="p">])</span>
|
||||
<span class="n">bang_def_output</span> <span class="o">=</span> <span class="n">bang_url</span> <span class="o">+</span> <span class="nb">chr</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="n">bang_rank</span>
|
||||
<span class="n">bang_def_output</span> <span class="o">=</span> <span class="n">bang_urls</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">bang_url</span><span class="p">,</span> <span class="n">bang_def_output</span><span class="p">)</span>
|
||||
|
||||
<span class="n">bang_urls</span><span class="p">[</span><span class="n">bang_url</span><span class="p">]</span> <span class="o">=</span> <span class="n">bang_def_output</span>
|
||||
|
||||
<span class="c1"># bang name</span>
|
||||
<span class="n">bang</span> <span class="o">=</span> <span class="n">bang_definition</span><span class="p">[</span><span class="s1">'t'</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># bang_trie</span>
|
||||
<span class="n">t</span> <span class="o">=</span> <span class="n">bang_trie</span>
|
||||
<span class="k">for</span> <span class="n">bang_letter</span> <span class="ow">in</span> <span class="n">bang</span><span class="p">:</span>
|
||||
<span class="n">t</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">bang_letter</span><span class="p">,</span> <span class="p">{})</span>
|
||||
<span class="n">t</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">LEAF_KEY</span><span class="p">,</span> <span class="n">bang_def_output</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># optimize the trie</span>
|
||||
<span class="n">merge_when_no_leaf</span><span class="p">(</span><span class="n">bang_trie</span><span class="p">)</span>
|
||||
<span class="n">optimize_leaf</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">bang_trie</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">bang_trie</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_bangs_filename</span><span class="p">():</span>
|
||||
<span class="k">return</span> <span class="n">join</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">,</span> <span class="s2">"data"</span><span class="p">),</span> <span class="s2">"external_bangs.json"</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||||
<span class="n">bangs_url</span><span class="p">,</span> <span class="n">bangs_version</span> <span class="o">=</span> <span class="n">get_bang_url</span><span class="p">()</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'fetch bangs from </span><span class="si">{</span><span class="n">bangs_url</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||
<span class="n">output</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'version'</span><span class="p">:</span> <span class="n">bangs_version</span><span class="p">,</span> <span class="s1">'trie'</span><span class="p">:</span> <span class="n">parse_ddg_bangs</span><span class="p">(</span><span class="n">fetch_ddg_bangs</span><span class="p">(</span><span class="n">bangs_url</span><span class="p">))}</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">get_bangs_filename</span><span class="p">(),</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">fp</span><span class="p">:</span>
|
||||
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,428 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>searxng_extra.update.update_languages — SearXNG Documentation (2023.1.23+522ba9a1)</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/tabs.css" />
|
||||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
|
||||
<script src="../../../_static/jquery.js"></script>
|
||||
<script src="../../../_static/underscore.js"></script>
|
||||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../../_static/doctools.js"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../../_static/tabs.js"></script>
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
</head><body>
|
||||
<div class="related" role="navigation" aria-label="related navigation">
|
||||
<h3>Navigation</h3>
|
||||
<ul>
|
||||
<li class="right" style="margin-right: 10px">
|
||||
<a href="../../../genindex.html" title="General Index"
|
||||
accesskey="I">index</a></li>
|
||||
<li class="right" >
|
||||
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||
>modules</a> |</li>
|
||||
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2023.1.23+522ba9a1)</a> »</li>
|
||||
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||
<li class="nav-item nav-item-this"><a href="">searxng_extra.update.update_languages</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="document">
|
||||
<div class="documentwrapper">
|
||||
<div class="bodywrapper">
|
||||
<div class="body" role="main">
|
||||
|
||||
<h1>Source code for searxng_extra.update.update_languages</h1><div class="highlight"><pre>
|
||||
<span></span><span class="ch">#!/usr/bin/env python</span>
|
||||
<span class="c1"># lint: pylint</span>
|
||||
|
||||
<span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||
<span class="sd">"""This script generates languages.py from intersecting each engine's supported</span>
|
||||
<span class="sd">languages.</span>
|
||||
|
||||
<span class="sd">Output files: :origin:`searx/data/engines_languages.json` and</span>
|
||||
<span class="sd">:origin:`searx/languages.py` (:origin:`CI Update data ...</span>
|
||||
<span class="sd"><.github/workflows/data-update.yml>`).</span>
|
||||
|
||||
<span class="sd">"""</span>
|
||||
|
||||
<span class="c1"># pylint: disable=invalid-name</span>
|
||||
<span class="kn">from</span> <span class="nn">unicodedata</span> <span class="kn">import</span> <span class="n">lookup</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">from</span> <span class="nn">pathlib</span> <span class="kn">import</span> <span class="n">Path</span>
|
||||
<span class="kn">from</span> <span class="nn">pprint</span> <span class="kn">import</span> <span class="n">pformat</span>
|
||||
<span class="kn">from</span> <span class="nn">babel</span> <span class="kn">import</span> <span class="n">Locale</span><span class="p">,</span> <span class="n">UnknownLocaleError</span>
|
||||
<span class="kn">from</span> <span class="nn">babel.languages</span> <span class="kn">import</span> <span class="n">get_global</span>
|
||||
<span class="kn">from</span> <span class="nn">babel.core</span> <span class="kn">import</span> <span class="n">parse_locale</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">searx_dir</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.engines</span> <span class="kn">import</span> <span class="n">load_engines</span><span class="p">,</span> <span class="n">engines</span>
|
||||
<span class="kn">from</span> <span class="nn">searx.network</span> <span class="kn">import</span> <span class="n">set_timeout_for_thread</span>
|
||||
|
||||
<span class="c1"># Output files.</span>
|
||||
<span class="n">engines_languages_file</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">)</span> <span class="o">/</span> <span class="s1">'data'</span> <span class="o">/</span> <span class="s1">'engines_languages.json'</span>
|
||||
<span class="n">languages_file</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">)</span> <span class="o">/</span> <span class="s1">'languages.py'</span>
|
||||
|
||||
|
||||
<span class="c1"># Fetches supported languages for each engine and writes json file with those.</span>
|
||||
<span class="k">def</span> <span class="nf">fetch_supported_languages</span><span class="p">():</span>
|
||||
<span class="n">set_timeout_for_thread</span><span class="p">(</span><span class="mf">10.0</span><span class="p">)</span>
|
||||
|
||||
<span class="n">engines_languages</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">names</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">engines</span><span class="p">)</span>
|
||||
<span class="n">names</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">],</span> <span class="s1">'fetch_supported_languages'</span><span class="p">):</span>
|
||||
<span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">fetch_supported_languages</span><span class="p">()</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"fetched </span><span class="si">%s</span><span class="s2"> languages from engine </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]),</span> <span class="n">engine_name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">])</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span> <span class="c1"># pylint: disable=unidiomatic-typecheck</span>
|
||||
<span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">])</span>
|
||||
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"fetched languages from </span><span class="si">%s</span><span class="s2"> engines"</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># write json file</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">engines_languages_file</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">engines_languages</span>
|
||||
|
||||
|
||||
<span class="c1"># Get babel Locale object from lang_code if possible.</span>
|
||||
<span class="k">def</span> <span class="nf">get_locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">):</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">locale</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="n">UnknownLocaleError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
|
||||
<span class="n">lang2emoji</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'ha'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\U0001F1F3\U0001F1EA</span><span class="s1">'</span><span class="p">,</span> <span class="c1"># Hausa / Niger</span>
|
||||
<span class="s1">'bs'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\U0001F1E7\U0001F1E6</span><span class="s1">'</span><span class="p">,</span> <span class="c1"># Bosnian / Bosnia & Herzegovina</span>
|
||||
<span class="s1">'jp'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\U0001F1EF\U0001F1F5</span><span class="s1">'</span><span class="p">,</span> <span class="c1"># Japanese</span>
|
||||
<span class="s1">'ua'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\U0001F1FA\U0001F1E6</span><span class="s1">'</span><span class="p">,</span> <span class="c1"># Ukrainian</span>
|
||||
<span class="s1">'he'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\U0001F1EE\U0001F1F7</span><span class="s1">'</span><span class="p">,</span> <span class="c1"># Hebrew</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_unicode_flag"><a class="viewcode-back" href="../../../dev/searxng_extra/update.html#searxng_extra.update.update_languages.get_unicode_flag">[docs]</a><span class="k">def</span> <span class="nf">get_unicode_flag</span><span class="p">(</span><span class="n">lang_code</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Determine a unicode flag (emoji) that fits to the ``lang_code``"""</span>
|
||||
|
||||
<span class="n">emoji</span> <span class="o">=</span> <span class="n">lang2emoji</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_code</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">emoji</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">emoji</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="s1">'</span><span class="se">\U0001F310</span><span class="s1">'</span>
|
||||
|
||||
<span class="n">language</span> <span class="o">=</span> <span class="n">territory</span> <span class="o">=</span> <span class="n">script</span> <span class="o">=</span> <span class="n">variant</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">language</span><span class="p">,</span> <span class="n">territory</span><span class="p">,</span> <span class="n">script</span><span class="p">,</span> <span class="n">variant</span> <span class="o">=</span> <span class="n">parse_locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">territory</span><span class="p">:</span>
|
||||
<span class="c1"># https://www.unicode.org/emoji/charts/emoji-list.html#country-flag</span>
|
||||
<span class="n">emoji</span> <span class="o">=</span> <span class="n">lang2emoji</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">language</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">emoji</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">%s</span><span class="s2"> --> language: </span><span class="si">%s</span><span class="s2"> / territory: </span><span class="si">%s</span><span class="s2"> / script: </span><span class="si">%s</span><span class="s2"> / variant: </span><span class="si">%s</span><span class="s2">"</span>
|
||||
<span class="o">%</span> <span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="n">language</span><span class="p">,</span> <span class="n">territory</span><span class="p">,</span> <span class="n">script</span><span class="p">,</span> <span class="n">variant</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">emoji</span>
|
||||
|
||||
<span class="n">emoji</span> <span class="o">=</span> <span class="n">lang2emoji</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">territory</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">emoji</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">emoji</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">c1</span> <span class="o">=</span> <span class="n">lookup</span><span class="p">(</span><span class="s1">'REGIONAL INDICATOR SYMBOL LETTER '</span> <span class="o">+</span> <span class="n">territory</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
||||
<span class="n">c2</span> <span class="o">=</span> <span class="n">lookup</span><span class="p">(</span><span class="s1">'REGIONAL INDICATOR SYMBOL LETTER '</span> <span class="o">+</span> <span class="n">territory</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
<span class="c1"># print("%s --> territory: %s --> %s%s" %(lang_code, territory, c1, c2 ))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> --> territory: </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_code</span><span class="p">,</span> <span class="n">territory</span><span class="p">,</span> <span class="n">exc</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">c1</span> <span class="o">+</span> <span class="n">c2</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">get_territory_name</span><span class="p">(</span><span class="n">lang_code</span><span class="p">):</span>
|
||||
<span class="n">country_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">country_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">get_territory_name</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">exc</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="n">country_name</span>
|
||||
|
||||
|
||||
<span class="c1"># Join all language lists.</span>
|
||||
<span class="k">def</span> <span class="nf">join_language_lists</span><span class="p">(</span><span class="n">engines_languages</span><span class="p">):</span>
|
||||
<span class="n">language_list</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">engines_languages</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="n">engines_languages</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]:</span>
|
||||
|
||||
<span class="c1"># apply custom fixes if necessary</span>
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">],</span> <span class="s1">'language_aliases'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
|
||||
<span class="n">lang_code</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
||||
<span class="n">lc</span> <span class="k">for</span> <span class="n">lc</span><span class="p">,</span> <span class="n">alias</span> <span class="ow">in</span> <span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">language_aliases</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">lang_code</span> <span class="o">==</span> <span class="n">alias</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># ensure that lang_code uses standard language and country codes</span>
|
||||
<span class="k">if</span> <span class="n">locale</span> <span class="ow">and</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||
<span class="n">lang_code</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{lang}</span><span class="s2">-</span><span class="si">{country}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lang</span><span class="o">=</span><span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">,</span> <span class="n">country</span><span class="o">=</span><span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">)</span>
|
||||
<span class="n">short_code</span> <span class="o">=</span> <span class="n">lang_code</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># add language without country if not in list</span>
|
||||
<span class="k">if</span> <span class="n">short_code</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">language_list</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="p">:</span>
|
||||
<span class="c1"># get language's data from babel's Locale object</span>
|
||||
<span class="n">language_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">get_language_name</span><span class="p">()</span><span class="o">.</span><span class="n">title</span><span class="p">()</span>
|
||||
<span class="n">english_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">english_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' ('</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="n">short_code</span> <span class="ow">in</span> <span class="n">engines_languages</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">]:</span>
|
||||
<span class="c1"># get language's data from wikipedia if not known by babel</span>
|
||||
<span class="n">language_name</span> <span class="o">=</span> <span class="n">engines_languages</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">][</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'name'</span><span class="p">]</span>
|
||||
<span class="n">english_name</span> <span class="o">=</span> <span class="n">engines_languages</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">][</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'english_name'</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">language_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">english_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># add language to list</span>
|
||||
<span class="n">language_list</span><span class="p">[</span><span class="n">short_code</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'name'</span><span class="p">:</span> <span class="n">language_name</span><span class="p">,</span>
|
||||
<span class="s1">'english_name'</span><span class="p">:</span> <span class="n">english_name</span><span class="p">,</span>
|
||||
<span class="s1">'counter'</span><span class="p">:</span> <span class="nb">set</span><span class="p">(),</span>
|
||||
<span class="s1">'countries'</span><span class="p">:</span> <span class="p">{},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># add language with country if not in list</span>
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="o">!=</span> <span class="n">short_code</span> <span class="ow">and</span> <span class="n">lang_code</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">language_list</span><span class="p">[</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'countries'</span><span class="p">]:</span>
|
||||
<span class="n">country_name</span> <span class="o">=</span> <span class="s1">''</span>
|
||||
<span class="k">if</span> <span class="n">locale</span><span class="p">:</span>
|
||||
<span class="c1"># get country name from babel's Locale object</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">country_name</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">get_territory_name</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">exc</span><span class="p">))</span>
|
||||
<span class="n">locale</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">language_list</span><span class="p">[</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'countries'</span><span class="p">][</span><span class="n">lang_code</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s1">'country_name'</span><span class="p">:</span> <span class="n">country_name</span><span class="p">,</span>
|
||||
<span class="s1">'counter'</span><span class="p">:</span> <span class="nb">set</span><span class="p">(),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># count engine for both language_country combination and language alone</span>
|
||||
<span class="n">language_list</span><span class="p">[</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'counter'</span><span class="p">]</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">lang_code</span> <span class="o">!=</span> <span class="n">short_code</span><span class="p">:</span>
|
||||
<span class="n">language_list</span><span class="p">[</span><span class="n">short_code</span><span class="p">][</span><span class="s1">'countries'</span><span class="p">][</span><span class="n">lang_code</span><span class="p">][</span><span class="s1">'counter'</span><span class="p">]</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">language_list</span>
|
||||
|
||||
|
||||
<span class="c1"># Filter language list so it only includes the most supported languages and countries</span>
|
||||
<span class="k">def</span> <span class="nf">filter_language_list</span><span class="p">(</span><span class="n">all_languages</span><span class="p">):</span>
|
||||
<span class="n">min_engines_per_lang</span> <span class="o">=</span> <span class="mi">12</span>
|
||||
<span class="n">min_engines_per_country</span> <span class="o">=</span> <span class="mi">7</span>
|
||||
<span class="c1"># pylint: disable=consider-using-dict-items, consider-iterating-dictionary</span>
|
||||
<span class="n">main_engines</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">engine_name</span>
|
||||
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">engines</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="s1">'general'</span> <span class="ow">in</span> <span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">categories</span>
|
||||
<span class="ow">and</span> <span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">supported_languages</span>
|
||||
<span class="ow">and</span> <span class="ow">not</span> <span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span><span class="o">.</span><span class="n">disabled</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="c1"># filter list to include only languages supported by most engines or all default general engines</span>
|
||||
<span class="n">filtered_languages</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="n">code</span><span class="p">:</span> <span class="n">lang</span>
|
||||
<span class="k">for</span> <span class="n">code</span><span class="p">,</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">all_languages</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="p">(</span>
|
||||
<span class="nb">len</span><span class="p">(</span><span class="n">lang</span><span class="p">[</span><span class="s1">'counter'</span><span class="p">])</span> <span class="o">>=</span> <span class="n">min_engines_per_lang</span>
|
||||
<span class="ow">or</span> <span class="nb">all</span><span class="p">(</span><span class="n">main_engine</span> <span class="ow">in</span> <span class="n">lang</span><span class="p">[</span><span class="s1">'counter'</span><span class="p">]</span> <span class="k">for</span> <span class="n">main_engine</span> <span class="ow">in</span> <span class="n">main_engines</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">_copy_lang_data</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="n">country_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">new_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">new_dict</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">all_languages</span><span class="p">[</span><span class="n">lang</span><span class="p">][</span><span class="s1">'name'</span><span class="p">]</span>
|
||||
<span class="n">new_dict</span><span class="p">[</span><span class="s1">'english_name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">all_languages</span><span class="p">[</span><span class="n">lang</span><span class="p">][</span><span class="s1">'english_name'</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">country_name</span><span class="p">:</span>
|
||||
<span class="n">new_dict</span><span class="p">[</span><span class="s1">'country_name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">country_name</span>
|
||||
<span class="k">return</span> <span class="n">new_dict</span>
|
||||
|
||||
<span class="c1"># for each language get country codes supported by most engines or at least one country code</span>
|
||||
<span class="n">filtered_languages_with_countries</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">lang</span><span class="p">,</span> <span class="n">lang_data</span> <span class="ow">in</span> <span class="n">filtered_languages</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">countries</span> <span class="o">=</span> <span class="n">lang_data</span><span class="p">[</span><span class="s1">'countries'</span><span class="p">]</span>
|
||||
<span class="n">filtered_countries</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="c1"># get language's country codes with enough supported engines</span>
|
||||
<span class="k">for</span> <span class="n">lang_country</span><span class="p">,</span> <span class="n">country_data</span> <span class="ow">in</span> <span class="n">countries</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">country_data</span><span class="p">[</span><span class="s1">'counter'</span><span class="p">])</span> <span class="o">>=</span> <span class="n">min_engines_per_country</span><span class="p">:</span>
|
||||
<span class="n">filtered_countries</span><span class="p">[</span><span class="n">lang_country</span><span class="p">]</span> <span class="o">=</span> <span class="n">_copy_lang_data</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="n">country_data</span><span class="p">[</span><span class="s1">'country_name'</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># add language without countries too if there's more than one country to choose from</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_countries</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="n">filtered_countries</span><span class="p">[</span><span class="n">lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">_copy_lang_data</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_countries</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="n">lang_country</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">filtered_countries</span><span class="p">))</span>
|
||||
|
||||
<span class="c1"># if no country has enough engines try to get most likely country code from babel</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">filtered_countries</span><span class="p">:</span>
|
||||
<span class="n">lang_country</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">subtags</span> <span class="o">=</span> <span class="n">get_global</span><span class="p">(</span><span class="s1">'likely_subtags'</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">subtags</span><span class="p">:</span>
|
||||
<span class="n">country_code</span> <span class="o">=</span> <span class="n">subtags</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">country_code</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
|
||||
<span class="n">lang_country</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{lang}</span><span class="s2">-</span><span class="si">{country}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lang</span><span class="o">=</span><span class="n">lang</span><span class="p">,</span> <span class="n">country</span><span class="o">=</span><span class="n">country_code</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">lang_country</span><span class="p">:</span>
|
||||
<span class="n">filtered_countries</span><span class="p">[</span><span class="n">lang_country</span><span class="p">]</span> <span class="o">=</span> <span class="n">_copy_lang_data</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">filtered_countries</span><span class="p">[</span><span class="n">lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">_copy_lang_data</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
|
||||
<span class="n">filtered_languages_with_countries</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">filtered_countries</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">filtered_languages_with_countries</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="UnicodeEscape"><a class="viewcode-back" href="../../../dev/searxng_extra/update.html#searxng_extra.update.update_languages.UnicodeEscape">[docs]</a><span class="k">class</span> <span class="nc">UnicodeEscape</span><span class="p">(</span><span class="nb">str</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Escape unicode string in :py:obj:`pprint.pformat`"""</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"'"</span> <span class="o">+</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">chr</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'unicode-escape'</span><span class="p">)])</span> <span class="o">+</span> <span class="s2">"'"</span></div>
|
||||
|
||||
|
||||
<span class="c1"># Write languages.py.</span>
|
||||
<span class="k">def</span> <span class="nf">write_languages_file</span><span class="p">(</span><span class="n">languages</span><span class="p">):</span>
|
||||
<span class="n">file_headers</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"# -*- coding: utf-8 -*-"</span><span class="p">,</span>
|
||||
<span class="s2">"# list of language codes"</span><span class="p">,</span>
|
||||
<span class="s2">"# this file is generated automatically by utils/fetch_languages.py"</span><span class="p">,</span>
|
||||
<span class="s2">"language_codes = (</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">language_codes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">code</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">languages</span><span class="p">):</span>
|
||||
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="n">languages</span><span class="p">[</span><span class="n">code</span><span class="p">][</span><span class="s1">'name'</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: languages['</span><span class="si">%s</span><span class="s2">'] --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">code</span><span class="p">,</span> <span class="n">languages</span><span class="p">[</span><span class="n">code</span><span class="p">]))</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">flag</span> <span class="o">=</span> <span class="n">get_unicode_flag</span><span class="p">(</span><span class="n">code</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span>
|
||||
<span class="n">item</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">code</span><span class="p">,</span>
|
||||
<span class="n">languages</span><span class="p">[</span><span class="n">code</span><span class="p">][</span><span class="s1">'name'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' ('</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
|
||||
<span class="n">get_territory_name</span><span class="p">(</span><span class="n">code</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
|
||||
<span class="n">languages</span><span class="p">[</span><span class="n">code</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'english_name'</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
|
||||
<span class="n">UnicodeEscape</span><span class="p">(</span><span class="n">flag</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">language_codes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||
|
||||
<span class="n">language_codes</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">language_codes</span><span class="p">)</span>
|
||||
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">languages_file</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">new_file</span><span class="p">:</span>
|
||||
<span class="n">file_content</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{file_headers}</span><span class="s2"> </span><span class="si">{language_codes}</span><span class="s2">,</span><span class="se">\n</span><span class="s2">)</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="c1"># fmt: off</span>
|
||||
<span class="n">file_headers</span> <span class="o">=</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">file_headers</span><span class="p">),</span>
|
||||
<span class="n">language_codes</span> <span class="o">=</span> <span class="n">pformat</span><span class="p">(</span><span class="n">language_codes</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">4</span><span class="p">)[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="c1"># fmt: on</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">file_content</span><span class="p">)</span>
|
||||
<span class="n">new_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
||||
<span class="n">load_engines</span><span class="p">(</span><span class="n">settings</span><span class="p">[</span><span class="s1">'engines'</span><span class="p">])</span>
|
||||
<span class="n">_engines_languages</span> <span class="o">=</span> <span class="n">fetch_supported_languages</span><span class="p">()</span>
|
||||
<span class="n">_all_languages</span> <span class="o">=</span> <span class="n">join_language_lists</span><span class="p">(</span><span class="n">_engines_languages</span><span class="p">)</span>
|
||||
<span class="n">_filtered_languages</span> <span class="o">=</span> <span class="n">filter_language_list</span><span class="p">(</span><span class="n">_all_languages</span><span class="p">)</span>
|
||||
<span class="n">write_languages_file</span><span class="p">(</span><span class="n">_filtered_languages</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span id="sidebar-top"></span>
|
||||
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
||||
<div class="sphinxsidebarwrapper">
|
||||
|
||||
|
||||
<p class="logo"><a href="../../../index.html">
|
||||
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo"/>
|
||||
</a></p>
|
||||
|
||||
|
||||
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../donate.html">Donate to searxng.org</a></li>
|
||||
</ul>
|
||||
|
||||
<h3>Project Links</h3>
|
||||
<ul>
|
||||
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||
|
||||
<li><a href="https://searx.space">Public instances</a>
|
||||
|
||||
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||
</ul><h3>Navigation</h3>
|
||||
<ul>
|
||||
<li><a href="../../../index.html">Overview</a>
|
||||
<ul>
|
||||
<li><a href="../../index.html">Module code</a>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="searchbox" style="display: none" role="search">
|
||||
<h3 id="searchlabel">Quick search</h3>
|
||||
<div class="searchformwrapper">
|
||||
<form class="search" action="../../../search.html" method="get">
|
||||
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||
<input type="submit" value="Go" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="clearer"></div>
|
||||
</div>
|
||||
|
||||
<div class="footer" role="contentinfo">
|
||||
© Copyright 2021 SearXNG team, 2015-2021 Adam Tauber, Noémi Ványi.
|
||||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
|
||||
</div>
|
||||
<script src="../../../_static/version_warning_offset.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -0,0 +1,96 @@
|
||||
.. _adminapi:
|
||||
|
||||
==================
|
||||
Administration API
|
||||
==================
|
||||
|
||||
Get configuration data
|
||||
======================
|
||||
|
||||
.. code:: http
|
||||
|
||||
GET /config HTTP/1.1
|
||||
|
||||
Sample response
|
||||
---------------
|
||||
|
||||
.. code:: json
|
||||
|
||||
{
|
||||
"autocomplete": "",
|
||||
"categories": [
|
||||
"map",
|
||||
"it",
|
||||
"images",
|
||||
],
|
||||
"default_locale": "",
|
||||
"default_theme": "simple",
|
||||
"engines": [
|
||||
{
|
||||
"categories": [
|
||||
"map"
|
||||
],
|
||||
"enabled": true,
|
||||
"name": "openstreetmap",
|
||||
"shortcut": "osm"
|
||||
},
|
||||
{
|
||||
"categories": [
|
||||
"it"
|
||||
],
|
||||
"enabled": true,
|
||||
"name": "arch linux wiki",
|
||||
"shortcut": "al"
|
||||
},
|
||||
{
|
||||
"categories": [
|
||||
"images"
|
||||
],
|
||||
"enabled": true,
|
||||
"name": "google images",
|
||||
"shortcut": "goi"
|
||||
},
|
||||
{
|
||||
"categories": [
|
||||
"it"
|
||||
],
|
||||
"enabled": false,
|
||||
"name": "bitbucket",
|
||||
"shortcut": "bb"
|
||||
},
|
||||
],
|
||||
"instance_name": "searx",
|
||||
"locales": {
|
||||
"de": "Deutsch (German)",
|
||||
"en": "English",
|
||||
"eo": "Esperanto (Esperanto)",
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"enabled": true,
|
||||
"name": "HTTPS rewrite"
|
||||
},
|
||||
{
|
||||
"enabled": false,
|
||||
"name": "Vim-like hotkeys"
|
||||
}
|
||||
],
|
||||
"safe_search": 0
|
||||
}
|
||||
|
||||
|
||||
Embed search bar
|
||||
================
|
||||
|
||||
The search bar can be embedded into websites. Just paste the example into the
|
||||
HTML of the site. URL of the SearXNG instance and values are customizable.
|
||||
|
||||
.. code:: html
|
||||
|
||||
<form method="post" action="https://example.org/">
|
||||
<!-- search --> <input type="text" name="q" />
|
||||
<!-- categories --> <input type="hidden" name="categories" value="general,social media" />
|
||||
<!-- language --> <input type="hidden" name="lang" value="all" />
|
||||
<!-- locale --> <input type="hidden" name="locale" value="en" />
|
||||
<!-- date filter --> <input type="hidden" name="time_range" value="month" />
|
||||
</form>
|
@ -0,0 +1,38 @@
|
||||
.. _architecture:
|
||||
|
||||
============
|
||||
Architecture
|
||||
============
|
||||
|
||||
.. sidebar:: Further reading
|
||||
|
||||
- Reverse Proxy: :ref:`Apache <apache searxng site>` & :ref:`nginx <nginx
|
||||
searxng site>`
|
||||
- uWSGI: :ref:`searxng uwsgi`
|
||||
- SearXNG: :ref:`installation basic`
|
||||
|
||||
Herein you will find some hints and suggestions about typical architectures of
|
||||
SearXNG infrastructures.
|
||||
|
||||
.. _architecture uWSGI:
|
||||
|
||||
uWSGI Setup
|
||||
===========
|
||||
|
||||
We start with a *reference* setup for public SearXNG instances which can be build
|
||||
up and maintained by the scripts from our :ref:`toolboxing`.
|
||||
|
||||
.. _arch public:
|
||||
|
||||
.. kernel-figure:: arch_public.dot
|
||||
:alt: arch_public.dot
|
||||
|
||||
Reference architecture of a public SearXNG setup.
|
||||
|
||||
The reference installation activates ``server.limiter``, ``server.image_proxy``
|
||||
and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml
|
||||
<utils/templates/etc/searxng/settings.yml>`)
|
||||
|
||||
.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml
|
||||
:language: yaml
|
||||
:end-before: # preferences:
|
@ -0,0 +1,155 @@
|
||||
.. _buildhosts:
|
||||
|
||||
==========
|
||||
Buildhosts
|
||||
==========
|
||||
|
||||
.. sidebar:: This article needs some work
|
||||
|
||||
If you have any contribution send us your :pull:`PR <../pulls>`, see
|
||||
:ref:`how to contribute`.
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
To get best results from build, its recommend to install additional packages
|
||||
on build hosts (see :ref:`searxng.sh`).::
|
||||
|
||||
sudo -H ./utils/searxng.sh install buildhost
|
||||
|
||||
This will install packages needed by searx:
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START distro-packages
|
||||
:end-before: END distro-packages
|
||||
|
||||
and packages needed to build docuemtation and run tests:
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START build-packages
|
||||
:end-before: END build-packages
|
||||
|
||||
.. _docs build:
|
||||
|
||||
Build docs
|
||||
==========
|
||||
|
||||
.. _Graphviz: https://graphviz.gitlab.io
|
||||
.. _ImageMagick: https://www.imagemagick.org
|
||||
.. _XeTeX: https://tug.org/xetex/
|
||||
.. _dvisvgm: https://dvisvgm.de/
|
||||
|
||||
.. sidebar:: Sphinx build needs
|
||||
|
||||
- ImageMagick_
|
||||
- Graphviz_
|
||||
- XeTeX_
|
||||
- dvisvgm_
|
||||
|
||||
Most of the sphinx requirements are installed from :origin:`setup.py` and the
|
||||
docs can be build from scratch with ``make docs.html``. For better math and
|
||||
image processing additional packages are needed. The XeTeX_ needed not only for
|
||||
PDF creation, its also needed for :ref:`math` when HTML output is build.
|
||||
|
||||
To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
|
||||
as images (``sphinx.ext.imgmath`` extension).
|
||||
|
||||
Here is the extract from the :origin:`docs/conf.py` file, setting math renderer
|
||||
to ``imgmath``:
|
||||
|
||||
.. literalinclude:: ../conf.py
|
||||
:language: python
|
||||
:start-after: # sphinx.ext.imgmath setup
|
||||
:end-before: # sphinx.ext.imgmath setup END
|
||||
|
||||
If your docs build (``make docs.html``) shows warnings like this::
|
||||
|
||||
WARNING: dot(1) not found, for better output quality install \
|
||||
graphviz from https://www.graphviz.org
|
||||
..
|
||||
WARNING: LaTeX command 'latex' cannot be run (needed for math \
|
||||
display), check the imgmath_latex setting
|
||||
|
||||
you need to install additional packages on your build host, to get better HTML
|
||||
output.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
|
||||
|
||||
|
||||
For PDF output you also need:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ sudo dnf install \
|
||||
texlive-collection-fontsrecommended texlive-collection-latex \
|
||||
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
|
||||
ImageMagick
|
||||
|
||||
.. _sh lint:
|
||||
|
||||
Lint shell scripts
|
||||
==================
|
||||
|
||||
.. _ShellCheck: https://github.com/koalaman/shellcheck
|
||||
|
||||
To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool.
|
||||
|
||||
.. SNIP sh lint requirements
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo apt install shellcheck
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo pacman -S shellcheck
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ sudo dnf install ShellCheck
|
||||
|
||||
.. SNAP sh lint requirements
|
@ -0,0 +1,79 @@
|
||||
.. _engine command:
|
||||
|
||||
====================
|
||||
Command Line Engines
|
||||
====================
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`command.py <searx/engines/command.py>`
|
||||
- :ref:`offline engines`
|
||||
|
||||
With *command engines* administrators can run engines to integrate arbitrary
|
||||
shell commands.
|
||||
|
||||
When creating and enabling a ``command`` engine on a public instance, you must
|
||||
be careful to avoid leaking private data. The easiest solution is to limit the
|
||||
access by setting ``tokens`` as described in section :ref:`private engines`.
|
||||
|
||||
The engine base is flexible. Only your imagination can limit the power of this
|
||||
engine (and maybe security concerns). The following options are available:
|
||||
|
||||
``command``:
|
||||
A comma separated list of the elements of the command. A special token
|
||||
``{{QUERY}}`` tells where to put the search terms of the user. Example:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
['ls', '-l', '-h', '{{QUERY}}']
|
||||
|
||||
``delimiter``:
|
||||
A mapping containing a delimiter ``char`` and the *titles* of each element in
|
||||
``keys``.
|
||||
|
||||
``parse_regex``:
|
||||
A dict containing the regular expressions for each result key.
|
||||
|
||||
``query_type``:
|
||||
|
||||
The expected type of user search terms. Possible values: ``path`` and
|
||||
``enum``.
|
||||
|
||||
``path``:
|
||||
Checks if the user provided path is inside the working directory. If not,
|
||||
the query is not executed.
|
||||
|
||||
``enum``:
|
||||
Is a list of allowed search terms. If the user submits something which is
|
||||
not included in the list, the query returns an error.
|
||||
|
||||
``query_enum``:
|
||||
A list containing allowed search terms if ``query_type`` is set to ``enum``.
|
||||
|
||||
``working_dir``:
|
||||
|
||||
The directory where the command has to be executed. Default: ``./``
|
||||
|
||||
``result_separator``:
|
||||
The character that separates results. Default: ``\n``
|
||||
|
||||
The example engine below can be used to find files with a specific name in the
|
||||
configured working directory:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: find
|
||||
engine: command
|
||||
command: ['find', '.', '-name', '{{QUERY}}']
|
||||
query_type: path
|
||||
shortcut: fnd
|
||||
delimiter:
|
||||
chars: ' '
|
||||
keys: ['line']
|
||||
|
||||
|
||||
Acknowledgment
|
||||
==============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_.
|
@ -0,0 +1,75 @@
|
||||
.. _configured engines:
|
||||
|
||||
==================
|
||||
Configured Engines
|
||||
==================
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`engines-dev`
|
||||
- :ref:`settings engine`
|
||||
|
||||
Explanation of the :ref:`general engine configuration` shown in the table
|
||||
:ref:`configured engines`.
|
||||
|
||||
.. jinja:: searx
|
||||
|
||||
SearXNG supports {{engines | length}} search engines (of which {{enabled_engine_count}} are enabled by default).
|
||||
|
||||
{% for category, engines in categories_as_tabs.items() %}
|
||||
|
||||
{{category}} search engines
|
||||
---------------------------------------
|
||||
|
||||
{% for group, engines in engines | group_engines_in_tab %}
|
||||
|
||||
{% if loop.length > 1 %}
|
||||
{{group}}
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
{% endif %}
|
||||
|
||||
.. flat-table::
|
||||
:header-rows: 2
|
||||
:stub-columns: 1
|
||||
|
||||
* - :cspan:`5` Engines configured by default (in :ref:`settings.yml <engine settings>`)
|
||||
- :cspan:`3` :ref:`Supported features <engine file>`
|
||||
|
||||
* - Name
|
||||
- Shortcut
|
||||
- Module
|
||||
- Disabled
|
||||
- Timeout
|
||||
- Weight
|
||||
- Paging
|
||||
- Language
|
||||
- Safe search
|
||||
- Time range
|
||||
|
||||
{% for mod in engines %}
|
||||
|
||||
* - `{{mod.name}} <{{mod.about and mod.about.website}}>`_
|
||||
- ``!{{mod.shortcut}}``
|
||||
- {%- if 'searx.engines.' + mod.__name__ in documented_modules %}
|
||||
:py:mod:`~searx.engines.{{mod.__name__}}`
|
||||
{%- else %}
|
||||
:origin:`{{mod.__name__}} <searx/engines/{{mod.__name__}}.py>`
|
||||
{%- endif %}
|
||||
- {{(mod.disabled and "y") or ""}}
|
||||
{%- if mod.about and mod.about.language %}
|
||||
({{mod.about.language | upper}})
|
||||
{%- endif %}
|
||||
- {{mod.timeout}}
|
||||
- {{mod.weight or 1 }}
|
||||
{% if mod.engine_type == 'online' %}
|
||||
- {{(mod.paging and "y") or ""}}
|
||||
- {{(mod.language_support and "y") or ""}}
|
||||
- {{(mod.safesearch and "y") or ""}}
|
||||
- {{(mod.time_range_support and "y") or ""}}
|
||||
{% else %}
|
||||
- :cspan:`3` not applicable ({{mod.engine_type}})
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
@ -0,0 +1,23 @@
|
||||
.. _engines and settings:
|
||||
|
||||
==================
|
||||
Engines & Settings
|
||||
==================
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`settings engine`
|
||||
- :ref:`engine settings` & :ref:`engine file`
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
settings
|
||||
configured_engines
|
||||
private-engines
|
||||
recoll
|
||||
sql-engines
|
||||
nosql-engines
|
||||
search-indexer-engines
|
||||
command-line-engines
|
||||
searx.engines.xpath
|
@ -0,0 +1,135 @@
|
||||
===============
|
||||
NoSQL databases
|
||||
===============
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- `NoSQL databases <https://en.wikipedia.org/wiki/NoSQL>`_
|
||||
- `redis.io <https://redis.io/>`_
|
||||
- `MongoDB <https://www.mongodb.com>`_
|
||||
|
||||
The following `NoSQL databases`_ are supported:
|
||||
|
||||
- :ref:`engine redis_server`
|
||||
- :ref:`engine mongodb`
|
||||
|
||||
All of the engines above are just commented out in the :origin:`settings.yml
|
||||
<searx/settings.yml>`, as you have to set various options and install
|
||||
dependencies before using them.
|
||||
|
||||
By default, the engines use the ``key-value`` template for displaying results /
|
||||
see :origin:`simple <searx/templates/simple/result_templates/key-value.html>`
|
||||
theme. If you are not satisfied with the original result layout, you can use
|
||||
your own template, set ``result_template`` attribute to ``{template_name}`` and
|
||||
place the templates at::
|
||||
|
||||
searx/templates/{theme_name}/result_templates/{template_name}
|
||||
|
||||
Furthermore, if you do not wish to expose these engines on a public instance, you
|
||||
can still add them and limit the access by setting ``tokens`` as described in
|
||||
section :ref:`private engines`.
|
||||
|
||||
|
||||
Configure the engines
|
||||
=====================
|
||||
|
||||
`NoSQL databases`_ are used for storing arbitrary data without first defining
|
||||
their structure.
|
||||
|
||||
|
||||
Extra Dependencies
|
||||
------------------
|
||||
|
||||
For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to
|
||||
install additional packages in Python's Virtual Environment of your SearXNG
|
||||
instance. To switch into the environment (:ref:`searxng-src`) you can use
|
||||
:ref:`searxng.sh`::
|
||||
|
||||
$ sudo utils/searxng.sh instance cmd bash
|
||||
(searxng-pyenv)$ pip install ...
|
||||
|
||||
|
||||
.. _engine redis_server:
|
||||
|
||||
Redis Server
|
||||
------------
|
||||
|
||||
.. _redis: https://github.com/andymccurdy/redis-py#installation
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- ``pip install`` redis_
|
||||
- redis.io_
|
||||
- :origin:`redis_server.py <searx/engines/redis_server.py>`
|
||||
|
||||
|
||||
Redis is an open source (BSD licensed), in-memory data structure (key value
|
||||
based) store. Before configuring the ``redis_server`` engine, you must install
|
||||
the dependency redis_.
|
||||
|
||||
Select a database to search in and set its index in the option ``db``. You can
|
||||
either look for exact matches or use partial keywords to find what you are
|
||||
looking for by configuring ``exact_match_only``. You find an example
|
||||
configuration below:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
# Required dependency: redis
|
||||
|
||||
- name: myredis
|
||||
shortcut : rds
|
||||
engine: redis_server
|
||||
exact_match_only: false
|
||||
host: '127.0.0.1'
|
||||
port: 6379
|
||||
enable_http: true
|
||||
password: ''
|
||||
db: 0
|
||||
|
||||
.. _engine mongodb:
|
||||
|
||||
MongoDB
|
||||
-------
|
||||
|
||||
.. _pymongo: https://github.com/mongodb/mongo-python-driver#installation
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- ``pip install`` pymongo_
|
||||
- MongoDB_
|
||||
- :origin:`mongodb.py <searx/engines/mongodb.py>`
|
||||
|
||||
MongoDB_ is a document based database program that handles JSON like data.
|
||||
Before configuring the ``mongodb`` engine, you must install the dependency
|
||||
redis_.
|
||||
|
||||
In order to query MongoDB_, you have to select a ``database`` and a
|
||||
``collection``. Furthermore, you have to select a ``key`` that is going to be
|
||||
searched. MongoDB_ also supports the option ``exact_match_only``, so configure
|
||||
it as you wish. Below is an example configuration for using a MongoDB
|
||||
collection:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
# MongoDB engine
|
||||
# Required dependency: pymongo
|
||||
|
||||
- name: mymongo
|
||||
engine: mongodb
|
||||
shortcut: md
|
||||
exact_match_only: false
|
||||
host: '127.0.0.1'
|
||||
port: 27017
|
||||
enable_http: true
|
||||
results_per_page: 20
|
||||
database: 'business'
|
||||
collection: 'reviews' # name of the db collection
|
||||
key: 'name' # key in the collection to search for
|
||||
|
||||
|
||||
Acknowledgment
|
||||
==============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_.
|
||||
|
@ -0,0 +1,49 @@
|
||||
.. _private engines:
|
||||
|
||||
============================
|
||||
Private Engines (``tokens``)
|
||||
============================
|
||||
|
||||
Administrators might find themselves wanting to limit access to some of the
|
||||
enabled engines on their instances. It might be because they do not want to
|
||||
expose some private information through :ref:`offline engines`. Or they would
|
||||
rather share engines only with their trusted friends or colleagues.
|
||||
|
||||
To solve this issue the concept of *private engines* exists.
|
||||
|
||||
|
||||
A new option was added to engines named `tokens`. It expects a list of
|
||||
strings. If the user making a request presents one of the tokens of an engine,
|
||||
they can access information about the engine and make search requests.
|
||||
|
||||
Example configuration to restrict access to the Arch Linux Wiki engine:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: arch linux wiki
|
||||
engine: archlinux
|
||||
shortcut: al
|
||||
tokens: [ 'my-secret-token' ]
|
||||
|
||||
|
||||
Unless a user has configured the right token, the engine is going
|
||||
to be hidden from him/her. It is not going to be included in the
|
||||
list of engines on the Preferences page and in the output of
|
||||
`/config` REST API call.
|
||||
|
||||
Tokens can be added to one's configuration on the Preferences page
|
||||
under "Engine tokens". The input expects a comma separated list of
|
||||
strings.
|
||||
|
||||
The distribution of the tokens from the administrator to the users
|
||||
is not carved in stone. As providing access to such engines
|
||||
implies that the admin knows and trusts the user, we do not see
|
||||
necessary to come up with a strict process. Instead,
|
||||
we would like to add guidelines to the documentation of the feature.
|
||||
|
||||
|
||||
Acknowledgment
|
||||
==============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_.
|
@ -0,0 +1,50 @@
|
||||
.. _engine recoll:
|
||||
|
||||
=============
|
||||
Recoll Engine
|
||||
=============
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- `Recoll <https://www.lesbonscomptes.com/recoll/>`_
|
||||
- `recoll-webui <https://framagit.org/medoc92/recollwebui.git>`_
|
||||
- :origin:`searx/engines/recoll.py`
|
||||
|
||||
Recoll_ is a desktop full-text search tool based on Xapian. By itself Recoll_
|
||||
does not offer WEB or API access, this can be achieved using recoll-webui_
|
||||
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
You must configure the following settings:
|
||||
|
||||
``base_url``:
|
||||
Location where recoll-webui can be reached.
|
||||
|
||||
``mount_prefix``:
|
||||
Location where the file hierarchy is mounted on your *local* filesystem.
|
||||
|
||||
``dl_prefix``:
|
||||
Location where the file hierarchy as indexed by recoll can be reached.
|
||||
|
||||
``search_dir``:
|
||||
Part of the indexed file hierarchy to be search, if empty the full domain is
|
||||
searched.
|
||||
|
||||
|
||||
Example
|
||||
=======
|
||||
|
||||
Scenario:
|
||||
|
||||
#. Recoll indexes a local filesystem mounted in ``/export/documents/reference``,
|
||||
#. the Recoll search interface can be reached at https://recoll.example.org/ and
|
||||
#. the contents of this filesystem can be reached though https://download.example.org/reference
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
base_url: https://recoll.example.org/
|
||||
mount_prefix: /export/documents
|
||||
dl_prefix: https://download.example.org
|
||||
search_dir: ''
|
@ -0,0 +1,136 @@
|
||||
====================
|
||||
Local Search Engines
|
||||
====================
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- `Comparison to alternatives
|
||||
<https://docs.meilisearch.com/learn/what_is_meilisearch/comparison_to_alternatives.html>`_
|
||||
|
||||
Administrators might find themselves wanting to integrate locally running search
|
||||
engines. The following ones are supported for now:
|
||||
|
||||
* `Elasticsearch`_
|
||||
* `Meilisearch`_
|
||||
* `Solr`_
|
||||
|
||||
Each search engine is powerful, capable of full-text search. All of the engines
|
||||
above are added to ``settings.yml`` just commented out, as you have to
|
||||
``base_url`` for all them.
|
||||
|
||||
Please note that if you are not using HTTPS to access these engines, you have to enable
|
||||
HTTP requests by setting ``enable_http`` to ``True``.
|
||||
|
||||
Furthermore, if you do not want to expose these engines on a public instance, you
|
||||
can still add them and limit the access by setting ``tokens`` as described in
|
||||
section :ref:`private engines`.
|
||||
|
||||
.. _engine meilisearch:
|
||||
|
||||
MeiliSearch
|
||||
===========
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`meilisearch.py <searx/engines/meilisearch.py>`
|
||||
- `MeiliSearch <https://www.meilisearch.com>`_
|
||||
- `MeiliSearch Documentation <https://docs.meilisearch.com/>`_
|
||||
- `Install MeiliSearch
|
||||
<https://docs.meilisearch.com/learn/getting_started/installation.html>`_
|
||||
|
||||
MeiliSearch_ is aimed at individuals and small companies. It is designed for
|
||||
small-scale (less than 10 million documents) data collections. E.g. it is great
|
||||
for storing web pages you have visited and searching in the contents later.
|
||||
|
||||
The engine supports faceted search, so you can search in a subset of documents
|
||||
of the collection. Furthermore, you can search in MeiliSearch_ instances that
|
||||
require authentication by setting ``auth_token``.
|
||||
|
||||
Here is a simple example to query a Meilisearch instance:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: meilisearch
|
||||
engine: meilisearch
|
||||
shortcut: mes
|
||||
base_url: http://localhost:7700
|
||||
index: my-index
|
||||
enable_http: true
|
||||
|
||||
|
||||
.. _engine elasticsearch:
|
||||
|
||||
Elasticsearch
|
||||
=============
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`elasticsearch.py <searx/engines/elasticsearch.py>`
|
||||
- `Elasticsearch <https://www.elastic.co/elasticsearch/>`_
|
||||
- `Elasticsearch Guide
|
||||
<https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html>`_
|
||||
- `Install Elasticsearch
|
||||
<https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html>`_
|
||||
|
||||
Elasticsearch_ supports numerous ways to query the data it is storing. At the
|
||||
moment the engine supports the most popular search methods (``query_type``):
|
||||
|
||||
- ``match``,
|
||||
- ``simple_query_string``,
|
||||
- ``term`` and
|
||||
- ``terms``.
|
||||
|
||||
If none of the methods fit your use case, you can select ``custom`` query type
|
||||
and provide the JSON payload to submit to Elasticsearch in
|
||||
``custom_query_json``.
|
||||
|
||||
The following is an example configuration for an Elasticsearch_ instance with
|
||||
authentication configured to read from ``my-index`` index.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: elasticsearch
|
||||
shortcut: es
|
||||
engine: elasticsearch
|
||||
base_url: http://localhost:9200
|
||||
username: elastic
|
||||
password: changeme
|
||||
index: my-index
|
||||
query_type: match
|
||||
# custom_query_json: '{ ... }'
|
||||
enable_http: true
|
||||
|
||||
.. _engine solr:
|
||||
|
||||
Solr
|
||||
====
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`solr.py <searx/engines/solr.py>`
|
||||
- `Solr <https://solr.apache.org>`_
|
||||
- `Solr Resources <https://solr.apache.org/resources.html>`_
|
||||
- `Install Solr <https://solr.apache.org/guide/installing-solr.html>`_
|
||||
|
||||
Solr_ is a popular search engine based on Lucene, just like Elasticsearch_. But
|
||||
instead of searching in indices, you can search in collections.
|
||||
|
||||
This is an example configuration for searching in the collection
|
||||
``my-collection`` and get the results in ascending order.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: solr
|
||||
engine: solr
|
||||
shortcut: slr
|
||||
base_url: http://localhost:8983
|
||||
collection: my-collection
|
||||
sort: asc
|
||||
enable_http: true
|
||||
|
||||
|
||||
Acknowledgment
|
||||
==============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_.
|
@ -0,0 +1,9 @@
|
||||
.. _xpath engine:
|
||||
|
||||
============
|
||||
XPath Engine
|
||||
============
|
||||
|
||||
.. automodule:: searx.engines.xpath
|
||||
:members:
|
||||
|
@ -0,0 +1,679 @@
|
||||
.. _settings.yml:
|
||||
|
||||
================
|
||||
``settings.yml``
|
||||
================
|
||||
|
||||
This page describe the options possibilities of the :origin:`searx/settings.yml`
|
||||
file.
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`use_default_settings.yml`
|
||||
- :ref:`search API`
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
.. _settings location:
|
||||
|
||||
settings.yml location
|
||||
=====================
|
||||
|
||||
The initial ``settings.yml`` we be load from these locations:
|
||||
|
||||
1. the full path specified in the ``SEARXNG_SETTINGS_PATH`` environment variable.
|
||||
2. ``/etc/searxng/settings.yml``
|
||||
|
||||
If these files don't exist (or are empty or can't be read), SearXNG uses the
|
||||
:origin:`searx/settings.yml` file. Read :ref:`settings use_default_settings` to
|
||||
see how you can simplify your *user defined* ``settings.yml``.
|
||||
|
||||
|
||||
.. _settings global:
|
||||
|
||||
Global Settings
|
||||
===============
|
||||
|
||||
.. _settings brand:
|
||||
|
||||
``brand:``
|
||||
----------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
brand:
|
||||
issue_url: https://github.com/searxng/searxng/issues
|
||||
docs_url: https://docs.searxng.org
|
||||
public_instances: https://searx.space
|
||||
wiki_url: https://github.com/searxng/searxng/wiki
|
||||
|
||||
``issue_url`` :
|
||||
If you host your own issue tracker change this URL.
|
||||
|
||||
``docs_url`` :
|
||||
If you host your own documentation change this URL.
|
||||
|
||||
``public_instances`` :
|
||||
If you host your own https://searx.space change this URL.
|
||||
|
||||
``wiki_url`` :
|
||||
Link to your wiki (or ``false``)
|
||||
|
||||
.. _settings general:
|
||||
|
||||
``general:``
|
||||
------------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
general:
|
||||
debug: false
|
||||
instance_name: "SearXNG"
|
||||
privacypolicy_url: false
|
||||
donation_url: https://docs.searxng.org/donate.html
|
||||
contact_url: false
|
||||
enable_metrics: true
|
||||
|
||||
``debug`` : ``$SEARXNG_DEBUG``
|
||||
Allow a more detailed log if you run SearXNG directly. Display *detailed* error
|
||||
messages in the browser too, so this must be deactivated in production.
|
||||
|
||||
``donation_url`` :
|
||||
At default the donation link points to the `SearXNG project
|
||||
<https://docs.searxng.org/donate.html>`_. Set value to ``true`` to use your
|
||||
own donation page written in the :ref:`searx/info/en/donate.md
|
||||
<searx.infopage>` and use ``false`` to disable the donation link altogether.
|
||||
|
||||
``privacypolicy_url``:
|
||||
Link to privacy policy.
|
||||
|
||||
``contact_url``:
|
||||
Contact ``mailto:`` address or WEB form.
|
||||
|
||||
``enable_metrics``:
|
||||
Enabled by default. Record various anonymous metrics availabled at ``/stats``,
|
||||
``/stats/errors`` and ``/preferences``.
|
||||
|
||||
.. _settings search:
|
||||
|
||||
``search:``
|
||||
-----------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
search:
|
||||
safe_search: 0
|
||||
autocomplete: ""
|
||||
default_lang: ""
|
||||
ban_time_on_fail: 5
|
||||
max_ban_time_on_fail: 120
|
||||
formats:
|
||||
- html
|
||||
|
||||
``safe_search``:
|
||||
Filter results.
|
||||
|
||||
- ``0``: None
|
||||
- ``1``: Moderate
|
||||
- ``2``: Strict
|
||||
|
||||
``autocomplete``:
|
||||
Existing autocomplete backends, leave blank to turn it off.
|
||||
|
||||
- ``dbpedia``
|
||||
- ``duckduckgo``
|
||||
- ``google``
|
||||
- ``startpage``
|
||||
- ``swisscows``
|
||||
- ``qwant``
|
||||
- ``wikipedia``
|
||||
|
||||
``default_lang``:
|
||||
Default search language - leave blank to detect from browser information or
|
||||
use codes from :origin:`searx/languages.py`.
|
||||
|
||||
``languages``:
|
||||
List of available languages - leave unset to use all codes from
|
||||
:origin:`searx/languages.py`. Otherwise list codes of available languages.
|
||||
The ``all`` value is shown as the ``Default language`` in the user interface
|
||||
(in most cases, it is meant to send the query without a language parameter ;
|
||||
in some cases, it means the English language) Example:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
languages:
|
||||
- all
|
||||
- en
|
||||
- en-US
|
||||
- de
|
||||
- it-IT
|
||||
- fr
|
||||
- fr-BE
|
||||
|
||||
``ban_time_on_fail``:
|
||||
Ban time in seconds after engine errors.
|
||||
|
||||
``max_ban_time_on_fail``:
|
||||
Max ban time in seconds after engine errors.
|
||||
|
||||
``formats``:
|
||||
Result formats available from web, remove format to deny access (use lower
|
||||
case).
|
||||
|
||||
- ``html``
|
||||
- ``csv``
|
||||
- ``json``
|
||||
- ``rss``
|
||||
|
||||
.. _settings server:
|
||||
|
||||
``server:``
|
||||
-----------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
server:
|
||||
base_url: false # set custom base_url (or false)
|
||||
port: 8888
|
||||
bind_address: "127.0.0.1" # address to listen on
|
||||
secret_key: "ultrasecretkey" # change this!
|
||||
limiter: false
|
||||
image_proxy: false # proxying image results through SearXNG
|
||||
default_http_headers:
|
||||
X-Content-Type-Options : nosniff
|
||||
X-XSS-Protection : 1; mode=block
|
||||
X-Download-Options : noopen
|
||||
X-Robots-Tag : noindex, nofollow
|
||||
Referrer-Policy : no-referrer
|
||||
|
||||
.. sidebar:: buildenv
|
||||
|
||||
Changing a value tagged by :ref:`buildenv <make buildenv>`, needs to
|
||||
rebuild instance's environment :ref:`utils/brand.env <make buildenv>`.
|
||||
|
||||
``base_url`` : :ref:`buildenv SEARXNG_URL <make buildenv>`
|
||||
The base URL where SearXNG is deployed. Used to create correct inbound links.
|
||||
If you change the value, don't forget to rebuild instance's environment
|
||||
(:ref:`utils/brand.env <make buildenv>`)
|
||||
|
||||
``port`` & ``bind_address``: :ref:`buildenv SEARXNG_PORT & SEARXNG_BIND_ADDRESS <make buildenv>`
|
||||
Port number and *bind address* of the SearXNG web application if you run it
|
||||
directly using ``python searx/webapp.py``. Doesn't apply to SearXNG running on
|
||||
Apache or Nginx.
|
||||
|
||||
``secret_key`` : ``$SEARXNG_SECRET``
|
||||
Used for cryptography purpose.
|
||||
|
||||
.. _limiter:
|
||||
|
||||
``limiter`` :
|
||||
Rate limit the number of request on the instance, block some bots. The
|
||||
:ref:`limiter plugin` requires a :ref:`settings redis` database.
|
||||
|
||||
.. _image_proxy:
|
||||
|
||||
``image_proxy`` :
|
||||
Allow your instance of SearXNG of being able to proxy images. Uses memory space.
|
||||
|
||||
.. _HTTP headers: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
|
||||
|
||||
``default_http_headers`` :
|
||||
Set additional HTTP headers, see `#755 <https://github.com/searx/searx/issues/715>`__
|
||||
|
||||
|
||||
.. _settings ui:
|
||||
|
||||
``ui:``
|
||||
-------
|
||||
|
||||
.. _cache busting:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
ui:
|
||||
static_use_hash: false
|
||||
default_locale: ""
|
||||
query_in_title: false
|
||||
infinite_scroll: false
|
||||
center_alignment: false
|
||||
cache_url: https://web.archive.org/web/
|
||||
default_theme: simple
|
||||
theme_args:
|
||||
simple_style: auto
|
||||
|
||||
.. _static_use_hash:
|
||||
|
||||
``static_use_hash`` :
|
||||
Enables `cache busting`_ of static files.
|
||||
|
||||
``default_locale`` :
|
||||
SearXNG interface language. If blank, the locale is detected by using the
|
||||
browser language. If it doesn't work, or you are deploying a language
|
||||
specific instance of searx, a locale can be defined using an ISO language
|
||||
code, like ``fr``, ``en``, ``de``.
|
||||
|
||||
``query_in_title`` :
|
||||
When true, the result page's titles contains the query it decreases the
|
||||
privacy, since the browser can records the page titles.
|
||||
|
||||
``infinite_scroll``:
|
||||
When true, automatically loads the next page when scrolling to bottom of the current page.
|
||||
|
||||
``center_alignment`` : default ``false``
|
||||
When enabled, the results are centered instead of being in the left (or RTL)
|
||||
side of the screen. This setting only affects the *desktop layout*
|
||||
(:origin:`min-width: @tablet <searx/static/themes/simple/src/less/definitions.less>`)
|
||||
|
||||
.. cache_url:
|
||||
|
||||
``cache_url`` : ``https://web.archive.org/web/``
|
||||
URL prefix of the internet archive or cache, don't forgett trailing slash (if
|
||||
needed). The default is https://web.archive.org/web/ alternatives are:
|
||||
|
||||
- https://webcache.googleusercontent.com/search?q=cache:
|
||||
- https://archive.today/
|
||||
|
||||
``default_theme`` :
|
||||
Name of the theme you want to use by default on your SearXNG instance.
|
||||
|
||||
``theme_args.simple_style``:
|
||||
Style of simple theme: ``auto``, ``light``, ``dark``
|
||||
|
||||
``results_on_new_tab``:
|
||||
Open result links in a new tab by default.
|
||||
|
||||
|
||||
.. _settings redis:
|
||||
|
||||
``redis:``
|
||||
----------
|
||||
|
||||
.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
|
||||
|
||||
A redis DB can be connected by an URL, in :py:obj:`searx.redisdb` you
|
||||
will find a description to test your redis connection in SerXNG. When using
|
||||
sockets, don't forget to check the access rights on the socket::
|
||||
|
||||
ls -la /usr/local/searxng-redis/run/redis.sock
|
||||
srwxrwx--- 1 searxng-redis searxng-redis ... /usr/local/searxng-redis/run/redis.sock
|
||||
|
||||
In this example read/write access is given to the *searxng-redis* group. To get
|
||||
access rights to redis instance (the socket), your SearXNG (or even your
|
||||
developer) account needs to be added to the *searxng-redis* group.
|
||||
|
||||
``url``
|
||||
URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`::
|
||||
|
||||
redis://[[username]:[password]]@localhost:6379/0
|
||||
rediss://[[username]:[password]]@localhost:6379/0
|
||||
unix://[[username]:[password]]@/path/to/socket.sock?db=0
|
||||
|
||||
.. admonition:: Tip for developers
|
||||
|
||||
To set up a local redis instance, first set the socket path of the Redis DB
|
||||
in your YAML setting:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
redis:
|
||||
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
||||
|
||||
Then use the following commands to install the redis instance ::
|
||||
|
||||
$ ./manage redis.build
|
||||
$ sudo -H ./manage redis.install
|
||||
$ sudo -H ./manage redis.addgrp "${USER}"
|
||||
# don't forget to logout & login to get member of group
|
||||
|
||||
|
||||
.. _settings outgoing:
|
||||
|
||||
``outgoing:``
|
||||
-------------
|
||||
|
||||
Communication with search engines.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
outgoing:
|
||||
request_timeout: 2.0 # default timeout in seconds, can be override by engine
|
||||
max_request_timeout: 10.0 # the maximum timeout in seconds
|
||||
useragent_suffix: "" # information like an email address to the administrator
|
||||
pool_connections: 100 # Maximum number of allowable connections, or null
|
||||
# for no limits. The default is 100.
|
||||
pool_maxsize: 10 # Number of allowable keep-alive connections, or null
|
||||
# to always allow. The default is 10.
|
||||
enable_http2: true # See https://www.python-httpx.org/http2/
|
||||
# uncomment below section if you want to use a custom server certificate
|
||||
# see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
|
||||
# and https://www.python-httpx.org/compatibility/#ssl-configuration
|
||||
# verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
|
||||
#
|
||||
# uncomment below section if you want to use a proxyq see: SOCKS proxies
|
||||
# https://2.python-requests.org/en/latest/user/advanced/#proxies
|
||||
# are also supported: see
|
||||
# https://2.python-requests.org/en/latest/user/advanced/#socks
|
||||
#
|
||||
# proxies:
|
||||
# all://:
|
||||
# - http://proxy1:8080
|
||||
# - http://proxy2:8080
|
||||
#
|
||||
# using_tor_proxy: true
|
||||
#
|
||||
# Extra seconds to add in order to account for the time taken by the proxy
|
||||
#
|
||||
# extra_proxy_timeout: 10.0
|
||||
#
|
||||
|
||||
``request_timeout`` :
|
||||
Global timeout of the requests made to others engines in seconds. A bigger
|
||||
timeout will allow to wait for answers from slow engines, but in consequence
|
||||
will slow SearXNG reactivity (the result page may take the time specified in the
|
||||
timeout to load). Can be override by :ref:`settings engine`
|
||||
|
||||
``useragent_suffix`` :
|
||||
Suffix to the user-agent SearXNG uses to send requests to others engines. If an
|
||||
engine wish to block you, a contact info here may be useful to avoid that.
|
||||
|
||||
``keepalive_expiry`` :
|
||||
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
|
||||
|
||||
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
|
||||
|
||||
``proxies`` :
|
||||
Define one or more proxies you wish to use, see `httpx proxies`_.
|
||||
If there are more than one proxy for one protocol (http, https),
|
||||
requests to the engines are distributed in a round-robin fashion.
|
||||
|
||||
``source_ips`` :
|
||||
If you use multiple network interfaces, define from which IP the requests must
|
||||
be made. Example:
|
||||
|
||||
* ``0.0.0.0`` any local IPv4 address.
|
||||
* ``::`` any local IPv6 address.
|
||||
* ``192.168.0.1``
|
||||
* ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses
|
||||
* ``fe80::60a2:1691:e5a2:ee1f``
|
||||
* ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.
|
||||
* ``[ 192.168.0.1, fe80::/126 ]``
|
||||
|
||||
``retries`` :
|
||||
Number of retry in case of an HTTP error. On each retry, SearXNG uses an
|
||||
different proxy and source ip.
|
||||
|
||||
``retry_on_http_error`` :
|
||||
Retry request on some HTTP status code.
|
||||
|
||||
Example:
|
||||
|
||||
* ``true`` : on HTTP status code between 400 and 599.
|
||||
* ``403`` : on HTTP status code 403.
|
||||
* ``[403, 429]``: on HTTP status code 403 and 429.
|
||||
|
||||
``enable_http2`` :
|
||||
Enable by default. Set to ``false`` to disable HTTP/2.
|
||||
|
||||
.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
|
||||
.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
|
||||
|
||||
``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
|
||||
Allow to specify a path to certificate.
|
||||
see `httpx verification defaults`_.
|
||||
|
||||
In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
|
||||
``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
|
||||
see `httpx ssl configuration`_.
|
||||
|
||||
``max_redirects`` :
|
||||
30 by default. Maximum redirect before it is an error.
|
||||
|
||||
``categories_as_tabs:``
|
||||
-----------------------
|
||||
|
||||
A list of the categories that are displayed as tabs in the user interface.
|
||||
Categories not listed here can still be searched with the :ref:`search-syntax`.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
categories_as_tabs:
|
||||
general:
|
||||
images:
|
||||
videos:
|
||||
news:
|
||||
map:
|
||||
music:
|
||||
it:
|
||||
science:
|
||||
files:
|
||||
social media:
|
||||
|
||||
.. _settings engine:
|
||||
|
||||
Engine settings
|
||||
===============
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`configured engines`
|
||||
- :ref:`engines-dev`
|
||||
|
||||
In the code example below a *full fledged* example of a YAML setup from a dummy
|
||||
engine is shown. Most of the options have a default value or even are optional.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: example engine
|
||||
engine: example
|
||||
shortcut: demo
|
||||
base_url: 'https://{language}.example.com/'
|
||||
send_accept_language_header: false
|
||||
categories: general
|
||||
timeout: 3.0
|
||||
api_key: 'apikey'
|
||||
disabled: false
|
||||
language: en_US
|
||||
tokens: [ 'my-secret-token' ]
|
||||
weigth: 1
|
||||
display_error_messages: true
|
||||
about:
|
||||
website: https://example.com
|
||||
wikidata_id: Q306656
|
||||
official_api_documentation: https://example.com/api-doc
|
||||
use_official_api: true
|
||||
require_api_key: true
|
||||
results: HTML
|
||||
enable_http: false
|
||||
enable_http2: false
|
||||
retries: 1
|
||||
retry_on_http_error: true # or 403 or [404, 429]
|
||||
max_connections: 100
|
||||
max_keepalive_connections: 10
|
||||
keepalive_expiry: 5.0
|
||||
proxies:
|
||||
http:
|
||||
- http://proxy1:8080
|
||||
- http://proxy2:8080
|
||||
https:
|
||||
- http://proxy1:8080
|
||||
- http://proxy2:8080
|
||||
- socks5://user:password@proxy3:1080
|
||||
- socks5h://user:password@proxy4:1080
|
||||
|
||||
``name`` :
|
||||
Name that will be used across SearXNG to define this engine. In settings, on
|
||||
the result page...
|
||||
|
||||
``engine`` :
|
||||
Name of the python file used to handle requests and responses to and from this
|
||||
search engine.
|
||||
|
||||
``shortcut`` :
|
||||
Code used to execute bang requests (in this case using ``!bi``)
|
||||
|
||||
``base_url`` : optional
|
||||
Part of the URL that should be stable across every request. Can be useful to
|
||||
use multiple sites using only one engine, or updating the site URL without
|
||||
touching at the code.
|
||||
|
||||
``send_accept_language_header`` :
|
||||
Several engines that support languages (or regions) deal with the HTTP header
|
||||
``Accept-Language`` to build a response that fits to the locale. When this
|
||||
option is activated, the language (locale) that is selected by the user is used
|
||||
to build and send a ``Accept-Language`` header in the request to the origin
|
||||
search engine.
|
||||
|
||||
``categories`` : optional
|
||||
Define in which categories this engine will be active. Most of the time, it is
|
||||
defined in the code of the engine, but in a few cases it is useful, like when
|
||||
describing multiple search engine using the same code.
|
||||
|
||||
``timeout`` : optional
|
||||
Timeout of the search with the current search engine. **Be careful, it will
|
||||
modify the global timeout of SearXNG.**
|
||||
|
||||
``api_key`` : optional
|
||||
In a few cases, using an API needs the use of a secret key. How to obtain them
|
||||
is described in the file.
|
||||
|
||||
``disabled`` : optional
|
||||
To disable by default the engine, but not deleting it. It will allow the user
|
||||
to manually activate it in the settings.
|
||||
|
||||
``language`` : optional
|
||||
If you want to use another language for a specific engine, you can define it
|
||||
by using the full ISO code of language and country, like ``fr_FR``, ``en_US``,
|
||||
``de_DE``.
|
||||
|
||||
``tokens`` : optional
|
||||
A list of secret tokens to make this engine *private*, more details see
|
||||
:ref:`private engines`.
|
||||
|
||||
``weigth`` : default ``1``
|
||||
Weighting of the results of this engine.
|
||||
|
||||
``display_error_messages`` : default ``true``
|
||||
When an engine returns an error, the message is displayed on the user interface.
|
||||
|
||||
``network`` : optional
|
||||
Use the network configuration from another engine.
|
||||
In addition, there are two default networks:
|
||||
|
||||
- ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)
|
||||
- ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)
|
||||
|
||||
.. note::
|
||||
|
||||
A few more options are possible, but they are pretty specific to some
|
||||
engines, and so won't be described here.
|
||||
|
||||
|
||||
Example: Multilingual Search
|
||||
----------------------------
|
||||
|
||||
SearXNG does not support true multilingual search. You have to use the language
|
||||
prefix in your search query when searching in a different language.
|
||||
|
||||
But there is a workaround: By adding a new search engine with a different
|
||||
language, SearXNG will search in your default and other language.
|
||||
|
||||
Example configuration in settings.yml for a German and English speaker:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
search:
|
||||
default_lang : "de"
|
||||
...
|
||||
|
||||
engines:
|
||||
- name : google english
|
||||
engine : google
|
||||
language : en
|
||||
...
|
||||
|
||||
When searching, the default google engine will return German results and
|
||||
"google english" will return English results.
|
||||
|
||||
|
||||
.. _settings use_default_settings:
|
||||
|
||||
use_default_settings
|
||||
====================
|
||||
|
||||
.. sidebar:: ``use_default_settings: true``
|
||||
|
||||
- :ref:`settings location`
|
||||
- :ref:`use_default_settings.yml`
|
||||
- :origin:`/etc/searxng/settings.yml <utils/templates/etc/searxng/settings.yml>`
|
||||
|
||||
The user defined ``settings.yml`` is loaded from the :ref:`settings location`
|
||||
and can relied on the default configuration :origin:`searx/settings.yml` using:
|
||||
|
||||
``use_default_settings: true``
|
||||
|
||||
``server:``
|
||||
In the following example, the actual settings are the default settings defined
|
||||
in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and
|
||||
the ``bind_address``:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings: true
|
||||
server:
|
||||
secret_key: "ultrasecretkey" # change this!
|
||||
bind_address: "0.0.0.0"
|
||||
|
||||
``engines:``
|
||||
With ``use_default_settings: true``, each settings can be override in a
|
||||
similar way, the ``engines`` section is merged according to the engine
|
||||
``name``. In this example, SearXNG will load all the engine and the arch linux
|
||||
wiki engine has a :ref:`token <private engines>`:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings: true
|
||||
server:
|
||||
secret_key: "ultrasecretkey" # change this!
|
||||
engines:
|
||||
- name: arch linux wiki
|
||||
tokens: ['$ecretValue']
|
||||
|
||||
``engines:`` / ``remove:``
|
||||
It is possible to remove some engines from the default settings. The following
|
||||
example is similar to the above one, but SearXNG doesn't load the the google
|
||||
engine:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings:
|
||||
engines:
|
||||
remove:
|
||||
- google
|
||||
server:
|
||||
secret_key: "ultrasecretkey" # change this!
|
||||
engines:
|
||||
- name: arch linux wiki
|
||||
tokens: ['$ecretValue']
|
||||
|
||||
``engines:`` / ``keep_only:``
|
||||
As an alternative, it is possible to specify the engines to keep. In the
|
||||
following example, SearXNG has only two engines:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings:
|
||||
engines:
|
||||
keep_only:
|
||||
- google
|
||||
- duckduckgo
|
||||
server:
|
||||
secret_key: "ultrasecretkey" # change this!
|
||||
engines:
|
||||
- name: google
|
||||
tokens: ['$ecretValue']
|
||||
- name: duckduckgo
|
||||
tokens: ['$ecretValue']
|
@ -0,0 +1,166 @@
|
||||
.. _sql engines:
|
||||
|
||||
===========
|
||||
SQL Engines
|
||||
===========
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- `SQLite <https://www.sqlite.org/index.html>`_
|
||||
- `PostgreSQL <https://www.postgresql.org>`_
|
||||
- `MySQL <https://www.mysql.com>`_
|
||||
|
||||
With the *SQL engines* you can bind SQL databases into SearXNG. The following
|
||||
Relational Database Management System (RDBMS) are supported:
|
||||
|
||||
- :ref:`engine sqlite`
|
||||
- :ref:`engine postgresql`
|
||||
- :ref:`engine mysql_server`
|
||||
|
||||
All of the engines above are just commented out in the :origin:`settings.yml
|
||||
<searx/settings.yml>`, as you have to set the required attributes for the
|
||||
engines, e.g. ``database:`` ...
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: ...
|
||||
engine: {sqlite|postgresql|mysql_server}
|
||||
database: ...
|
||||
result_template: {template_name}
|
||||
query_str: ...
|
||||
|
||||
By default, the engines use the ``key-value`` template for displaying results /
|
||||
see :origin:`simple <searx/templates/simple/result_templates/key-value.html>`
|
||||
theme. If you are not satisfied with the original result layout, you can use
|
||||
your own template, set ``result_template`` attribute to ``{template_name}`` and
|
||||
place the templates at::
|
||||
|
||||
searx/templates/{theme_name}/result_templates/{template_name}
|
||||
|
||||
If you do not wish to expose these engines on a public instance, you can still
|
||||
add them and limit the access by setting ``tokens`` as described in section
|
||||
:ref:`private engines`.
|
||||
|
||||
|
||||
Configure the engines
|
||||
=====================
|
||||
|
||||
The configuration of the new database engines are similar. You must put a valid
|
||||
SQL-SELECT query in ``query_str``. At the moment you can only bind at most one
|
||||
parameter in your query. By setting the attribute ``limit`` you can define how
|
||||
many results you want from the SQL server. Basically, it is the same as the
|
||||
``LIMIT`` keyword in SQL.
|
||||
|
||||
Please, do not include ``LIMIT`` or ``OFFSET`` in your SQL query as the engines
|
||||
rely on these keywords during paging. If you want to configure the number of
|
||||
returned results use the option ``limit``.
|
||||
|
||||
.. _engine sqlite:
|
||||
|
||||
SQLite
|
||||
------
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`sqlite.py <searx/engines/sqlite.py>`
|
||||
|
||||
.. _MediathekView: https://mediathekview.de/
|
||||
|
||||
SQLite is a small, fast and reliable SQL database engine. It does not require
|
||||
any extra dependency. To demonstrate the power of database engines, here is a
|
||||
more complex example which reads from a MediathekView_ (DE) movie database. For
|
||||
this example of the SQlite engine download the database:
|
||||
|
||||
- https://liste.mediathekview.de/filmliste-v2.db.bz2
|
||||
|
||||
and unpack into ``searx/data/filmliste-v2.db``. To search the database use e.g
|
||||
Query to test: ``!mediathekview concert``
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: mediathekview
|
||||
engine: sqlite
|
||||
disabled: False
|
||||
categories: general
|
||||
result_template: default.html
|
||||
database: searx/data/filmliste-v2.db
|
||||
query_str: >-
|
||||
SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
|
||||
COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
|
||||
description AS content
|
||||
FROM film
|
||||
WHERE title LIKE :wildcard OR description LIKE :wildcard
|
||||
ORDER BY duration DESC
|
||||
|
||||
|
||||
Extra Dependencies
|
||||
------------------
|
||||
|
||||
For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to
|
||||
install additional packages in Python's Virtual Environment of your SearXNG
|
||||
instance. To switch into the environment (:ref:`searxng-src`) you can use
|
||||
:ref:`searxng.sh`::
|
||||
|
||||
$ sudo utils/searxng.sh instance cmd bash
|
||||
(searxng-pyenv)$ pip install ...
|
||||
|
||||
|
||||
.. _engine postgresql:
|
||||
|
||||
PostgreSQL
|
||||
----------
|
||||
|
||||
.. _psycopg2: https://www.psycopg.org/install
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`postgresql.py <searx/engines/postgresql.py>`
|
||||
- ``pip install`` psycopg2_
|
||||
|
||||
PostgreSQL is a powerful and robust open source database. Before configuring
|
||||
the PostgreSQL engine, you must install the dependency ``psychopg2``. You can
|
||||
find an example configuration below:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: my_database
|
||||
engine: postgresql
|
||||
database: my_database
|
||||
username: searxng
|
||||
password: password
|
||||
query_str: 'SELECT * from my_table WHERE my_column = %(query)s'
|
||||
|
||||
.. _engine mysql_server:
|
||||
|
||||
MySQL
|
||||
-----
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :origin:`mysql_server.py <searx/engines/mysql_server.py>`
|
||||
- ``pip install`` :pypi:`mysql-connector-python <mysql-connector-python>`
|
||||
|
||||
MySQL is said to be the most popular open source database. Before enabling MySQL
|
||||
engine, you must install the package ``mysql-connector-python``.
|
||||
|
||||
The authentication plugin is configurable by setting ``auth_plugin`` in the
|
||||
attributes. By default it is set to ``caching_sha2_password``. This is an
|
||||
example configuration for querying a MySQL server:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: my_database
|
||||
engine: mysql_server
|
||||
database: my_database
|
||||
username: searxng
|
||||
password: password
|
||||
limit: 5
|
||||
query_str: 'SELECT * from my_table WHERE my_column=%(query)s'
|
||||
|
||||
|
||||
Acknowledgment
|
||||
==============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_.
|
||||
|
@ -0,0 +1,21 @@
|
||||
===========================
|
||||
Administrator documentation
|
||||
===========================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
installation
|
||||
installation-docker
|
||||
installation-scripts
|
||||
installation-searxng
|
||||
installation-uwsgi
|
||||
installation-nginx
|
||||
installation-apache
|
||||
update-searxng
|
||||
engines/index
|
||||
api
|
||||
architecture
|
||||
plugins
|
||||
buildhosts
|
@ -0,0 +1,388 @@
|
||||
.. _installation apache:
|
||||
|
||||
======
|
||||
Apache
|
||||
======
|
||||
|
||||
.. _Apache: https://httpd.apache.org/
|
||||
.. _Apache Debian:
|
||||
https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x):
|
||||
.. _apache2.README.Debian:
|
||||
https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian
|
||||
.. _Apache Arch Linux:
|
||||
https://wiki.archlinux.org/index.php/Apache_HTTP_Server
|
||||
.. _Apache Fedora:
|
||||
https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html
|
||||
.. _Apache directives:
|
||||
https://httpd.apache.org/docs/trunk/mod/directives.html
|
||||
.. _Getting Started:
|
||||
https://httpd.apache.org/docs/current/en/getting-started.html
|
||||
.. _Terms Used to Describe Directives:
|
||||
https://httpd.apache.org/docs/current/en/mod/directive-dict.html
|
||||
.. _Configuration Files:
|
||||
https://httpd.apache.org/docs/current/en/configuring.html
|
||||
.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost
|
||||
.. _LoadModule:
|
||||
https://httpd.apache.org/docs/mod/mod_so.html#loadmodule
|
||||
.. _IncludeOptional:
|
||||
https://httpd.apache.org/docs/mod/core.html#includeoptional
|
||||
.. _DocumentRoot:
|
||||
https://httpd.apache.org/docs/trunk/mod/core.html#documentroot
|
||||
.. _Location:
|
||||
https://httpd.apache.org/docs/trunk/mod/core.html#location
|
||||
.. _uWSGI Apache support:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html
|
||||
.. _mod_proxy_uwsgi:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi
|
||||
.. _mod_proxy_http:
|
||||
https://httpd.apache.org/docs/current/mod/mod_proxy_http.html
|
||||
.. _mod_proxy:
|
||||
https://httpd.apache.org/docs/current/mod/mod_proxy.html
|
||||
|
||||
|
||||
This section explains how to set up a SearXNG instance using the HTTP server Apache_.
|
||||
If you did use the :ref:`installation scripts` and do not have any special preferences
|
||||
you can install the :ref:`SearXNG site <apache searxng site>` using
|
||||
:ref:`searxng.sh <searxng.sh overview>`:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/searxng.sh install apache
|
||||
|
||||
If you have special interests or problems with setting up Apache, the following
|
||||
section might give you some guidance.
|
||||
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- `Apache Arch Linux`_
|
||||
- `Apache Debian`_
|
||||
- `apache2.README.Debian`_
|
||||
- `Apache Fedora`_
|
||||
- `Apache directives`_
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
The Apache HTTP server
|
||||
======================
|
||||
|
||||
If Apache_ is not installed, install it now. If apache_ is new to you, the
|
||||
`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe
|
||||
Directives`_ documentation gives first orientation. There is also a list of
|
||||
`Apache directives`_ *to keep in the pocket*.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H apt-get install apache2
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H pacman -S apache
|
||||
sudo -H systemctl enable httpd
|
||||
sudo -H systemctl start http
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H dnf install httpd
|
||||
sudo -H systemctl enable httpd
|
||||
sudo -H systemctl start httpd
|
||||
|
||||
Now at http://localhost you should see some kind of *Welcome* or *Test* page.
|
||||
How this default site is configured, depends on the linux distribution
|
||||
(compare `Apache directives`_).
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
less /etc/apache2/sites-enabled/000-default.conf
|
||||
|
||||
In this file, there is a line setting the `DocumentRoot`_ directive:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
DocumentRoot /var/www/html
|
||||
|
||||
And the *welcome* page is the HTML file at ``/var/www/html/index.html``.
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
less /etc/httpd/conf/httpd.conf
|
||||
|
||||
In this file, there is a line setting the `DocumentRoot`_ directive:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
DocumentRoot "/srv/http"
|
||||
<Directory "/srv/http">
|
||||
Options Indexes FollowSymLinks
|
||||
AllowOverride None
|
||||
Require all granted
|
||||
</Directory>
|
||||
|
||||
The *welcome* page of Arch Linux is a page showing the directory located
|
||||
at ``DocumentRoot``. This *directory* page is generated by the Module
|
||||
`mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
LoadModule autoindex_module modules/mod_autoindex.so
|
||||
...
|
||||
Include conf/extra/httpd-autoindex.conf
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code:: bash
|
||||
|
||||
less /etc/httpd/conf/httpd.conf
|
||||
|
||||
In this file, there is a line setting the ``DocumentRoot`` directive:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
DocumentRoot "/var/www/html"
|
||||
...
|
||||
<Directory "/var/www">
|
||||
AllowOverride None
|
||||
# Allow open access:
|
||||
Require all granted
|
||||
</Directory>
|
||||
|
||||
On fresh installations, the ``/var/www`` is empty and the *default
|
||||
welcome page* is shown, the configuration is located at::
|
||||
|
||||
less /etc/httpd/conf.d/welcome.conf
|
||||
|
||||
|
||||
.. _Debian's Apache layout:
|
||||
|
||||
Debian's Apache layout
|
||||
----------------------
|
||||
|
||||
Be aware, Debian's Apache layout is quite different from the standard Apache
|
||||
configuration. For details look at the apache2.README.Debian_
|
||||
(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
|
||||
Debian:
|
||||
|
||||
* :man:`apache2ctl`: Apache HTTP server control interface
|
||||
* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
|
||||
* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
|
||||
* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites
|
||||
|
||||
.. _apache modules:
|
||||
|
||||
Apache modules
|
||||
--------------
|
||||
|
||||
To load additional modules, in most distributions you have to un-comment the
|
||||
lines with the corresponding LoadModule_ directive, except in :ref:`Debian's
|
||||
Apache layout`.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
:ref:`Debian's Apache layout` uses :man:`a2enmod` and :man:`a2dismod` to
|
||||
activate or disable modules:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H a2enmod ssl
|
||||
sudo -H a2enmod headers
|
||||
sudo -H a2enmod proxy
|
||||
sudo -H a2enmod proxy_http
|
||||
sudo -H a2enmod proxy_uwsgi
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_
|
||||
directives:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
LoadModule ssl_module modules/mod_ssl.so
|
||||
LoadModule headers_module modules/mod_headers.so
|
||||
LoadModule proxy_module modules/mod_proxy.so
|
||||
LoadModule proxy_http_module modules/mod_proxy_http.so
|
||||
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_
|
||||
directives:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
LoadModule ssl_module modules/mod_ssl.so
|
||||
LoadModule headers_module modules/mod_headers.so
|
||||
LoadModule proxy_module modules/mod_proxy.so
|
||||
LoadModule proxy_http_module modules/mod_proxy_http.so
|
||||
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
|
||||
|
||||
|
||||
.. _apache sites:
|
||||
|
||||
Apache sites
|
||||
------------
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
In :ref:`Debian's Apache layout` you create a ``searxng.conf`` with the
|
||||
``<Location /searxng >`` directive and save this file in the *sites
|
||||
available* folder at ``/etc/apache2/sites-available``. To enable the
|
||||
``searxng.conf`` use :man:`a2ensite`:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H a2ensite searxng.conf
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_
|
||||
directive:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
IncludeOptional sites-enabled/*.conf
|
||||
|
||||
Create two folders, one for the *available sites* and one for the *enabled sites*:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
mkdir -p /etc/httpd/sites-available
|
||||
mkdir -p /etc/httpd/sites-enabled
|
||||
|
||||
Create configuration at ``/etc/httpd/sites-available`` and place a
|
||||
symlink to ``sites-enabled``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H ln -s /etc/httpd/sites-available/searxng.conf \
|
||||
/etc/httpd/sites-enabled/searxng.conf
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_
|
||||
directive:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
IncludeOptional sites-enabled/*.conf
|
||||
|
||||
Create two folders, one for the *available sites* and one for the *enabled sites*:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
mkdir -p /etc/httpd/sites-available
|
||||
mkdir -p /etc/httpd/sites-enabled
|
||||
|
||||
Create configuration at ``/etc/httpd/sites-available`` and place a
|
||||
symlink to ``sites-enabled``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H ln -s /etc/httpd/sites-available/searxng.conf \
|
||||
/etc/httpd/sites-enabled/searxng.conf
|
||||
|
||||
|
||||
.. _apache searxng site:
|
||||
|
||||
Apache's SearXNG site
|
||||
=====================
|
||||
|
||||
.. _mod_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi
|
||||
|
||||
.. sidebar:: uWSGI
|
||||
|
||||
Use mod_proxy_uwsgi_ / don't use the old mod_uwsgi_ anymore.
|
||||
|
||||
To proxy the incoming requests to the SearXNG instance Apache needs the
|
||||
mod_proxy_ module (:ref:`apache modules`).
|
||||
|
||||
.. sidebar:: HTTP headers
|
||||
|
||||
With ProxyPreserveHost_ the incoming ``Host`` header is passed to the proxied
|
||||
host.
|
||||
|
||||
Depending on what your SearXNG installation is listening on, you need a http
|
||||
mod_proxy_http_) or socket (mod_proxy_uwsgi_) communication to upstream.
|
||||
|
||||
The :ref:`installation scripts` installs the :ref:`reference setup
|
||||
<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default.
|
||||
You can install and activate your own ``searxng.conf`` like shown in
|
||||
:ref:`apache sites`.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: socket
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START apache socket
|
||||
:end-before: END apache socket
|
||||
|
||||
.. group-tab:: http
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START apache http
|
||||
:end-before: END apache http
|
||||
|
||||
.. _restart apache:
|
||||
|
||||
Restart service:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart apache2
|
||||
sudo -H service uwsgi restart searxng
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart httpd
|
||||
sudo -H systemctl restart uwsgi@searxng
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart httpd
|
||||
sudo -H touch /etc/uwsgi.d/searxng.ini
|
||||
|
||||
|
||||
disable logs
|
||||
============
|
||||
|
||||
For better privacy you can disable Apache logs. In the examples above activate
|
||||
one of the lines and `restart apache`_:
|
||||
|
||||
.. code:: apache
|
||||
|
||||
SetEnvIf Request_URI "/searxng" dontlog
|
||||
# CustomLog /dev/null combined env=dontlog
|
||||
|
||||
The ``CustomLog`` directive disables logs for the entire (virtual) server, use it
|
||||
when the URL of the service does not have a path component (``/searxng``), so when
|
||||
SearXNG is located at root (``/``).
|
@ -0,0 +1,194 @@
|
||||
.. _installation docker:
|
||||
|
||||
================
|
||||
Docker Container
|
||||
================
|
||||
|
||||
.. _ENTRYPOINT: https://docs.docker.com/engine/reference/builder/#entrypoint
|
||||
.. _searxng/searxng @dockerhub: https://hub.docker.com/r/searxng/searxng
|
||||
.. _searxng-docker: https://github.com/searxng/searxng-docker
|
||||
.. _[caddy]: https://hub.docker.com/_/caddy
|
||||
.. _Redis: https://redis.io/
|
||||
|
||||
----
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- `searxng/searxng @dockerhub`_
|
||||
- :origin:`Dockerfile`
|
||||
- `Docker overview <https://docs.docker.com/get-started/overview>`_
|
||||
- `Docker Cheat Sheet <https://docs.docker.com/get-started/docker_cheatsheet.pdf>`_
|
||||
- `Alpine Linux <https://alpinelinux.org>`_
|
||||
`(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__
|
||||
`apt packages <https://pkgs.alpinelinux.org/packages>`_
|
||||
- Alpine's ``/bin/sh`` is :man:`dash`
|
||||
|
||||
**If you intend to create a public instance using Docker, use our well maintained
|
||||
docker container**
|
||||
|
||||
- `searxng/searxng @dockerhub`_.
|
||||
|
||||
.. sidebar:: hint
|
||||
|
||||
The rest of this article is of interest only to those who want to create and
|
||||
maintain their own Docker images.
|
||||
|
||||
The sources are hosted at searxng-docker_ and the container includes:
|
||||
|
||||
- a HTTPS reverse proxy `[caddy]`_ and
|
||||
- a Redis_ DB
|
||||
|
||||
The `default SearXNG setup <https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml>`_
|
||||
of this container:
|
||||
|
||||
- enables :ref:`limiter <limiter>` to protect against bots
|
||||
- enables :ref:`image proxy <image_proxy>` for better privacy
|
||||
- enables :ref:`cache busting <static_use_hash>` to save bandwith
|
||||
|
||||
----
|
||||
|
||||
|
||||
Get Docker
|
||||
==========
|
||||
|
||||
If you plan to build and maintain a docker image by yourself, make sure you have
|
||||
`Docker installed <https://docs.docker.com/get-docker/>`_. On Linux don't
|
||||
forget to add your user to the docker group (log out and log back in so that
|
||||
your group membership is re-evaluated):
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ sudo usermod -a -G docker $USER
|
||||
|
||||
|
||||
searxng/searxng
|
||||
===============
|
||||
|
||||
.. sidebar:: ``docker run``
|
||||
|
||||
- `-\-rm <https://docs.docker.com/engine/reference/run/#clean-up---rm>`__
|
||||
automatically clean up when container exits
|
||||
- `-d <https://docs.docker.com/engine/reference/run/#detached--d>`__ start
|
||||
detached container
|
||||
- `-v <https://docs.docker.com/engine/reference/run/#volume-shared-filesystems>`__
|
||||
mount volume ``HOST:CONTAINER``
|
||||
|
||||
The docker image is based on :origin:`Dockerfile` and available from
|
||||
`searxng/searxng @dockerhub`_. Using the docker image is quite easy, for
|
||||
instance you can pull the `searxng/searxng @dockerhub`_ image and deploy a local
|
||||
instance using `docker run <https://docs.docker.com/engine/reference/run/>`_:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ mkdir my-instance
|
||||
$ cd my-instance
|
||||
$ export PORT=8080
|
||||
$ docker pull searxng/searxng
|
||||
$ docker run --rm \
|
||||
-d -p ${PORT}:8080 \
|
||||
-v "${PWD}/searxng:/etc/searxng" \
|
||||
-e "BASE_URL=http://localhost:$PORT/" \
|
||||
-e "INSTANCE_NAME=my-instance" \
|
||||
searxng/searxng
|
||||
2f998.... # container's ID
|
||||
|
||||
Open your WEB browser and visit the URL:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ xdg-open "http://localhost:$PORT"
|
||||
|
||||
Inside ``${PWD}/searxng``, you will find ``settings.yml`` and ``uwsgi.ini``. You
|
||||
can modify these files according to your needs and restart the Docker image.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ docker container restart 2f998
|
||||
|
||||
Use command ``container ls`` to list running containers, add flag `-a
|
||||
<https://docs.docker.com/engine/reference/commandline/container_ls>`__ to list
|
||||
exited containers also. With ``container stop`` a running container can be
|
||||
stoped. To get rid of a container use ``container rm``:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ docker container ls
|
||||
CONTAINER ID IMAGE COMMAND CREATED ...
|
||||
2f998d725993 searxng/searxng "/sbin/tini -- /usr/…" 7 minutes ago ...
|
||||
|
||||
$ docker container stop 2f998
|
||||
$ docker container rm 2f998
|
||||
|
||||
.. sidebar:: Warning
|
||||
|
||||
This might remove all docker items, not only those from SearXNG.
|
||||
|
||||
If you won't use docker anymore and want to get rid of all conatiners & images
|
||||
use the following *prune* command:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ docker stop $(docker ps -aq) # stop all containers
|
||||
$ docker system prune # make some housekeeping
|
||||
$ docker rmi -f $(docker images -q) # drop all images
|
||||
|
||||
|
||||
shell inside container
|
||||
----------------------
|
||||
|
||||
.. sidebar:: Bashism
|
||||
|
||||
- `A tale of two shells: bash or dash <https://lwn.net/Articles/343924/>`_
|
||||
- `How to make bash scripts work in dash <http://mywiki.wooledge.org/Bashism>`_
|
||||
- `Checking for Bashisms <https://dev.to/bowmanjd/writing-bash-scripts-that-are-not-only-bash-checking-for-bashisms-and-testing-with-dash-1bli>`_
|
||||
|
||||
Like in many other distributions, Alpine's `/bin/sh
|
||||
<https://wiki.ubuntu.com/DashAsBinSh>`__ is :man:`dash`. Dash is meant to be
|
||||
`POSIX-compliant <https://pubs.opengroup.org/onlinepubs/9699919799>`__.
|
||||
Compared to debian, in the Alpine image :man:`bash` is not installed. The
|
||||
:origin:`dockerfiles/docker-entrypoint.sh` script is checked *against dash*
|
||||
(``make tests.shell``).
|
||||
|
||||
To open a shell inside the container:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ docker exec -it 2f998 sh
|
||||
|
||||
|
||||
Build the image
|
||||
===============
|
||||
|
||||
It's also possible to build SearXNG from the embedded :origin:`Dockerfile`::
|
||||
|
||||
$ git clone https://github.com/searxng/searxng.git
|
||||
$ cd searxng
|
||||
$ make docker.build
|
||||
...
|
||||
Successfully built 49586c016434
|
||||
Successfully tagged searxng/searxng:latest
|
||||
Successfully tagged searxng/searxng:1.0.0-209-9c823800-dirty
|
||||
|
||||
$ docker images
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
searxng/searxng 1.0.0-209-9c823800-dirty 49586c016434 13 minutes ago 308MB
|
||||
searxng/searxng latest 49586c016434 13 minutes ago 308MB
|
||||
alpine 3.13 6dbb9cc54074 3 weeks ago 5.61MB
|
||||
|
||||
|
||||
Command line
|
||||
============
|
||||
|
||||
.. sidebar:: docker run
|
||||
|
||||
Use flags ``-it`` for `interactive processes
|
||||
<https://docs.docker.com/engine/reference/run/#foreground>`__.
|
||||
|
||||
In the :origin:`Dockerfile` the ENTRYPOINT_ is defined as
|
||||
:origin:`dockerfiles/docker-entrypoint.sh`
|
||||
|
||||
.. code:: sh
|
||||
|
||||
docker run --rm -it searxng/searxng -h
|
||||
|
||||
.. program-output:: ../dockerfiles/docker-entrypoint.sh -h
|
@ -0,0 +1,252 @@
|
||||
.. _installation nginx:
|
||||
|
||||
=====
|
||||
NGINX
|
||||
=====
|
||||
|
||||
.. _nginx:
|
||||
https://docs.nginx.com/nginx/admin-guide/
|
||||
.. _nginx server configuration:
|
||||
https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers
|
||||
.. _nginx beginners guide:
|
||||
https://nginx.org/en/docs/beginners_guide.html
|
||||
.. _Getting Started wiki:
|
||||
https://www.nginx.com/resources/wiki/start/
|
||||
.. _uWSGI support from nginx:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html
|
||||
.. _uwsgi_params:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx
|
||||
.. _SCRIPT_NAME:
|
||||
https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name
|
||||
|
||||
This section explains how to set up a SearXNG instance using the HTTP server nginx_.
|
||||
If you have used the :ref:`installation scripts` and do not have any special preferences
|
||||
you can install the :ref:`SearXNG site <nginx searxng site>` using
|
||||
:ref:`searxng.sh <searxng.sh overview>`:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/searxng.sh install nginx
|
||||
|
||||
If you have special interests or problems with setting up nginx, the following
|
||||
section might give you some guidance.
|
||||
|
||||
|
||||
.. sidebar:: further reading
|
||||
|
||||
- nginx_
|
||||
- `nginx beginners guide`_
|
||||
- `nginx server configuration`_
|
||||
- `Getting Started wiki`_
|
||||
- `uWSGI support from nginx`_
|
||||
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
The nginx HTTP server
|
||||
=====================
|
||||
|
||||
If nginx_ is not installed, install it now.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H apt-get install nginx
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
sudo -H pacman -S nginx-mainline
|
||||
sudo -H systemctl enable nginx
|
||||
sudo -H systemctl start nginx
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
sudo -H dnf install nginx
|
||||
sudo -H systemctl enable nginx
|
||||
sudo -H systemctl start nginx
|
||||
|
||||
Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you
|
||||
see a *Fedora Webserver - Test Page*. The test page comes from the default
|
||||
`nginx server configuration`_. How this default site is configured,
|
||||
depends on the linux distribution:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
less /etc/nginx/nginx.conf
|
||||
|
||||
There is one line that includes site configurations from:
|
||||
|
||||
.. code:: nginx
|
||||
|
||||
include /etc/nginx/sites-enabled/*;
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
less /etc/nginx/nginx.conf
|
||||
|
||||
There is a configuration section named ``server``:
|
||||
|
||||
.. code-block:: nginx
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
# ...
|
||||
}
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
less /etc/nginx/nginx.conf
|
||||
|
||||
There is one line that includes site configurations from:
|
||||
|
||||
.. code:: nginx
|
||||
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
|
||||
.. _nginx searxng site:
|
||||
|
||||
NGINX's SearXNG site
|
||||
====================
|
||||
|
||||
Now you have to create a configuration file (``searxng.conf``) for the SearXNG
|
||||
site. If nginx_ is new to you, the `nginx beginners guide`_ is a good starting
|
||||
point and the `Getting Started wiki`_ is always a good resource *to keep in the
|
||||
pocket*.
|
||||
|
||||
Depending on what your SearXNG installation is listening on, you need a http or socket
|
||||
communication to upstream.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: socket
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START nginx socket
|
||||
:end-before: END nginx socket
|
||||
|
||||
.. group-tab:: http
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START nginx http
|
||||
:end-before: END nginx http
|
||||
|
||||
The :ref:`installation scripts` installs the :ref:`reference setup
|
||||
<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
Create configuration at ``/etc/nginx/sites-available/`` and place a
|
||||
symlink to ``sites-enabled``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H ln -s /etc/nginx/sites-available/searxng.conf \
|
||||
/etc/nginx/sites-enabled/searxng.conf
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
In the ``/etc/nginx/nginx.conf`` file, in the ``server`` section add a
|
||||
`include <https://nginx.org/en/docs/ngx_core_module.html#include>`_
|
||||
directive:
|
||||
|
||||
.. code:: nginx
|
||||
|
||||
server {
|
||||
# ...
|
||||
include /etc/nginx/default.d/*.conf;
|
||||
# ...
|
||||
}
|
||||
|
||||
Create two folders, one for the *available sites* and one for the *enabled sites*:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
mkdir -p /etc/nginx/default.d
|
||||
mkdir -p /etc/nginx/default.apps-available
|
||||
|
||||
Create configuration at ``/etc/nginx/default.apps-available`` and place a
|
||||
symlink to ``default.d``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \
|
||||
/etc/nginx/default.d/searxng.conf
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
Create a folder for the *available sites*:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
mkdir -p /etc/nginx/default.apps-available
|
||||
|
||||
Create configuration at ``/etc/nginx/default.apps-available`` and place a
|
||||
symlink to ``conf.d``:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \
|
||||
/etc/nginx/conf.d/searxng.conf
|
||||
|
||||
Restart services:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart nginx
|
||||
sudo -H service uwsgi restart searxng
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart nginx
|
||||
sudo -H systemctl restart uwsgi@searxng
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo -H systemctl restart nginx
|
||||
sudo -H touch /etc/uwsgi.d/searxng.ini
|
||||
|
||||
|
||||
Disable logs
|
||||
============
|
||||
|
||||
For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``.
|
||||
|
||||
.. code:: nginx
|
||||
|
||||
http {
|
||||
# ...
|
||||
access_log /dev/null;
|
||||
error_log /dev/null;
|
||||
# ...
|
||||
}
|
@ -0,0 +1,62 @@
|
||||
.. _installation scripts:
|
||||
|
||||
===================
|
||||
Installation Script
|
||||
===================
|
||||
|
||||
.. sidebar:: Update the OS first!
|
||||
|
||||
To avoid unwanted side effects, update your OS before installing SearXNG.
|
||||
|
||||
The following will install a setup as shown in :ref:`the reference architecture
|
||||
<arch public>`. First you need to get a clone of the repository. The clone is only needed for
|
||||
the installation procedure and some maintenance tasks.
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- :ref:`toolboxing`
|
||||
|
||||
Jump to a folder that is readable by *others* and start to clone SearXNG,
|
||||
alternatively you can create your own fork and clone from there.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ cd ~/Downloads
|
||||
$ git clone https://github.com/searxng/searxng.git searxng
|
||||
$ cd searxng
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- :ref:`inspect searxng`
|
||||
|
||||
To install a SearXNG :ref:`reference setup <use_default_settings.yml>`
|
||||
including a :ref:`uWSGI setup <architecture uWSGI>` as described in the
|
||||
:ref:`installation basic` and in the :ref:`searxng uwsgi` section type:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/searxng.sh install all
|
||||
|
||||
.. attention::
|
||||
|
||||
For the installation procedure, use a *sudoer* login to run the scripts. If
|
||||
you install from ``root``, take into account that the scripts are creating a
|
||||
``searxng`` user. In the installation procedure this new created user does
|
||||
need read access to the cloned SearXNG repository, which is not the case if you clone
|
||||
it into a folder below ``/root``!
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- :ref:`update searxng`
|
||||
|
||||
.. _caddy: https://hub.docker.com/_/caddy
|
||||
|
||||
When all services are installed and running fine, you can add SearXNG to your
|
||||
HTTP server. We do not have any preferences for the HTTP server, you can use
|
||||
whatever you prefer.
|
||||
|
||||
We use caddy in our :ref:`docker image <installation docker>` and we have
|
||||
implemented installation procedures for:
|
||||
|
||||
- :ref:`installation nginx`
|
||||
- :ref:`installation apache`
|
@ -0,0 +1,132 @@
|
||||
.. _installation basic:
|
||||
|
||||
=========================
|
||||
Step by step installation
|
||||
=========================
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
In this section we show the setup of a SearXNG instance that will be installed
|
||||
by the :ref:`installation scripts`.
|
||||
|
||||
.. _install packages:
|
||||
|
||||
Install packages
|
||||
================
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START distro-packages
|
||||
:end-before: END distro-packages
|
||||
|
||||
.. hint::
|
||||
|
||||
This installs also the packages needed by :ref:`searxng uwsgi`
|
||||
|
||||
.. _create searxng user:
|
||||
|
||||
Create user
|
||||
===========
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START create user
|
||||
:end-before: END create user
|
||||
|
||||
.. _searxng-src:
|
||||
|
||||
Install SearXNG & dependencies
|
||||
==============================
|
||||
|
||||
Start a interactive shell from new created user and clone SearXNG:
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START clone searxng
|
||||
:end-before: END clone searxng
|
||||
|
||||
In the same shell create *virtualenv*:
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START create virtualenv
|
||||
:end-before: END create virtualenv
|
||||
|
||||
To install SearXNG's dependencies, exit the SearXNG *bash* session you opened above
|
||||
and start a new one. Before installing, check if your *virtualenv* was sourced
|
||||
from the login (*~/.profile*):
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START manage.sh update_packages
|
||||
:end-before: END manage.sh update_packages
|
||||
|
||||
.. tip::
|
||||
|
||||
Open a second terminal for the configuration tasks and leave the ``(searx)$``
|
||||
terminal open for the tasks below.
|
||||
|
||||
|
||||
.. _use_default_settings.yml:
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
.. sidebar:: ``use_default_settings: True``
|
||||
|
||||
- :ref:`settings global`
|
||||
- :ref:`settings location`
|
||||
- :ref:`settings use_default_settings`
|
||||
- :origin:`/etc/searxng/settings.yml <utils/templates/etc/searxng/settings.yml>`
|
||||
|
||||
To create a initial ``/etc/searxng/settings.yml`` we recommend to start with a
|
||||
copy of the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup
|
||||
:ref:`use default settings <settings use_default_settings>` from
|
||||
:origin:`searx/settings.yml` and is shown in the tab *"Use default settings"*
|
||||
below. This setup:
|
||||
|
||||
- enables :ref:`limiter <limiter>` to protect against bots
|
||||
- enables :ref:`image proxy <image_proxy>` for better privacy
|
||||
- enables :ref:`cache busting <static_use_hash>` to save bandwith
|
||||
|
||||
Modify the ``/etc/searxng/settings.yml`` to your needs:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Use default settings
|
||||
|
||||
.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml
|
||||
:language: yaml
|
||||
:end-before: # hostname_replace:
|
||||
|
||||
To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml`
|
||||
|
||||
.. group-tab:: searx/settings.yml
|
||||
|
||||
.. literalinclude:: ../../searx/settings.yml
|
||||
:language: yaml
|
||||
:end-before: # hostname_replace:
|
||||
|
||||
To see the entire file jump to :origin:`searx/settings.yml`
|
||||
|
||||
For a *minimal setup* you need to set ``server:secret_key``.
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng config
|
||||
:end-before: END searxng config
|
||||
|
||||
|
||||
Check
|
||||
=====
|
||||
|
||||
To check your SearXNG setup, optional enable debugging and start the *webapp*.
|
||||
SearXNG looks at the exported environment ``$SEARXNG_SETTINGS_PATH`` for a
|
||||
configuration file.
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START check searxng installation
|
||||
:end-before: END check searxng installation
|
||||
|
||||
If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the
|
||||
debug option in ``settings.yml``. You can now exit SearXNG user bash session (enter exit
|
||||
command twice). At this point SearXNG is not demonized; uwsgi allows this.
|
||||
|
@ -0,0 +1,268 @@
|
||||
.. _searxng uwsgi:
|
||||
|
||||
=====
|
||||
uWSGI
|
||||
=====
|
||||
|
||||
.. sidebar:: further reading
|
||||
|
||||
- `systemd.unit`_
|
||||
- `uWSGI Emperor`_
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
|
||||
.. _One service per app in systemd:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
|
||||
.. _uWSGI Emperor:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
|
||||
.. _uwsgi ini file:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files
|
||||
.. _systemd unit template:
|
||||
http://0pointer.de/blog/projects/instances.html
|
||||
|
||||
|
||||
Origin uWSGI
|
||||
============
|
||||
|
||||
.. _Tyrant mode:
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting
|
||||
|
||||
How uWSGI is implemented by distributors varies. The uWSGI project itself
|
||||
recommends two methods:
|
||||
|
||||
1. `systemd.unit`_ template file as described here `One service per app in systemd`_:
|
||||
|
||||
There is one `systemd unit template`_ on the system installed and one `uwsgi
|
||||
ini file`_ per uWSGI-app placed at dedicated locations. Take archlinux and a
|
||||
``searxng.ini`` as example::
|
||||
|
||||
systemd template unit: /usr/lib/systemd/system/uwsgi@.service
|
||||
contains: [Service]
|
||||
ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini
|
||||
|
||||
SearXNG application: /etc/uwsgi/searxng.ini
|
||||
links to: /etc/uwsgi/apps-available/searxng.ini
|
||||
|
||||
The SearXNG app (template ``/etc/uwsgi/%I.ini``) can be maintained as known
|
||||
from common systemd units:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ systemctl enable uwsgi@searxng
|
||||
$ systemctl start uwsgi@searxng
|
||||
$ systemctl restart uwsgi@searxng
|
||||
$ systemctl stop uwsgi@searxng
|
||||
|
||||
2. The `uWSGI Emperor`_ which fits for maintaining a large range of uwsgi
|
||||
apps and there is a `Tyrant mode`_ to secure multi-user hosting.
|
||||
|
||||
The Emperor mode is a special uWSGI instance that will monitor specific
|
||||
events. The Emperor mode (the service) is started by a (common, not template)
|
||||
systemd unit.
|
||||
|
||||
The Emperor service will scan specific directories for `uwsgi ini file`_\s
|
||||
(also know as *vassals*). If a *vassal* is added, removed or the timestamp is
|
||||
modified, a corresponding action takes place: a new uWSGI instance is started,
|
||||
reload or stopped. Take Fedora and a ``searxng.ini`` as example::
|
||||
|
||||
to install & start SearXNG instance create --> /etc/uwsgi.d/searxng.ini
|
||||
to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini
|
||||
to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini
|
||||
|
||||
|
||||
Distributors
|
||||
============
|
||||
|
||||
The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors
|
||||
mostly offer their users, even if they differ in the way they implement both
|
||||
modes and their defaults. Another point they might differ in is the packaging of
|
||||
plugins (if so, compare :ref:`install packages`) and what the default python
|
||||
interpreter is (python2 vs. python3).
|
||||
|
||||
While archlinux does not start a uWSGI service by default, Fedora (RHEL) starts
|
||||
a Emperor in `Tyrant mode`_ by default (you should have read :ref:`uWSGI Tyrant
|
||||
mode pitfalls`). Worth to know; debian (ubuntu) follow a complete different
|
||||
approach, read see :ref:`Debian's uWSGI layout`.
|
||||
|
||||
.. _Debian's uWSGI layout:
|
||||
|
||||
Debian's uWSGI layout
|
||||
---------------------
|
||||
|
||||
.. _uwsgi.README.Debian:
|
||||
https://salsa.debian.org/uwsgi-team/uwsgi/-/raw/debian/latest/debian/uwsgi.README.Debian
|
||||
|
||||
Be aware, Debian's uWSGI layout is quite different from the standard uWSGI
|
||||
configuration. Your are familiar with :ref:`Debian's Apache layout`? .. they do a
|
||||
similar thing for the uWSGI infrastructure. The folders are::
|
||||
|
||||
/etc/uwsgi/apps-available/
|
||||
/etc/uwsgi/apps-enabled/
|
||||
|
||||
The `uwsgi ini file`_ is enabled by a symbolic link::
|
||||
|
||||
ln -s /etc/uwsgi/apps-available/searxng.ini /etc/uwsgi/apps-enabled/
|
||||
|
||||
More details can be found in the uwsgi.README.Debian_
|
||||
(``/usr/share/doc/uwsgi/README.Debian.gz``). Some commands you should know on
|
||||
Debian:
|
||||
|
||||
.. code:: none
|
||||
|
||||
Commands recognized by init.d script
|
||||
====================================
|
||||
|
||||
You can issue to init.d script following commands:
|
||||
* start | starts daemon
|
||||
* stop | stops daemon
|
||||
* reload | sends to daemon SIGHUP signal
|
||||
* force-reload | sends to daemon SIGTERM signal
|
||||
* restart | issues 'stop', then 'start' commands
|
||||
* status | shows status of daemon instance (running/not running)
|
||||
|
||||
'status' command must be issued with exactly one argument: '<confname>'.
|
||||
|
||||
Controlling specific instances of uWSGI
|
||||
=======================================
|
||||
|
||||
You could control specific instance(s) by issuing:
|
||||
|
||||
SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi <command> <confname> <confname>...
|
||||
|
||||
where:
|
||||
* <command> is one of 'start', 'stop' etc.
|
||||
* <confname> is the name of configuration file (without extension)
|
||||
|
||||
For example, this is how instance for /etc/uwsgi/apps-enabled/hello.xml is
|
||||
started:
|
||||
|
||||
SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi start hello
|
||||
|
||||
|
||||
.. _uWSGI maintenance:
|
||||
|
||||
uWSGI maintenance
|
||||
=================
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-description ubuntu-20.04
|
||||
:end-before: END searxng uwsgi-description ubuntu-20.04
|
||||
|
||||
.. hotfix: a bug group-tab need this comment
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-description arch
|
||||
:end-before: END searxng uwsgi-description arch
|
||||
|
||||
.. hotfix: a bug group-tab need this comment
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-description fedora
|
||||
:end-before: END searxng uwsgi-description fedora
|
||||
|
||||
|
||||
.. _uwsgi setup:
|
||||
|
||||
uWSGI setup
|
||||
===========
|
||||
|
||||
Create the configuration ini-file according to your distribution and restart the
|
||||
uwsgi application. As shown below, the :ref:`installation scripts` installs by
|
||||
default:
|
||||
|
||||
- a uWSGI setup that listens on a socket and
|
||||
- enables :ref:`cache busting <static_use_hash>`.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: Ubuntu / debian
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-appini ubuntu-20.04
|
||||
:end-before: END searxng uwsgi-appini ubuntu-20.04
|
||||
|
||||
.. hotfix: a bug group-tab need this comment
|
||||
|
||||
.. group-tab:: Arch Linux
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-appini arch
|
||||
:end-before: END searxng uwsgi-appini arch
|
||||
|
||||
.. hotfix: a bug group-tab need this comment
|
||||
|
||||
.. group-tab:: Fedora / RHEL
|
||||
|
||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||
:start-after: START searxng uwsgi-appini fedora
|
||||
:end-before: END searxng uwsgi-appini fedora
|
||||
|
||||
|
||||
.. _uWSGI Tyrant mode pitfalls:
|
||||
|
||||
Pitfalls of the Tyrant mode
|
||||
===========================
|
||||
|
||||
The implementation of the process owners and groups in the `Tyrant mode`_ is
|
||||
somewhat unusual and requires special consideration. In `Tyrant mode`_ mode the
|
||||
Emperor will run the vassal using the UID/GID of the vassal configuration file
|
||||
(user and group of the app ``.ini`` file).
|
||||
|
||||
.. _#2099@uWSGI: https://github.com/unbit/uwsgi/issues/2099
|
||||
.. _#752@uWSGI: https://github.com/unbit/uwsgi/pull/752
|
||||
.. _#2425uWSGI: https://github.com/unbit/uwsgi/issues/2425
|
||||
|
||||
Without option ``emperor-tyrant-initgroups=true`` in ``/etc/uwsgi.ini`` the
|
||||
process won't get the additional groups, but this option is not available in
|
||||
2.0.x branch (see `#2099@uWSGI`_) the feature `#752@uWSGI`_ has been merged (on
|
||||
Oct. 2014) to the master branch of uWSGI but had never been released; the last
|
||||
major release is from Dec. 2013, since the there had been only bugfix releases
|
||||
(see `#2425uWSGI`_). To shorten up:
|
||||
|
||||
**In Tyrant mode, there is no way to get additional groups, and the uWSGI
|
||||
process misses additional permissions that may be needed.**
|
||||
|
||||
For example on Fedora (RHEL): If you try to install a redis DB with socket
|
||||
communication and you want to connect to it from the SearXNG uWSGI, you will see a
|
||||
*Permission denied* in the log of your instance::
|
||||
|
||||
ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ...
|
||||
ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied.
|
||||
ERROR:searx.plugins.limiter: init limiter DB failed!!!
|
||||
|
||||
Even if your *searxng* user of the uWSGI process is added to additional groups
|
||||
to give access to the socket from the redis DB::
|
||||
|
||||
$ groups searxng
|
||||
searxng : searxng searxng-redis
|
||||
|
||||
To see the effective groups of the uwsgi process, you have to look at the status
|
||||
of the process, by example::
|
||||
|
||||
$ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini'
|
||||
searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini
|
||||
searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini
|
||||
|
||||
Here you can see that the additional "Groups" of PID 186 are unset (missing gid
|
||||
of ``searxng-redis``)::
|
||||
|
||||
$ cat /proc/186/task/186/status
|
||||
...
|
||||
Uid: 993 993 993 993
|
||||
Gid: 993 993 993 993
|
||||
FDSize: 128
|
||||
Groups:
|
||||
...
|
@ -0,0 +1,22 @@
|
||||
.. _installation:
|
||||
|
||||
============
|
||||
Installation
|
||||
============
|
||||
|
||||
*You're spoilt for choice*, choose your preferred method of installation.
|
||||
|
||||
- :ref:`installation docker`
|
||||
- :ref:`installation scripts`
|
||||
- :ref:`installation basic`
|
||||
|
||||
The :ref:`installation basic` is an excellent illustration of *how a SearXNG
|
||||
instance is build up* (see :ref:`architecture uWSGI`). If you do not have any
|
||||
special preferences, its recommend to use the :ref:`installation docker` or the
|
||||
:ref:`installation scripts`.
|
||||
|
||||
.. attention::
|
||||
|
||||
SearXNG is growing rapidly, you should regularly read our :ref:`migrate and
|
||||
stay tuned` section. If you want to upgrade an existing instance or migrate
|
||||
from searx to SearXNG, you should read this section first!
|
@ -0,0 +1,39 @@
|
||||
.. _plugins generic:
|
||||
|
||||
===============
|
||||
Plugins builtin
|
||||
===============
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`dev plugin`
|
||||
|
||||
Configuration defaults (at built time):
|
||||
|
||||
:DO: Default on
|
||||
|
||||
.. _configured plugins:
|
||||
|
||||
.. jinja:: searx
|
||||
|
||||
.. flat-table:: Plugins configured at built time (defaults)
|
||||
:header-rows: 1
|
||||
:stub-columns: 1
|
||||
:widths: 3 1 9
|
||||
|
||||
* - Name
|
||||
- DO
|
||||
- Description
|
||||
|
||||
JS & CSS dependencies
|
||||
|
||||
{% for plgin in plugins %}
|
||||
|
||||
* - {{plgin.name}}
|
||||
- {{(plgin.default_on and "y") or ""}}
|
||||
- {{plgin.description}}
|
||||
|
||||
{% for dep in (plgin.js_dependencies + plgin.css_dependencies) %}
|
||||
| ``{{dep}}`` {% endfor %}
|
||||
|
||||
{% endfor %}
|
@ -0,0 +1,136 @@
|
||||
===================
|
||||
SearXNG maintenance
|
||||
===================
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- :ref:`toolboxing`
|
||||
- :ref:`uWSGI maintenance`
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
.. _update searxng:
|
||||
|
||||
How to update
|
||||
=============
|
||||
|
||||
How to update depends on the :ref:`installation` method. If you have used the
|
||||
:ref:`installation scripts`, use the ``update`` command from the :ref:`searxng.sh`
|
||||
script.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
sudo -H ./utils/searxng.sh instance update
|
||||
|
||||
.. _inspect searxng:
|
||||
|
||||
How to inspect & debug
|
||||
======================
|
||||
|
||||
How to debug depends on the :ref:`installation` method. If you have used the
|
||||
:ref:`installation scripts`, use the ``inspect`` command from the :ref:`searxng.sh`
|
||||
script.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
sudo -H ./utils/searxng.sh instance inspect
|
||||
|
||||
.. _migrate and stay tuned:
|
||||
|
||||
Migrate and stay tuned!
|
||||
=======================
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
- :pull:`1332`
|
||||
- :pull:`456`
|
||||
- :pull:`A comment about rolling release <446#issuecomment-954730358>`
|
||||
|
||||
SearXNG is a *rolling release*; each commit to the master branch is a release.
|
||||
SearXNG is growing rapidly, the services and opportunities are change every now
|
||||
and then, to name just a few:
|
||||
|
||||
- Bot protection has been switched from filtron to SearXNG's :ref:`limiter
|
||||
<limiter>`, this requires a :ref:`Redis <settings redis>` database.
|
||||
|
||||
- The image proxy morty is no longer needed, it has been replaced by the
|
||||
:ref:`image proxy <image_proxy>` from SearXNG.
|
||||
|
||||
- To save bandwith :ref:`cache busting <static_use_hash>` has been implemented.
|
||||
To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi
|
||||
setup`.
|
||||
|
||||
To stay tuned and get in use of the new features, instance maintainers have to
|
||||
update the SearXNG code regularly (see :ref:`update searxng`). As the above
|
||||
examples show, this is not always enough, sometimes services have to be set up
|
||||
or reconfigured and sometimes services that are no longer needed should be
|
||||
uninstalled.
|
||||
|
||||
.. hint::
|
||||
|
||||
First of all: SearXNG is installed by the script :ref:`searxng.sh`. If you
|
||||
have old filtron, morty or searx setup you should consider complete
|
||||
uninstall/reinstall.
|
||||
|
||||
Here you will find a list of changes that affect the infrastructure. Please
|
||||
check to what extent it is necessary to update your installations:
|
||||
|
||||
:pull:`1595`: ``[fix] uWSGI: increase buffer-size``
|
||||
Re-install uWSGI (:ref:`searxng.sh`) or fix your uWSGI ``searxng.ini``
|
||||
file manually.
|
||||
|
||||
|
||||
remove obsolete services
|
||||
------------------------
|
||||
|
||||
If your searx instance was installed *"Step by step"* or by the *"Installation
|
||||
scripts"*, you need to undo the installation procedure completely. If you have
|
||||
morty & filtron installed, it is recommended to uninstall these services also.
|
||||
In case of scripts, to uninstall use the scripts from the origin you installed
|
||||
searx from or try::
|
||||
|
||||
$ sudo -H ./utils/filtron.sh remove all
|
||||
$ sudo -H ./utils/morty.sh remove all
|
||||
$ sudo -H ./utils/searx.sh remove all
|
||||
|
||||
.. hint::
|
||||
|
||||
If you are migrate from searx take into account that the ``.config.sh`` is no
|
||||
longer used.
|
||||
|
||||
If you upgrade from searx or from before :pull:`1332` has been merged and you
|
||||
have filtron and/or morty installed, don't forget to remove HTTP sites.
|
||||
|
||||
Apache::
|
||||
|
||||
$ sudo -H ./utils/filtron.sh apache remove
|
||||
$ sudo -H ./utils/morty.sh apache remove
|
||||
|
||||
nginx::
|
||||
|
||||
$ sudo -H ./utils/filtron.sh nginx remove
|
||||
$ sudo -H ./utils/morty.sh nginx remove
|
||||
|
||||
|
||||
|
||||
Check after Installation
|
||||
------------------------
|
||||
|
||||
Once you have done your installation, you can run a SearXNG *check* procedure,
|
||||
to see if there are some left overs. In this example there exists a *old*
|
||||
``/etc/searx/settings.yml``::
|
||||
|
||||
$ sudo -H ./utils/searxng.sh instance check
|
||||
|
||||
SearXNG checks
|
||||
--------------
|
||||
ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/
|
||||
INFO: [OK] (old) account 'searx' does not exists
|
||||
INFO: [OK] (old) account 'filtron' does not exists
|
||||
INFO: [OK] (old) account 'morty' does not exists
|
||||
...
|
||||
INFO searx.redisdb : connecting to Redis db=0 path='/usr/local/searxng-redis/run/redis.sock'
|
||||
INFO searx.redisdb : connected to Redis
|
@ -0,0 +1,185 @@
|
||||
.. _how to contribute:
|
||||
|
||||
=================
|
||||
How to contribute
|
||||
=================
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
Prime directives: Privacy, Hackability
|
||||
======================================
|
||||
|
||||
SearXNG has two prime directives, **privacy-by-design and hackability** . The
|
||||
hackability comes in three levels:
|
||||
|
||||
- support of search engines
|
||||
- plugins to alter search behaviour
|
||||
- hacking SearXNG itself
|
||||
|
||||
Note the lack of "world domination" among the directives. SearXNG has no
|
||||
intention of wide mass-adoption, rounded corners, etc. The prime directive
|
||||
"privacy" deserves a separate chapter, as it's quite uncommon unfortunately.
|
||||
|
||||
Privacy-by-design
|
||||
-----------------
|
||||
|
||||
SearXNG was born out of the need for a **privacy-respecting** search tool which
|
||||
can be extended easily to maximize both, its search and its privacy protecting
|
||||
capabilities.
|
||||
|
||||
A few widely used features work differently or turned off by default or not
|
||||
implemented at all **as a consequence of privacy-by-design**.
|
||||
|
||||
If a feature reduces the privacy preserving aspects of searx, it should be
|
||||
switched off by default or should not implemented at all. There are plenty of
|
||||
search engines already providing such features. If a feature reduces the
|
||||
protection of searx, users must be informed about the effect of choosing to
|
||||
enable it. Features that protect privacy but differ from the expectations of
|
||||
the user should also be explained.
|
||||
|
||||
Also, if you think that something works weird with searx, it's might be because
|
||||
of the tool you use is designed in a way to interfere with the privacy respect.
|
||||
Submitting a bugreport to the vendor of the tool that misbehaves might be a good
|
||||
feedback to reconsider the disrespect to its customers (e.g. ``GET`` vs ``POST``
|
||||
requests in various browsers).
|
||||
|
||||
Remember the other prime directive of SearXNG is to be hackable, so if the above
|
||||
privacy concerns do not fancy you, simply fork it.
|
||||
|
||||
*Happy hacking.*
|
||||
|
||||
Code
|
||||
====
|
||||
|
||||
.. _PEP8: https://www.python.org/dev/peps/pep-0008/
|
||||
.. _Conventional Commits: https://www.conventionalcommits.org/
|
||||
.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages
|
||||
.. _Structural split of changes:
|
||||
https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes
|
||||
.. _gitmoji: https://gitmoji.carloscuesta.me/
|
||||
.. _Semantic PR: https://github.com/zeke/semantic-pull-requests
|
||||
|
||||
.. sidebar:: Create good commits!
|
||||
|
||||
- `Structural split of changes`_
|
||||
- `Conventional Commits`_
|
||||
- `Git Commit Good Practice`_
|
||||
- some like to use: gitmoji_
|
||||
- not yet active: `Semantic PR`_
|
||||
|
||||
In order to submit a patch, please follow the steps below:
|
||||
|
||||
- Follow coding conventions.
|
||||
|
||||
- PEP8_ standards apply, except the convention of line length
|
||||
- Maximum line length is 120 characters
|
||||
|
||||
- The cardinal rule for creating good commits is to ensure there is only one
|
||||
*logical change* per commit / read `Structural split of changes`_
|
||||
|
||||
- Check if your code breaks existing tests. If so, update the tests or fix your
|
||||
code.
|
||||
|
||||
- If your code can be unit-tested, add unit tests.
|
||||
|
||||
- Add yourself to the :origin:`AUTHORS.rst` file.
|
||||
|
||||
- Choose meaningful commit messages, read `Conventional Commits`_
|
||||
|
||||
.. code::
|
||||
|
||||
<type>[optional scope]: <description>
|
||||
|
||||
[optional body]
|
||||
|
||||
[optional footer(s)]
|
||||
|
||||
- Create a pull request.
|
||||
|
||||
For more help on getting started with SearXNG development, see :ref:`devquickstart`.
|
||||
|
||||
|
||||
Translation
|
||||
===========
|
||||
|
||||
Translation currently takes place on :ref:`weblate <translation>`.
|
||||
|
||||
|
||||
.. _contrib docs:
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
||||
.. _Sphinx: https://www.sphinx-doc.org
|
||||
.. _reST: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
|
||||
|
||||
.. sidebar:: The reST sources
|
||||
|
||||
has been moved from ``gh-branch`` into ``master`` (:origin:`docs`).
|
||||
|
||||
The documentation is built using Sphinx_. So in order to be able to generate
|
||||
the required files, you have to install it on your system. Much easier, use
|
||||
our :ref:`makefile`.
|
||||
|
||||
Here is an example which makes a complete rebuild:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ make docs.clean docs.html
|
||||
...
|
||||
The HTML pages are in dist/docs.
|
||||
|
||||
.. _make docs.live:
|
||||
|
||||
live build
|
||||
----------
|
||||
|
||||
.. _sphinx-autobuild:
|
||||
https://github.com/executablebooks/sphinx-autobuild/blob/master/README.md
|
||||
|
||||
.. sidebar:: docs.clean
|
||||
|
||||
It is recommended to assert a complete rebuild before deploying (use
|
||||
``docs.clean``).
|
||||
|
||||
Live build is like WYSIWYG. If you want to edit the documentation, its
|
||||
recommended to use. The Makefile target ``docs.live`` builds the docs, opens
|
||||
URL in your favorite browser and rebuilds every time a reST file has been
|
||||
changed.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ make docs.live
|
||||
...
|
||||
The HTML pages are in dist/docs.
|
||||
... Serving on http://0.0.0.0:8000
|
||||
... Start watching changes
|
||||
|
||||
Live builds are implemented by sphinx-autobuild_. Use environment
|
||||
``$(SPHINXOPTS)`` to pass arguments to the sphinx-autobuild_ command. Except
|
||||
option ``--host`` (which is always set to ``0.0.0.0``) you can pass any
|
||||
argument. E.g to find and use a free port, use:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ SPHINXOPTS="--port 0" make docs.live
|
||||
...
|
||||
... Serving on http://0.0.0.0:50593
|
||||
...
|
||||
|
||||
|
||||
.. _deploy on github.io:
|
||||
|
||||
deploy on github.io
|
||||
-------------------
|
||||
|
||||
To deploy documentation at :docs:`github.io <.>` use Makefile target :ref:`make
|
||||
docs.gh-pages`, which builds the documentation and runs all the needed git add,
|
||||
commit and push:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$ make docs.clean docs.gh-pages
|
@ -0,0 +1,402 @@
|
||||
.. _engines-dev:
|
||||
|
||||
===============
|
||||
Engine Overview
|
||||
===============
|
||||
|
||||
.. _metasearch-engine: https://en.wikipedia.org/wiki/Metasearch_engine
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`configured engines`
|
||||
- :ref:`settings engine`
|
||||
|
||||
.. contents::
|
||||
:depth: 3
|
||||
:backlinks: entry
|
||||
|
||||
SearXNG is a metasearch-engine_, so it uses different search engines to provide
|
||||
better results.
|
||||
|
||||
Because there is no general search API which could be used for every search
|
||||
engine, an adapter has to be built between SearXNG and the external search
|
||||
engines. Adapters are stored under the folder :origin:`searx/engines`.
|
||||
|
||||
.. _general engine configuration:
|
||||
|
||||
General Engine Configuration
|
||||
============================
|
||||
|
||||
It is required to tell SearXNG the type of results the engine provides. The
|
||||
arguments can be set in the engine file or in the settings file (normally
|
||||
``settings.yml``). The arguments in the settings file override the ones in the
|
||||
engine file.
|
||||
|
||||
It does not matter if an option is stored in the engine file or in the settings.
|
||||
However, the standard way is the following:
|
||||
|
||||
.. _engine file:
|
||||
|
||||
Engine File
|
||||
-----------
|
||||
|
||||
.. table:: Common options in the engine module
|
||||
:width: 100%
|
||||
|
||||
======================= =========== ========================================================
|
||||
argument type information
|
||||
======================= =========== ========================================================
|
||||
categories list pages, in which the engine is working
|
||||
paging boolean support multiple pages
|
||||
time_range_support boolean support search time range
|
||||
engine_type str - ``online`` :ref:`[ref] <demo online engine>` by
|
||||
default, other possibles values are:
|
||||
- ``offline`` :ref:`[ref] <offline engines>`
|
||||
- ``online_dictionary``
|
||||
- ``online_currency``
|
||||
======================= =========== ========================================================
|
||||
|
||||
.. _engine settings:
|
||||
|
||||
Engine ``settings.yml``
|
||||
-----------------------
|
||||
|
||||
For a more detailed description, see :ref:`settings engine` in the :ref:`settings.yml`.
|
||||
|
||||
.. table:: Common options in the engine setup (``settings.yml``)
|
||||
:width: 100%
|
||||
|
||||
======================= =========== ==================================================
|
||||
argument type information
|
||||
======================= =========== ==================================================
|
||||
name string name of search-engine
|
||||
engine string name of searxng-engine (file name without ``.py``)
|
||||
enable_http bool enable HTTP (by default only HTTPS is enabled).
|
||||
shortcut string shortcut of search-engine
|
||||
timeout string specific timeout for search-engine
|
||||
display_error_messages boolean display error messages on the web UI
|
||||
proxies dict set proxies for a specific engine
|
||||
(e.g. ``proxies : {http: socks5://proxy:port,
|
||||
https: socks5://proxy:port}``)
|
||||
======================= =========== ==================================================
|
||||
|
||||
.. _engine overrides:
|
||||
|
||||
Overrides
|
||||
---------
|
||||
|
||||
A few of the options have default values in the namespace of engine's python
|
||||
modul, but are often overwritten by the settings. If ``None`` is assigned to an
|
||||
option in the engine file, it has to be redefined in the settings, otherwise
|
||||
SearXNG will not start with that engine (global names with a leading underline can
|
||||
be ``None``).
|
||||
|
||||
Here is an very simple example of the global names in the namespace of engine's
|
||||
module:
|
||||
|
||||
.. code:: python
|
||||
|
||||
# engine dependent config
|
||||
categories = ['general']
|
||||
paging = True
|
||||
_non_overwritten_global = 'foo'
|
||||
|
||||
|
||||
.. table:: The naming of overrides is arbitrary / recommended overrides are:
|
||||
:width: 100%
|
||||
|
||||
======================= =========== ===========================================
|
||||
argument type information
|
||||
======================= =========== ===========================================
|
||||
base_url string base-url, can be overwritten to use same
|
||||
engine on other URL
|
||||
number_of_results int maximum number of results per request
|
||||
language string ISO code of language and country like en_US
|
||||
api_key string api-key if required by engine
|
||||
======================= =========== ===========================================
|
||||
|
||||
.. _engine request:
|
||||
|
||||
Making a Request
|
||||
================
|
||||
|
||||
To perform a search an URL have to be specified. In addition to specifying an
|
||||
URL, arguments can be passed to the query.
|
||||
|
||||
.. _engine request arguments:
|
||||
|
||||
Passed Arguments (request)
|
||||
--------------------------
|
||||
|
||||
These arguments can be used to construct the search query. Furthermore,
|
||||
parameters with default value can be redefined for special purposes.
|
||||
|
||||
|
||||
.. table:: If the ``engine_type`` is ``online``
|
||||
:width: 100%
|
||||
|
||||
====================== ============== ========================================================================
|
||||
argument type default-value, information
|
||||
====================== ============== ========================================================================
|
||||
url str ``''``
|
||||
method str ``'GET'``
|
||||
headers set ``{}``
|
||||
data set ``{}``
|
||||
cookies set ``{}``
|
||||
verify bool ``True``
|
||||
headers.User-Agent str a random User-Agent
|
||||
category str current category, like ``'general'``
|
||||
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
|
||||
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
|
||||
pageno int current pagenumber
|
||||
language str specific language code like ``'en_US'``, or ``'all'`` if unspecified
|
||||
====================== ============== ========================================================================
|
||||
|
||||
|
||||
.. table:: If the ``engine_type`` is ``online_dictionary``, in addition to the
|
||||
``online`` arguments:
|
||||
:width: 100%
|
||||
|
||||
====================== ============== ========================================================================
|
||||
argument type default-value, information
|
||||
====================== ============== ========================================================================
|
||||
from_lang str specific language code like ``'en_US'``
|
||||
to_lang str specific language code like ``'en_US'``
|
||||
query str the text query without the languages
|
||||
====================== ============== ========================================================================
|
||||
|
||||
.. table:: If the ``engine_type`` is ``online_currency```, in addition to the
|
||||
``online`` arguments:
|
||||
:width: 100%
|
||||
|
||||
====================== ============== ========================================================================
|
||||
argument type default-value, information
|
||||
====================== ============== ========================================================================
|
||||
amount float the amount to convert
|
||||
from str ISO 4217 code
|
||||
to str ISO 4217 code
|
||||
from_name str currency name
|
||||
to_name str currency name
|
||||
====================== ============== ========================================================================
|
||||
|
||||
|
||||
Specify Request
|
||||
---------------
|
||||
|
||||
The function :py:func:`def request(query, params):
|
||||
<searx.engines.demo_online.request>` always returns the ``params`` variable, the
|
||||
following parameters can be used to specify a search request:
|
||||
|
||||
.. table::
|
||||
:width: 100%
|
||||
|
||||
=================== =========== ==========================================================================
|
||||
argument type information
|
||||
=================== =========== ==========================================================================
|
||||
url str requested url
|
||||
method str HTTP request method
|
||||
headers set HTTP header information
|
||||
data set HTTP data information
|
||||
cookies set HTTP cookies
|
||||
verify bool Performing SSL-Validity check
|
||||
allow_redirects bool Follow redirects
|
||||
max_redirects int maximum redirects, hard limit
|
||||
soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
|
||||
raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
|
||||
=================== =========== ==========================================================================
|
||||
|
||||
|
||||
.. _engine results:
|
||||
.. _engine media types:
|
||||
|
||||
Media Types
|
||||
===========
|
||||
|
||||
Each result item of an engine can be of different media-types. Currently the
|
||||
following media-types are supported. To set another media-type as ``default``,
|
||||
the parameter ``template`` must be set to the desired type.
|
||||
|
||||
.. table:: Parameter of the **default** media type:
|
||||
:width: 100%
|
||||
|
||||
========================= =====================================================
|
||||
result-parameter information
|
||||
========================= =====================================================
|
||||
url string, url of the result
|
||||
title string, title of the result
|
||||
content string, general result-text
|
||||
publishedDate :py:class:`datetime.datetime`, time of publish
|
||||
========================= =====================================================
|
||||
|
||||
|
||||
.. table:: Parameter of the **images** media type:
|
||||
:width: 100%
|
||||
|
||||
========================= =====================================================
|
||||
result-parameter information
|
||||
------------------------- -----------------------------------------------------
|
||||
template is set to ``images.html``
|
||||
========================= =====================================================
|
||||
url string, url to the result site
|
||||
title string, title of the result *(partly implemented)*
|
||||
content *(partly implemented)*
|
||||
publishedDate :py:class:`datetime.datetime`,
|
||||
time of publish *(partly implemented)*
|
||||
img\_src string, url to the result image
|
||||
thumbnail\_src string, url to a small-preview image
|
||||
========================= =====================================================
|
||||
|
||||
|
||||
.. table:: Parameter of the **videos** media type:
|
||||
:width: 100%
|
||||
|
||||
========================= =====================================================
|
||||
result-parameter information
|
||||
------------------------- -----------------------------------------------------
|
||||
template is set to ``videos.html``
|
||||
========================= =====================================================
|
||||
url string, url of the result
|
||||
title string, title of the result
|
||||
content *(not implemented yet)*
|
||||
publishedDate :py:class:`datetime.datetime`, time of publish
|
||||
thumbnail string, url to a small-preview image
|
||||
========================= =====================================================
|
||||
|
||||
.. _magnetlink: https://en.wikipedia.org/wiki/Magnet_URI_scheme
|
||||
|
||||
.. table:: Parameter of the **torrent** media type:
|
||||
:width: 100%
|
||||
|
||||
========================= =====================================================
|
||||
result-parameter information
|
||||
------------------------- -----------------------------------------------------
|
||||
template is set to ``torrent.html``
|
||||
========================= =====================================================
|
||||
url string, url of the result
|
||||
title string, title of the result
|
||||
content string, general result-text
|
||||
publishedDate :py:class:`datetime.datetime`,
|
||||
time of publish *(not implemented yet)*
|
||||
seed int, number of seeder
|
||||
leech int, number of leecher
|
||||
filesize int, size of file in bytes
|
||||
files int, number of files
|
||||
magnetlink string, magnetlink_ of the result
|
||||
torrentfile string, torrentfile of the result
|
||||
========================= =====================================================
|
||||
|
||||
.. table:: Parameter of the **map** media type:
|
||||
:width: 100%
|
||||
|
||||
========================= =====================================================
|
||||
result-parameter information
|
||||
------------------------- -----------------------------------------------------
|
||||
template is set to ``map.html``
|
||||
========================= =====================================================
|
||||
url string, url of the result
|
||||
title string, title of the result
|
||||
content string, general result-text
|
||||
publishedDate :py:class:`datetime.datetime`, time of publish
|
||||
latitude latitude of result (in decimal format)
|
||||
longitude longitude of result (in decimal format)
|
||||
boundingbox boundingbox of result (array of 4. values
|
||||
``[lat-min, lat-max, lon-min, lon-max]``)
|
||||
geojson geojson of result (https://geojson.org/)
|
||||
osm.type type of osm-object (if OSM-Result)
|
||||
osm.id id of osm-object (if OSM-Result)
|
||||
address.name name of object
|
||||
address.road street name of object
|
||||
address.house_number house number of object
|
||||
address.locality city, place of object
|
||||
address.postcode postcode of object
|
||||
address.country country of object
|
||||
========================= =====================================================
|
||||
|
||||
.. _BibTeX format: https://www.bibtex.com/g/bibtex-format/
|
||||
.. _BibTeX field types: https://en.wikipedia.org/wiki/BibTeX#Field_types
|
||||
|
||||
.. list-table:: Parameter of the **paper** media type /
|
||||
see `BibTeX field types`_ and `BibTeX format`_
|
||||
:header-rows: 2
|
||||
:width: 100%
|
||||
|
||||
* - result-parameter
|
||||
- Python type
|
||||
- information
|
||||
|
||||
* - template
|
||||
- :py:class:`str`
|
||||
- is set to ``paper.html``
|
||||
|
||||
* - title
|
||||
- :py:class:`str`
|
||||
- title of the result
|
||||
|
||||
* - content
|
||||
- :py:class:`str`
|
||||
- abstract
|
||||
|
||||
* - comments
|
||||
- :py:class:`str`
|
||||
- free text display in italic below the content
|
||||
|
||||
* - tags
|
||||
- :py:class:`List <list>`\ [\ :py:class:`str`\ ]
|
||||
- free tag list
|
||||
|
||||
* - publishedDate
|
||||
- :py:class:`datetime <datetime.datetime>`
|
||||
- last publication date
|
||||
|
||||
* - type
|
||||
- :py:class:`str`
|
||||
- short description of medium type, e.g. *book*, *pdf* or *html* ...
|
||||
|
||||
* - authors
|
||||
- :py:class:`List <list>`\ [\ :py:class:`str`\ ]
|
||||
- list of authors of the work (authors with a "s")
|
||||
|
||||
* - editor
|
||||
- :py:class:`str`
|
||||
- list of editors of a book
|
||||
|
||||
* - publisher
|
||||
- :py:class:`str`
|
||||
- name of the publisher
|
||||
|
||||
* - journal
|
||||
- :py:class:`str`
|
||||
- name of the journal or magazine the article was
|
||||
published in
|
||||
|
||||
* - volume
|
||||
- :py:class:`str`
|
||||
- volume number
|
||||
|
||||
* - pages
|
||||
- :py:class:`str`
|
||||
- page range where the article is
|
||||
|
||||
* - number
|
||||
- :py:class:`str`
|
||||
- number of the report or the issue number for a journal article
|
||||
|
||||
* - doi
|
||||
- :py:class:`str`
|
||||
- DOI number (like ``10.1038/d41586-018-07848-2``)
|
||||
|
||||
* - issn
|
||||
- :py:class:`List <list>`\ [\ :py:class:`str`\ ]
|
||||
- ISSN number like ``1476-4687``
|
||||
|
||||
* - isbn
|
||||
- :py:class:`List <list>`\ [\ :py:class:`str`\ ]
|
||||
- ISBN number like ``9780201896831``
|
||||
|
||||
* - pdf_url
|
||||
- :py:class:`str`
|
||||
- URL to the full article, the PDF version
|
||||
|
||||
* - html_url
|
||||
- :py:class:`str`
|
||||
- URL to full article, HTML version
|
@ -0,0 +1,19 @@
|
||||
=======================
|
||||
Developer documentation
|
||||
=======================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
quickstart
|
||||
contribution_guide
|
||||
engine_overview
|
||||
offline_engines
|
||||
search_api
|
||||
plugins
|
||||
translation
|
||||
lxcdev
|
||||
makefile
|
||||
reST
|
||||
searxng_extra/index
|
@ -0,0 +1,403 @@
|
||||
.. _lxcdev:
|
||||
|
||||
==============================
|
||||
Developing in Linux Containers
|
||||
==============================
|
||||
|
||||
.. _LXC: https://linuxcontainers.org/lxc/introduction/
|
||||
|
||||
In this article we will show, how you can make use of Linux Containers (LXC_) in
|
||||
*distributed and heterogeneous development cycles* (TL;DR; jump to the
|
||||
:ref:`lxcdev summary`).
|
||||
|
||||
.. sidebar:: Audience
|
||||
|
||||
This blog post is written for experienced admins and developers. Readers
|
||||
should have a serious meaning about the terms: *distributed*, *merge* and
|
||||
*linux container*.
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
Motivation
|
||||
==========
|
||||
|
||||
Usually in our development cycle, we edit the sources and run some test and/or
|
||||
builds by using ``make`` :ref:`[ref] <makefile>` before we commit. This cycle
|
||||
is simple and perfect but might fail in some aspects we should not overlook.
|
||||
|
||||
**The environment in which we run all our development processes matters!**
|
||||
|
||||
The :ref:`makefile` and the :ref:`make install` encapsulate a lot for us, but
|
||||
they do not have access to all prerequisites. For example, there may have
|
||||
dependencies on packages that are installed on the developer's desktop, but
|
||||
usually are not preinstalled on a server or client system. Another example is;
|
||||
settings have been made to the software on developer's desktop that would never
|
||||
be set on a *production* system.
|
||||
|
||||
**Linux Containers are isolate environments and not to mix up all the
|
||||
prerequisites from various projects on developer's desktop is always a good
|
||||
choice.**
|
||||
|
||||
The scripts from :ref:`searx_utils` can divide in those to install and maintain
|
||||
software:
|
||||
|
||||
- :ref:`searxng.sh`
|
||||
|
||||
and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or
|
||||
even development tasks over a stack of isolated containers / what we call the:
|
||||
|
||||
**SearXNG LXC suite**
|
||||
|
||||
.. hint::
|
||||
|
||||
If you see any problems with the internet connectivity of your
|
||||
containers read section :ref:`internet connectivity docker`.
|
||||
|
||||
|
||||
Gentlemen, start your engines!
|
||||
==============================
|
||||
|
||||
.. _LXD: https://linuxcontainers.org/lxd/introduction/
|
||||
.. _archlinux: https://www.archlinux.org/
|
||||
|
||||
Before you can start with containers, you need to install and initiate LXD_
|
||||
once:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ snap install lxd
|
||||
$ lxd init --auto
|
||||
|
||||
And you need to clone from origin or if you have your own fork, clone from your
|
||||
fork:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ cd ~/Downloads
|
||||
$ git clone https://github.com/searxng/searxng.git searxng
|
||||
$ cd searxng
|
||||
|
||||
The :ref:`lxc-searxng.env` consists of several images, see ``export
|
||||
LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post
|
||||
we exercise on a archlinux_ image. The container of this image is named
|
||||
``searxng-archlinux``. Lets build the container, but be sure that this container
|
||||
does not already exists, so first lets remove possible old one:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh remove searxng-archlinux
|
||||
$ sudo -H ./utils/lxc.sh build searxng-archlinux
|
||||
|
||||
.. sidebar:: The ``searxng-archlinux`` container
|
||||
|
||||
is the base of all our exercises here.
|
||||
|
||||
In this container we install all services :ref:`including searx, morty & filtron
|
||||
<lxc.sh install suite>` in once:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh install suite searxng-archlinux
|
||||
|
||||
To proxy HTTP from filtron and morty in the container to the outside of the
|
||||
container, install nginx into the container. Once for the bot blocker filtron:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
./utils/filtron.sh nginx install
|
||||
...
|
||||
INFO: got 429 from http://10.174.184.156/searx
|
||||
|
||||
and once for the content sanitizer (content proxy morty):
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
./utils/morty.sh nginx install
|
||||
...
|
||||
INFO: got 200 from http://10.174.184.156/morty/
|
||||
|
||||
.. sidebar:: Fully functional SearXNG suite
|
||||
|
||||
From here on you have a fully functional SearXNG suite running with bot
|
||||
blocker (filtron) and WEB content sanitizer (content proxy morty), both are
|
||||
needed for a *privacy protecting* search engine.
|
||||
|
||||
On your system, the IP of your ``searxng-archlinux`` container differs from
|
||||
http://10.174.184.156/searx, just open the URL reported in your installation
|
||||
protocol in your WEB browser from the desktop to test the instance from outside
|
||||
of the container.
|
||||
|
||||
In such a earXNG suite admins can maintain and access the debug log of the
|
||||
different services quite easy.
|
||||
|
||||
.. _working in containers:
|
||||
|
||||
In containers, work as usual
|
||||
============================
|
||||
|
||||
Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers
|
||||
open the root-bash in the container using ``./utils/lxc.sh cmd
|
||||
searxng-archlinux``:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
|
||||
INFO: [searxng-archlinux] bash
|
||||
[root@searxng-archlinux searx]# pwd
|
||||
/share/searxng
|
||||
|
||||
The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user in
|
||||
the searxng-container. To debug the running SearXNG instance use:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: root@searxng-archlinux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ ./utils/searx.sh inspect service
|
||||
...
|
||||
use [CTRL-C] to stop monitoring the log
|
||||
...
|
||||
|
||||
Back in the browser on your desktop open the service http://10.174.184.156/searx
|
||||
and run your application tests while the debug log is shown in the terminal from
|
||||
above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug
|
||||
option"* in SearXNG's settings file and restarts the SearXNG uwsgi application.
|
||||
To debug services from filtron and morty analogous use:
|
||||
|
||||
Another point we have to notice is that the service (:ref:`SearXNG <searxng.sh>`
|
||||
runs under dedicated system user account with the same name (compare
|
||||
:ref:`create searxng user`). To get a shell from these accounts, simply call:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: root@searxng-archlinux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ ./utils/searxng.sh instance cmd bash
|
||||
|
||||
To get in touch, open a shell from the service user (searxng@searxng-archlinux):
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash
|
||||
INFO: [searxng-archlinux] ./utils/searxng.sh instance cmd bash
|
||||
[searxng@searxng-archlinux ~]$
|
||||
|
||||
The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system
|
||||
user ``searx`` in the ``searxng-archlinux`` container and the python *virtualenv*
|
||||
``(searxng-pyenv)`` environment is activated.
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: searxng@searxng-archlinux
|
||||
|
||||
.. code:: bash
|
||||
|
||||
(searxng-pyenv) [searxng@searxng-archlinux ~]$ pwd
|
||||
/usr/local/searxng
|
||||
|
||||
|
||||
Wrap production into developer suite
|
||||
====================================
|
||||
|
||||
In this section we will see how to change the *"Fully functional SearXNG suite"*
|
||||
from a LXC container (which is quite ready for production) into a developer
|
||||
suite. For this, we have to keep an eye on the :ref:`installation basic`:
|
||||
|
||||
- SearXNG setup in: ``/etc/searxng/settings.yml``
|
||||
- SearXNG user's home: ``/usr/local/searxng``
|
||||
- virtualenv in: ``/usr/local/searxng/searxng-pyenv``
|
||||
- SearXNG software in: ``/usr/local/searxng/searxng-src``
|
||||
|
||||
With the use of the :ref:`searxng.sh` the SearXNG service was installed as
|
||||
:ref:`uWSGI application <searxng uwsgi>`. To maintain this service, we can use
|
||||
``systemctl`` (compare :ref:`uWSGI maintenance`).
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
systemctl stop uwsgi@searxng
|
||||
|
||||
With the command above, we stopped the SearXNG uWSGI-App in the archlinux
|
||||
container.
|
||||
|
||||
The uWSGI-App for the archlinux dsitros is configured in
|
||||
:origin:`utils/templates/etc/uwsgi/apps-archlinux/searxng.ini`, from where at
|
||||
least you should attend the settings of ``uid``, ``chdir``, ``env`` and
|
||||
``http``::
|
||||
|
||||
env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml
|
||||
http = 127.0.0.1:8888
|
||||
|
||||
chdir = /usr/local/searxng/searxng-src/searx
|
||||
virtualenv = /usr/local/searxng/searxng-pyenv
|
||||
pythonpath = /usr/local/searxng/searxng-src
|
||||
|
||||
If you have read the :ref:`"Good to know section" <lxc.sh>` you remember, that
|
||||
each container shares the root folder of the repository and the command
|
||||
``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the
|
||||
SearXNG installation into a developer one, we simple have to create a smylink to
|
||||
the **transparent** reposetory from the desktop. Now lets replace the
|
||||
repository at ``searxng-src`` in the container with the working tree from outside
|
||||
of the container:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: container becomes a developer suite
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
ln -s /share/searx/ /usr/local/searxng/searxng-src
|
||||
|
||||
Now we can develop as usual in the working tree of our desktop system. Every
|
||||
time the software was changed, you have to restart the SearXNG service (in the
|
||||
container):
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
systemctl restart uwsgi@searx
|
||||
|
||||
|
||||
Remember: :ref:`working in containers` .. here are just some examples from my
|
||||
daily usage:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
To *inspect* the SearXNG instance (already described above):
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
./utils/searx.sh inspect service
|
||||
|
||||
Run :ref:`makefile`, e.g. to test inside the container:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
make test
|
||||
|
||||
To install all prerequisites needed for a :ref:`buildhosts`:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
./utils/searxng.sh install buildhost
|
||||
|
||||
To build the docs on a buildhost :ref:`buildhosts`:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
|
||||
make docs.html
|
||||
|
||||
.. _lxcdev summary:
|
||||
|
||||
Summary
|
||||
=======
|
||||
|
||||
We build up a fully functional SearXNG suite in a archlinux container:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh install suite searxng-archlinux
|
||||
|
||||
To access HTTP from the desktop we installed nginx for the services inside the
|
||||
container:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: [root@searxng-archlinux]
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ ./utils/filtron.sh nginx install
|
||||
$ ./utils/morty.sh nginx install
|
||||
|
||||
To wrap the suite into a developer one, we created a symbolic link to the
|
||||
repository which is shared **transparent** from the desktop's file system into
|
||||
the container :
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: [root@searxng-archlinux]
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old
|
||||
$ ln -s /share/searx/ /usr/local/searxng/searxng-src
|
||||
$ systemctl restart uwsgi@searx
|
||||
|
||||
To get information about the searxNG suite in the archlinux container we can
|
||||
use:
|
||||
|
||||
.. tabs::
|
||||
|
||||
.. group-tab:: desktop
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo -H ./utils/lxc.sh show suite searxng-archlinux
|
||||
...
|
||||
[searxng-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx
|
||||
[searxng-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/
|
||||
[searxng-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/
|
||||
[searxng-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]
|
||||
...
|
||||
|
@ -0,0 +1,290 @@
|
||||
.. _makefile:
|
||||
|
||||
========
|
||||
Makefile
|
||||
========
|
||||
|
||||
.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
|
||||
|
||||
.. sidebar:: build environment
|
||||
|
||||
Before looking deeper at the targets, first read about :ref:`make
|
||||
install`.
|
||||
|
||||
To install system requirements follow :ref:`buildhosts`.
|
||||
|
||||
All relevant build tasks are implemented in :origin:`manage` and for CI or
|
||||
IDE integration a small ``Makefile`` wrapper is available. If you are not
|
||||
familiar with Makefiles, we recommend to read gnu-make_ introduction.
|
||||
|
||||
The usage is simple, just type ``make {target-name}`` to *build* a target.
|
||||
Calling the ``help`` target gives a first overview (``make help``):
|
||||
|
||||
.. program-output:: bash -c "cd ..; make --no-print-directory help"
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
.. _make install:
|
||||
|
||||
Python environment (``make install``)
|
||||
=====================================
|
||||
|
||||
.. sidebar:: activate environment
|
||||
|
||||
``source ./local/py3/bin/activate``
|
||||
|
||||
We do no longer need to build up the virtualenv manually. Jump into your git
|
||||
working tree and release a ``make install`` to get a virtualenv with a
|
||||
*developer install* of SearXNG (:origin:`setup.py`). ::
|
||||
|
||||
$ cd ~/searxng-clone
|
||||
$ make install
|
||||
PYENV [virtualenv] installing ./requirements*.txt into local/py3
|
||||
...
|
||||
PYENV OK
|
||||
PYENV [install] pip install -e 'searx[test]'
|
||||
...
|
||||
Successfully installed argparse-1.4.0 searx
|
||||
BUILDENV INFO:searx:load the default settings from ./searx/settings.yml
|
||||
BUILDENV INFO:searx:Initialisation done
|
||||
BUILDENV build utils/brand.env
|
||||
|
||||
If you release ``make install`` multiple times the installation will only
|
||||
rebuild if the sha256 sum of the *requirement files* fails. With other words:
|
||||
the check fails if you edit the requirements listed in
|
||||
:origin:`requirements-dev.txt` and :origin:`requirements.txt`). ::
|
||||
|
||||
$ make install
|
||||
PYENV OK
|
||||
PYENV [virtualenv] requirements.sha256 failed
|
||||
[virtualenv] - 6cea6eb6def9e14a18bf32f8a3e... ./requirements-dev.txt
|
||||
[virtualenv] - 471efef6c73558e391c3adb35f4... ./requirements.txt
|
||||
...
|
||||
PYENV [virtualenv] installing ./requirements*.txt into local/py3
|
||||
...
|
||||
PYENV OK
|
||||
PYENV [install] pip install -e 'searx[test]'
|
||||
...
|
||||
Successfully installed argparse-1.4.0 searx
|
||||
BUILDENV INFO:searx:load the default settings from ./searx/settings.yml
|
||||
BUILDENV INFO:searx:Initialisation done
|
||||
BUILDENV build utils/brand.env
|
||||
|
||||
.. sidebar:: drop environment
|
||||
|
||||
To get rid of the existing environment before re-build use :ref:`clean target
|
||||
<make clean>` first.
|
||||
|
||||
If you think, something goes wrong with your ./local environment or you change
|
||||
the :origin:`setup.py` file, you have to call :ref:`make clean`.
|
||||
|
||||
.. _make buildenv:
|
||||
|
||||
``make buildenv``
|
||||
=================
|
||||
|
||||
Rebuild instance's environment with the modified settings from the
|
||||
:ref:`settings brand` and :ref:`settings server` section of your
|
||||
:ref:`settings.yml <settings location>`.
|
||||
|
||||
We have all SearXNG setups are centralized in the :ref:`settings.yml` file.
|
||||
This setup is available as long we are in a *installed instance*. E.g. the
|
||||
*installed instance* on the server or the *installed developer instance* at
|
||||
``./local`` (the later one is created by a :ref:`make install <make
|
||||
install>` or :ref:`make run <make run>`).
|
||||
|
||||
Tasks running outside of an *installed instance*, especially those tasks and
|
||||
scripts running at (pre-) installation time do not have access to the SearXNG
|
||||
setup (from a *installed instance*). Those tasks need a *build environment*.
|
||||
|
||||
The ``make buildenv`` target will update the *build environment* in:
|
||||
|
||||
- :origin:`utils/brand.env`
|
||||
|
||||
Tasks running outside of an *installed instance*, need the following settings
|
||||
from the YAML configuration:
|
||||
|
||||
- ``SEARXNG_URL`` from :ref:`server.base_url <settings server>` (aka
|
||||
``PUBLIC_URL``)
|
||||
- ``SEARXNG_BIND_ADDRESS`` from :ref:`server.bind_address <settings server>`
|
||||
- ``SEARXNG_PORT`` from :ref:`server.port <settings server>`
|
||||
|
||||
.. _make node.env:
|
||||
|
||||
Node.js environment (``make node.env``)
|
||||
=======================================
|
||||
|
||||
.. _Node.js: https://nodejs.org/
|
||||
.. _nvm: https://github.com/nvm-sh
|
||||
.. _npm: https://www.npmjs.com/
|
||||
|
||||
.. jinja:: searx
|
||||
|
||||
Node.js_ version {{version.node}} or higher is required to build the themes.
|
||||
If the requirement is not met, the build chain uses nvm_ (Node Version
|
||||
Manager) to install latest LTS of Node.js_ locally: there is no need to
|
||||
install nvm_ or npm_ on your system.
|
||||
|
||||
Use ``make nvm.status`` to get the current status of you Node.js_ and nvm_ setup.
|
||||
|
||||
Here is the output you will typically get on a Ubuntu 20.04 system which serves
|
||||
only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
|
||||
`Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
|
||||
|
||||
::
|
||||
|
||||
$ make nvm.status
|
||||
INFO: Node.js is installed at /usr/bin/node
|
||||
INFO: Node.js is version v10.19.0
|
||||
WARN: minimal Node.js version is 16.13.0
|
||||
INFO: npm is installed at /usr/bin/npm
|
||||
INFO: npm is version 6.14.4
|
||||
WARN: NVM is not installed
|
||||
INFO: to install NVM and Node.js (LTS) use: manage nvm install --lts
|
||||
|
||||
To install you can also use :ref:`make nvm.nodejs`
|
||||
|
||||
.. _make nvm.nodejs:
|
||||
|
||||
``make nvm.nodejs``
|
||||
===================
|
||||
|
||||
Install latest Node.js_ LTS locally (uses nvm_)::
|
||||
|
||||
$ make nvm.nodejs
|
||||
INFO: install (update) NVM at /share/searxng/.nvm
|
||||
INFO: clone: https://github.com/nvm-sh/nvm.git
|
||||
...
|
||||
Downloading and installing node v16.13.0...
|
||||
...
|
||||
INFO: Node.js is installed at searxng/.nvm/versions/node/v16.13.0/bin/node
|
||||
INFO: Node.js is version v16.13.0
|
||||
INFO: npm is installed at searxng/.nvm/versions/node/v16.13.0/bin/npm
|
||||
INFO: npm is version 8.1.0
|
||||
INFO: NVM is installed at searxng/.nvm
|
||||
|
||||
.. _make run:
|
||||
|
||||
``make run``
|
||||
============
|
||||
|
||||
To get up a running a developer instance simply call ``make run``. This enables
|
||||
*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
|
||||
instance and opens the URL in your favorite WEB browser (:man:`xdg-open`)::
|
||||
|
||||
$ make run
|
||||
|
||||
Changes to theme's HTML templates (jinja2) are instant. Changes to the CSS & JS
|
||||
sources of the theme need to be rebuild. You can do that by running::
|
||||
|
||||
$ make themes.all
|
||||
|
||||
Alternatively to ``themes.all`` you can run *live builds* of the theme you are
|
||||
modify::
|
||||
|
||||
$ LIVE_THEME=simple make run
|
||||
|
||||
.. _make clean:
|
||||
|
||||
``make clean``
|
||||
==============
|
||||
|
||||
Drops all intermediate files, all builds, but keep sources untouched. Before
|
||||
calling ``make clean`` stop all processes using the :ref:`make install` or
|
||||
:ref:`make node.env`. ::
|
||||
|
||||
$ make clean
|
||||
CLEAN pyenv
|
||||
PYENV [virtualenv] drop local/py3
|
||||
CLEAN docs -- build/docs dist/docs
|
||||
CLEAN themes -- locally installed npm dependencies
|
||||
...
|
||||
CLEAN test stuff
|
||||
CLEAN common files
|
||||
|
||||
.. _make docs:
|
||||
|
||||
``make docs docs.autobuild docs.clean``
|
||||
=======================================
|
||||
|
||||
We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
|
||||
Documentation <contrib docs>` section. If you want to edit the documentation
|
||||
read our :ref:`make docs.live` section. If you are working in your own brand,
|
||||
adjust your :ref:`settings global`.
|
||||
|
||||
.. _make docs.gh-pages:
|
||||
|
||||
``make docs.gh-pages``
|
||||
======================
|
||||
|
||||
To deploy on github.io first adjust your :ref:`settings global`. For any
|
||||
further read :ref:`deploy on github.io`.
|
||||
|
||||
.. _make test:
|
||||
|
||||
``make test``
|
||||
=============
|
||||
|
||||
Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
|
||||
and ``test.robot``. You can run tests selective, e.g.::
|
||||
|
||||
$ make test.pep8 test.unit test.sh
|
||||
TEST test.pep8 OK
|
||||
...
|
||||
TEST test.unit OK
|
||||
...
|
||||
TEST test.sh OK
|
||||
|
||||
.. _make test.shell:
|
||||
|
||||
``make test.shell``
|
||||
===================
|
||||
|
||||
:ref:`sh lint` / if you have changed some bash scripting run this test before
|
||||
commit.
|
||||
|
||||
.. _make test.pylint:
|
||||
|
||||
``make test.pylint``
|
||||
====================
|
||||
|
||||
.. _Pylint: https://www.pylint.org/
|
||||
|
||||
Pylint_ is known as one of the best source-code, bug and quality checker for the
|
||||
Python programming language. The pylint profile used in the SearXNG project is
|
||||
found in project's root folder :origin:`.pylintrc`.
|
||||
|
||||
.. _make search.checker:
|
||||
|
||||
``search.checker.{engine name}``
|
||||
================================
|
||||
|
||||
To check all engines::
|
||||
|
||||
make search.checker
|
||||
|
||||
To check a engine with whitespace in the name like *google news* replace space
|
||||
by underline::
|
||||
|
||||
make search.checker.google_news
|
||||
|
||||
To see HTTP requests and more use SEARXNG_DEBUG::
|
||||
|
||||
make SEARXNG_DEBUG=1 search.checker.google_news
|
||||
|
||||
.. _3xx: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_redirection
|
||||
|
||||
To filter out HTTP redirects (3xx_)::
|
||||
|
||||
make SEARXNG_DEBUG=1 search.checker.google_news | grep -A1 "HTTP/1.1\" 3[0-9][0-9]"
|
||||
...
|
||||
Engine google news Checking
|
||||
https://news.google.com:443 "GET /search?q=life&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
|
||||
https://news.google.com:443 "GET /search?q=life&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
|
||||
--
|
||||
https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
|
||||
https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
|
||||
--
|
@ -0,0 +1,78 @@
|
||||
.. _offline engines:
|
||||
|
||||
===============
|
||||
Offline Engines
|
||||
===============
|
||||
|
||||
.. sidebar:: offline engines
|
||||
|
||||
- :ref:`demo offline engine`
|
||||
- :ref:`sql engines`
|
||||
- :ref:`engine command`
|
||||
- :origin:`Redis <searx/engines/redis_server.py>`
|
||||
|
||||
To extend the functionality of SearXNG, offline engines are going to be
|
||||
introduced. An offline engine is an engine which does not need Internet
|
||||
connection to perform a search and does not use HTTP to communicate.
|
||||
|
||||
Offline engines can be configured, by adding those to the `engines` list of
|
||||
:origin:`settings.yml <searx/settings.yml>`. An example skeleton for offline
|
||||
engines can be found in :ref:`demo offline engine` (:origin:`demo_offline.py
|
||||
<searx/engines/demo_offline.py>`).
|
||||
|
||||
|
||||
Programming Interface
|
||||
=====================
|
||||
|
||||
:py:func:`init(engine_settings=None) <searx.engines.demo_offline.init>`
|
||||
All offline engines can have their own init function to setup the engine before
|
||||
accepting requests. The function gets the settings from settings.yml as a
|
||||
parameter. This function can be omitted, if there is no need to setup anything
|
||||
in advance.
|
||||
|
||||
:py:func:`search(query, params) <searx.engines.demo_offline.searc>`
|
||||
|
||||
Each offline engine has a function named ``search``. This function is
|
||||
responsible to perform a search and return the results in a presentable
|
||||
format. (Where *presentable* means presentable by the selected result
|
||||
template.)
|
||||
|
||||
The return value is a list of results retrieved by the engine.
|
||||
|
||||
Engine representation in ``/config``
|
||||
If an engine is offline, the attribute ``offline`` is set to ``True``.
|
||||
|
||||
.. _offline requirements:
|
||||
|
||||
Extra Dependencies
|
||||
==================
|
||||
|
||||
If an offline engine depends on an external tool, SearXNG does not install it by
|
||||
default. When an administrator configures such engine and starts the instance,
|
||||
the process returns an error with the list of missing dependencies. Also,
|
||||
required dependencies will be added to the comment/description of the engine, so
|
||||
admins can install packages in advance.
|
||||
|
||||
If there is a need to install additional packages in *Python's Virtual
|
||||
Environment* of your SearXNG instance you need to switch into the environment
|
||||
(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`::
|
||||
|
||||
$ sudo utils/searxng.sh instance cmd bash
|
||||
(searxng-pyenv)$ pip install ...
|
||||
|
||||
|
||||
Private engines (Security)
|
||||
==========================
|
||||
|
||||
To limit the access to offline engines, if an instance is available publicly,
|
||||
administrators can set token(s) for each of the :ref:`private engines`. If a
|
||||
query contains a valid token, then SearXNG performs the requested private
|
||||
search. If not, requests from an offline engines return errors.
|
||||
|
||||
|
||||
Acknowledgement
|
||||
===============
|
||||
|
||||
This development was sponsored by `Search and Discovery Fund
|
||||
<https://nlnet.nl/discovery>`_ of `NLnet Foundation <https://nlnet.nl/>`_ .
|
||||
|
@ -0,0 +1,110 @@
|
||||
.. _dev plugin:
|
||||
|
||||
=======
|
||||
Plugins
|
||||
=======
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`plugins generic`
|
||||
|
||||
Plugins can extend or replace functionality of various components of searx.
|
||||
|
||||
Example plugin
|
||||
==============
|
||||
|
||||
.. code:: python
|
||||
|
||||
name = 'Example plugin'
|
||||
description = 'This plugin extends the suggestions with the word "example"'
|
||||
default_on = False # disabled by default
|
||||
|
||||
js_dependencies = tuple() # optional, list of static js files
|
||||
css_dependencies = tuple() # optional, list of static css files
|
||||
|
||||
|
||||
# attach callback to the post search hook
|
||||
# request: flask request object
|
||||
# ctx: the whole local context of the post search hook
|
||||
def post_search(request, search):
|
||||
search.result_container.suggestions.add('example')
|
||||
return True
|
||||
|
||||
External plugins
|
||||
================
|
||||
|
||||
SearXNG supports *external plugins* / there is no need to install one, SearXNG
|
||||
runs out of the box. But to demonstrate; in the example below we install the
|
||||
SearXNG plugins from *The Green Web Foundation* `[ref]
|
||||
<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
$ sudo utils/searxng.sh instance cmd bash
|
||||
(searxng-pyenv)$ pip install git+https://github.com/return42/tgwf-searx-plugins
|
||||
|
||||
In the :ref:`settings.yml` activate the ``plugins:`` section and add module
|
||||
``only_show_green_results`` from ``tgwf-searx-plugins``.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
plugins:
|
||||
...
|
||||
- only_show_green_results
|
||||
...
|
||||
|
||||
|
||||
Plugin entry points
|
||||
===================
|
||||
|
||||
Entry points (hooks) define when a plugin runs. Right now only three hooks are
|
||||
implemented. So feel free to implement a hook if it fits the behaviour of your
|
||||
plugin. A plugin doesn't need to implement all the hooks.
|
||||
|
||||
|
||||
.. py:function:: pre_search(request, search) -> bool
|
||||
|
||||
Runs BEFORE the search request.
|
||||
|
||||
`search.result_container` can be changed.
|
||||
|
||||
Return a boolean:
|
||||
|
||||
* True to continue the search
|
||||
* False to stop the search
|
||||
|
||||
:param flask.request request:
|
||||
:param searx.search.SearchWithPlugins search:
|
||||
:return: False to stop the search
|
||||
:rtype: bool
|
||||
|
||||
|
||||
.. py:function:: post_search(request, search) -> None
|
||||
|
||||
Runs AFTER the search request.
|
||||
|
||||
:param flask.request request: Flask request.
|
||||
:param searx.search.SearchWithPlugins search: Context.
|
||||
|
||||
|
||||
.. py:function:: on_result(request, search, result) -> bool
|
||||
|
||||
Runs for each result of each engine.
|
||||
|
||||
`result` can be changed.
|
||||
|
||||
If `result["url"]` is defined, then `result["parsed_url"] = urlparse(result['url'])`
|
||||
|
||||
.. warning::
|
||||
`result["url"]` can be changed, but `result["parsed_url"]` must be updated too.
|
||||
|
||||
Return a boolean:
|
||||
|
||||
* True to keep the result
|
||||
* False to remove the result
|
||||
|
||||
:param flask.request request:
|
||||
:param searx.search.SearchWithPlugins search:
|
||||
:param typing.Dict result: Result, see - :ref:`engine results`
|
||||
:return: True to keep the result
|
||||
:rtype: bool
|
@ -0,0 +1,72 @@
|
||||
.. _devquickstart:
|
||||
|
||||
======================
|
||||
Development Quickstart
|
||||
======================
|
||||
|
||||
.. _npm: https://www.npmjs.com/
|
||||
.. _Node.js: https://nodejs.org/
|
||||
|
||||
SearXNG loves developers, just clone and start hacking. All the rest is done for
|
||||
you simply by using :ref:`make <makefile>`.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
git clone https://github.com/searxng/searxng.git searxng
|
||||
|
||||
Here is how a minimal workflow looks like:
|
||||
|
||||
1. *start* hacking
|
||||
2. *run* your code: :ref:`make run`
|
||||
3. *test* your code: :ref:`make test`
|
||||
|
||||
If you think at some point something fails, go back to *start*. Otherwise,
|
||||
choose a meaningful commit message and we are happy to receive your pull
|
||||
request. To not end in *wild west* we have some directives, please pay attention
|
||||
to our ":ref:`how to contribute`" guideline.
|
||||
|
||||
If you implement themes, you will need to setup a :ref:`make node.env` once:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make node.env
|
||||
|
||||
Before you call *make run* (2.), you need to compile the modified styles and
|
||||
JavaScript:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make themes.all
|
||||
|
||||
Alternatively you can also compile selective the theme you have modified,
|
||||
e.g. the *simple* theme.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make themes.simple
|
||||
|
||||
.. tip::
|
||||
|
||||
To get live builds while modifying CSS & JS use: ``LIVE_THEME=simple make run``
|
||||
|
||||
If you finished your *tests* you can start to commit your changes. To separate
|
||||
the modified source code from the build products first run:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make static.build.restore
|
||||
|
||||
This will restore the old build products and only your changes of the code
|
||||
remain in the working tree which can now be added & commited. When all sources
|
||||
are commited, you can commit the build products simply by:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make static.build.commit
|
||||
|
||||
Commiting the build products should be the last step, just before you send us
|
||||
your PR. There is also a make target to rewind this last build commit:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
make static.build.drop
|
@ -0,0 +1,126 @@
|
||||
.. _search API:
|
||||
|
||||
==========
|
||||
Search API
|
||||
==========
|
||||
|
||||
The search supports both ``GET`` and ``POST``.
|
||||
|
||||
Furthermore, two endpoints ``/`` and ``/search`` are available for querying.
|
||||
|
||||
|
||||
``GET /``
|
||||
|
||||
``GET /search``
|
||||
|
||||
Parameters
|
||||
==========
|
||||
|
||||
.. sidebar:: Further reading ..
|
||||
|
||||
- :ref:`engines-dev`
|
||||
- :ref:`settings.yml`
|
||||
- :ref:`configured engines`
|
||||
|
||||
``q`` : required
|
||||
The search query. This string is passed to external search services. Thus,
|
||||
SearXNG supports syntax of each search service. For example, ``site:github.com
|
||||
SearXNG`` is a valid query for Google. However, if simply the query above is
|
||||
passed to any search engine which does not filter its results based on this
|
||||
syntax, you might not get the results you wanted.
|
||||
|
||||
See more at :ref:`search-syntax`
|
||||
|
||||
``categories`` : optional
|
||||
Comma separated list, specifies the active search categories (see
|
||||
:ref:`configured engines`)
|
||||
|
||||
``engines`` : optional
|
||||
Comma separated list, specifies the active search engines (see
|
||||
:ref:`configured engines`).
|
||||
|
||||
``language`` : default from :ref:`settings search`
|
||||
Code of the language.
|
||||
|
||||
``pageno`` : default ``1``
|
||||
Search page number.
|
||||
|
||||
``time_range`` : optional
|
||||
[ ``day``, ``month``, ``year`` ]
|
||||
|
||||
Time range of search for engines which support it. See if an engine supports
|
||||
time range search in the preferences page of an instance.
|
||||
|
||||
``format`` : optional
|
||||
[ ``json``, ``csv``, ``rss`` ]
|
||||
|
||||
Output format of results. Format needs to be activated in :ref:`settings
|
||||
search`.
|
||||
|
||||
``results_on_new_tab`` : default ``0``
|
||||
[ ``0``, ``1`` ]
|
||||
|
||||
Open search results on new tab.
|
||||
|
||||
``image_proxy`` : default from :ref:`settings server`
|
||||
[ ``True``, ``False`` ]
|
||||
|
||||
Proxy image results through SearXNG.
|
||||
|
||||
``autocomplete`` : default from :ref:`settings search`
|
||||
[ ``google``, ``dbpedia``, ``duckduckgo``, ``startpage``, ``wikipedia``,
|
||||
``swisscows``, ``qwant`` ]
|
||||
|
||||
Service which completes words as you type.
|
||||
|
||||
``safesearch`` : default from :ref:`settings search`
|
||||
[ ``0``, ``1``, ``2`` ]
|
||||
|
||||
Filter search results of engines which support safe search. See if an engine
|
||||
supports safe search in the preferences page of an instance.
|
||||
|
||||
``theme`` : default ``simple``
|
||||
[ ``simple`` ]
|
||||
|
||||
Theme of instance.
|
||||
|
||||
Please note, available themes depend on an instance. It is possible that an
|
||||
instance administrator deleted, created or renamed themes on their instance.
|
||||
See the available options in the preferences page of the instance.
|
||||
|
||||
``enabled_plugins`` : optional
|
||||
List of enabled plugins.
|
||||
|
||||
:default:
|
||||
``Hash_plugin``, ``Search_on_category_select``,
|
||||
``Self_Information``, ``Tracker_URL_remover``,
|
||||
``Ahmia_blacklist``
|
||||
|
||||
:values:
|
||||
.. enabled by default
|
||||
|
||||
``Hash_plugin``, ``Search_on_category_select``,
|
||||
``Self_Information``, ``Tracker_URL_remover``,
|
||||
``Ahmia_blacklist``,
|
||||
|
||||
.. disabled by default
|
||||
|
||||
``Hostname_replace``, ``Open_Access_DOI_rewrite``,
|
||||
``Vim-like_hotkeys``, ``Tor_check_plugin``
|
||||
|
||||
``disabled_plugins``: optional
|
||||
List of disabled plugins.
|
||||
|
||||
:default:
|
||||
``Hostname_replace``, ``Open_Access_DOI_rewrite``,
|
||||
``Vim-like_hotkeys``, ``Tor_check_plugin``
|
||||
|
||||
:values:
|
||||
see values from ``enabled_plugins``
|
||||
|
||||
``enabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
|
||||
List of enabled engines.
|
||||
|
||||
``disabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
|
||||
List of disabled engines.
|
||||
|
@ -0,0 +1,15 @@
|
||||
.. _searxng_extra:
|
||||
|
||||
=============================
|
||||
Tooling box ``searxng_extra``
|
||||
=============================
|
||||
|
||||
In the folder :origin:`searxng_extra/` we maintain some tools useful for CI and
|
||||
developers.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
update
|
||||
standalone_searx.py
|
@ -0,0 +1,9 @@
|
||||
|
||||
.. _standalone_searx.py:
|
||||
|
||||
=====================================
|
||||
``searxng_extra/standalone_searx.py``
|
||||
=====================================
|
||||
|
||||
.. automodule:: searxng_extra.standalone_searx
|
||||
:members:
|
@ -0,0 +1,88 @@
|
||||
=========================
|
||||
``searxng_extra/update/``
|
||||
=========================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/__init__.py>`
|
||||
|
||||
Scripts to update static data in :origin:`searx/data/`
|
||||
|
||||
.. _update_ahmia_blacklist.py:
|
||||
|
||||
``update_ahmia_blacklist.py``
|
||||
=============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_ahmia_blacklist.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_ahmia_blacklist
|
||||
:members:
|
||||
|
||||
|
||||
``update_currencies.py``
|
||||
========================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_currencies.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_currencies
|
||||
:members:
|
||||
|
||||
``update_engine_descriptions.py``
|
||||
=================================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_engine_descriptions.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_engine_descriptions
|
||||
:members:
|
||||
|
||||
|
||||
``update_external_bangs.py``
|
||||
============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_external_bangs.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_external_bangs
|
||||
:members:
|
||||
|
||||
|
||||
``update_firefox_version.py``
|
||||
=============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_firefox_version.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_firefox_version
|
||||
:members:
|
||||
|
||||
|
||||
``update_languages.py``
|
||||
=======================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_languages.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_languages
|
||||
:members:
|
||||
|
||||
|
||||
``update_osm_keys_tags.py``
|
||||
===========================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_osm_keys_tags.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_osm_keys_tags
|
||||
:members:
|
||||
|
||||
|
||||
``update_pygments.py``
|
||||
======================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_pygments.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_pygments
|
||||
:members:
|
||||
|
||||
|
||||
``update_wikidata_units.py``
|
||||
============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_wikidata_units
|
||||
:members:
|
@ -0,0 +1,81 @@
|
||||
.. _translation:
|
||||
|
||||
===========
|
||||
Translation
|
||||
===========
|
||||
|
||||
.. _translate.codeberg.org: https://translate.codeberg.org/projects/searxng/
|
||||
.. _Weblate: https://docs.weblate.org
|
||||
.. _translations branch: https://github.com/searxng/searxng/tree/translations
|
||||
.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt
|
||||
.. _Weblate repository: https://translate.codeberg.org/projects/searxng/searxng/#repository
|
||||
.. _wlc: https://docs.weblate.org/en/latest/wlc.html
|
||||
|
||||
.. |translated| image:: https://translate.codeberg.org/widgets/searxng/-/searxng/svg-badge.svg
|
||||
:target: https://translate.codeberg.org/projects/searxng/
|
||||
|
||||
.. sidebar:: |translated|
|
||||
|
||||
- :ref:`searx.babel_extract`
|
||||
- Weblate_
|
||||
- SearXNG `translations branch`_
|
||||
- SearXNG `Weblate repository`_
|
||||
- Weblate Client: wlc_
|
||||
- Babel Command-Line: `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
|
||||
- `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
|
||||
|
||||
Translation takes place on translate.codeberg.org_.
|
||||
|
||||
Translations which has been added by translators on the translate.codeberg.org_ UI are
|
||||
committed to Weblate's counterpart of the SearXNG *origin* repository which is
|
||||
located at ``https://translate.codeberg.org/git/searxng/searxng``.
|
||||
|
||||
There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take
|
||||
care of the synchronization with the *origin*. To avoid merging commits from
|
||||
the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull
|
||||
request* (PR) is created by this workflow.
|
||||
|
||||
Weblate monitors the `translations branch`_, not the ``master`` branch. This
|
||||
branch is an `orphan branch`_, decoupled from the master branch (we already know
|
||||
orphan branches from the ``gh-pages``). The `translations branch`_ contains
|
||||
only the
|
||||
|
||||
- ``translation/messages.pot`` and the
|
||||
- ``translation/*/messages.po`` files, nothing else.
|
||||
|
||||
|
||||
.. _SearXNG Weblate workflow:
|
||||
|
||||
.. figure:: translation.svg
|
||||
|
||||
SearXNG's PR workflow to be in sync with Weblate
|
||||
|
||||
Sync from *origin* to *weblate*: using ``make weblate.push.translations``
|
||||
For each commit on the ``master`` branch of SearXNG *origin* the GitHub job
|
||||
:origin:`babel / Update translations branch
|
||||
<.github/workflows/integration.yml>` checks for updated translations.
|
||||
|
||||
Sync from *weblate* to *origin*: using ``make weblate.translations.commit``
|
||||
Every Friday, the GitHub workflow :origin:`babel / create PR for additons from
|
||||
weblate <.github/workflows/translations-update.yml>` creates a PR with the
|
||||
updated translation files:
|
||||
|
||||
- ``translation/messages.pot``,
|
||||
- ``translation/*/messages.po`` and
|
||||
- ``translation/*/messages.mo``
|
||||
|
||||
wlc
|
||||
===
|
||||
|
||||
.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config
|
||||
.. _API key: https://translate.codeberg.org/accounts/profile/#api
|
||||
|
||||
All weblate integration is done by GitHub workflows, but if you want to use wlc_,
|
||||
copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate``
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[keys]
|
||||
https://translate.codeberg.org/api/ = APIKEY
|
||||
|
||||
Replace ``APIKEY`` by your `API key`_.
|
@ -0,0 +1,40 @@
|
||||
Donate to searxng.org
|
||||
=====================
|
||||
|
||||
Why donating?
|
||||
-------------
|
||||
|
||||
If you want to support the SearXNG team you can make a donation.
|
||||
|
||||
This will help us to pay the costs for:
|
||||
|
||||
- the two VPS servers for searx.space
|
||||
- the domain names (searxng.org and searx.space)
|
||||
- the protonmail account
|
||||
|
||||
If there is enough fund we can ask for a security audit or pay an User Experience (UX) designer.
|
||||
|
||||
Payment methods
|
||||
---------------
|
||||
|
||||
- Credit/debit card and bank transfer
|
||||
|
||||
- `Liberapay`_ (recurrent donation)
|
||||
- `Buy Me a Coffee`_ (one time donation)
|
||||
|
||||
- Cryptocurrency
|
||||
|
||||
- Bitcoin: `bc1qn3rw8t86h05cs3grx2kmwmptw9k4kt4hyzktqj`_ (Segwit
|
||||
compatible)
|
||||
- Bitcoin cash: `qpead2yu482e3h9amy5zk45l8qrfhk59jcpw3cth9e`_
|
||||
- Ethereum: `0xCf82c7eb915Ee70b5B69C1bBB5525e157F35FA43`_
|
||||
- Dogecoin: `DBCYS9issTt84pddXSsTHpQxyQDTFp1TE4`_
|
||||
- Litecoin: `ltc1q5j6x6f4f2htldhq570e353clc8fmw44ra5er5q`_
|
||||
|
||||
.. _Liberapay: https://liberapay.com/SearXNG/
|
||||
.. _Buy Me a Coffee: https://buymeacoffee.com/searxng
|
||||
.. _bc1qn3rw8t86h05cs3grx2kmwmptw9k4kt4hyzktqj: bitcoin:bc1qn3rw8t86h05cs3grx2kmwmptw9k4kt4hyzktqj
|
||||
.. _qpead2yu482e3h9amy5zk45l8qrfhk59jcpw3cth9e: bitcoincash:qpead2yu482e3h9amy5zk45l8qrfhk59jcpw3cth9e
|
||||
.. _0xCf82c7eb915Ee70b5B69C1bBB5525e157F35FA43: ethereum:0xCf82c7eb915Ee70b5B69C1bBB5525e157F35FA43
|
||||
.. _DBCYS9issTt84pddXSsTHpQxyQDTFp1TE4: dogecoin:DBCYS9issTt84pddXSsTHpQxyQDTFp1TE4
|
||||
.. _ltc1q5j6x6f4f2htldhq570e353clc8fmw44ra5er5q: litecoin:ltc1q5j6x6f4f2htldhq570e353clc8fmw44ra5er5q
|
@ -0,0 +1,41 @@
|
||||
==================
|
||||
Welcome to SearXNG
|
||||
==================
|
||||
|
||||
*Search without being tracked.*
|
||||
|
||||
SearXNG is a free internet metasearch engine which aggregates results from more
|
||||
than 70 search services. Users are neither tracked nor profiled. Additionally,
|
||||
SearXNG can be used over Tor for online anonymity.
|
||||
|
||||
Get started with SearXNG by using one of the instances listed at searx.space_.
|
||||
If you don't trust anyone, you can set up your own, see :ref:`installation`.
|
||||
|
||||
.. sidebar:: Features
|
||||
|
||||
- Self hosted
|
||||
- No user tracking
|
||||
- No user profiling
|
||||
- About 70 supported search engines
|
||||
- Easy integration with any search engine
|
||||
- Cookies are not used by default
|
||||
- Secure, encrypted connections (HTTPS/SSL)
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
SearXNG development has been started in the middle of 2021 as a fork of the
|
||||
searx project.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
user/index
|
||||
own-instance
|
||||
admin/index
|
||||
dev/index
|
||||
utils/index
|
||||
src/index
|
||||
donate
|
||||
|
||||
.. _searx.space: https://searx.space
|
@ -0,0 +1,79 @@
|
||||
===========================
|
||||
Why use a private instance?
|
||||
===========================
|
||||
|
||||
*"Is it worth to run my own instance?"*
|
||||
|
||||
\.\. is a common question among SearXNG users. Before answering this question,
|
||||
see what options a SearXNG user has.
|
||||
|
||||
Public instances are open to everyone who has access to its URL. Usually, these
|
||||
are operated by unknown parties (from the users' point of view). Private
|
||||
instances can be used by a select group of people. It is for example a SearXNG of
|
||||
group of friends or a company which can be accessed through VPN. Also it can be
|
||||
single user one which runs on the user's laptop.
|
||||
|
||||
To gain more insight on how these instances work let's dive into how SearXNG
|
||||
protects its users.
|
||||
|
||||
How does SearXNG protect privacy?
|
||||
=================================
|
||||
|
||||
SearXNG protects the privacy of its users in multiple ways regardless of the type
|
||||
of the instance (private, public). Removal of private data from search requests
|
||||
comes in three forms:
|
||||
|
||||
1. removal of private data from requests going to search services
|
||||
2. not forwarding anything from a third party services through search services
|
||||
(e.g. advertisement)
|
||||
3. removal of private data from requests going to the result pages
|
||||
|
||||
Removing private data means not sending cookies to external search engines and
|
||||
generating a random browser profile for every request. Thus, it does not matter
|
||||
if a public or private instance handles the request, because it is anonymized in
|
||||
both cases. IP addresses will be the IP of the instance. But SearXNG can be
|
||||
configured to use proxy or Tor. `Result proxy
|
||||
<https://github.com/asciimoo/morty>`__ is supported, too.
|
||||
|
||||
SearXNG does not serve ads or tracking content unlike most search services. So
|
||||
private data is not forwarded to third parties who might monetize it. Besides
|
||||
protecting users from search services, both referring page and search query are
|
||||
hidden from visited result pages.
|
||||
|
||||
|
||||
What are the consequences of using public instances?
|
||||
----------------------------------------------------
|
||||
|
||||
If someone uses a public instance, they have to trust the administrator of that
|
||||
instance. This means that the user of the public instance does not know whether
|
||||
their requests are logged, aggregated and sent or sold to a third party.
|
||||
|
||||
Also, public instances without proper protection are more vulnerable to abusing
|
||||
the search service, In this case the external service in exchange returns
|
||||
CAPTCHAs or bans the IP of the instance. Thus, search requests return less
|
||||
results.
|
||||
|
||||
I see. What about private instances?
|
||||
------------------------------------
|
||||
|
||||
If users run their :ref:`own instances <installation>`, everything is in their
|
||||
control: the source code, logging settings and private data. Unknown instance
|
||||
administrators do not have to be trusted.
|
||||
|
||||
Furthermore, as the default settings of their instance is editable, there is no
|
||||
need to use cookies to tailor SearXNG to their needs. So preferences will not be
|
||||
reset to defaults when clearing browser cookies. As settings are stored on
|
||||
their computer, it will not be accessible to others as long as their computer is
|
||||
not compromised.
|
||||
|
||||
Conclusion
|
||||
==========
|
||||
|
||||
Always use an instance which is operated by people you trust. The privacy
|
||||
features of SearXNG are available to users no matter what kind of instance they
|
||||
use.
|
||||
|
||||
If someone is on the go or just wants to try SearXNG for the first time public
|
||||
instances are the best choices. Additionally, public instance are making a
|
||||
world a better place, because those who cannot or do not want to run an
|
||||
instance, have access to a privacy respecting search service.
|
@ -0,0 +1,14 @@
|
||||
===========
|
||||
Source-Code
|
||||
===========
|
||||
|
||||
This is a partial documentation of our source code. We are not aiming to document
|
||||
every item from the source code, but we will add documentation when requested.
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
:glob:
|
||||
|
||||
searx.*
|
@ -0,0 +1,8 @@
|
||||
.. _searx.babel_extract:
|
||||
|
||||
===============================
|
||||
Custom message extractor (i18n)
|
||||
===============================
|
||||
|
||||
.. automodule:: searx.babel_extract
|
||||
:members:
|
@ -0,0 +1,9 @@
|
||||
.. _demo offline engine:
|
||||
|
||||
===================
|
||||
Demo Offline Engine
|
||||
===================
|
||||
|
||||
.. automodule:: searx.engines.demo_offline
|
||||
:members:
|
||||
|
@ -0,0 +1,9 @@
|
||||
.. _demo online engine:
|
||||
|
||||
==================
|
||||
Demo Online Engine
|
||||
==================
|
||||
|
||||
.. automodule:: searx.engines.demo_online
|
||||
:members:
|
||||
|
@ -0,0 +1,55 @@
|
||||
.. _google engines:
|
||||
|
||||
==============
|
||||
Google Engines
|
||||
==============
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
.. _google API:
|
||||
|
||||
google API
|
||||
==========
|
||||
|
||||
.. _Query Parameter Definitions:
|
||||
https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
|
||||
|
||||
For detailed description of the *REST-full* API see: `Query Parameter
|
||||
Definitions`_. Not all parameters can be appied and some engines are *special*
|
||||
(e.g. :ref:`google news engine`).
|
||||
|
||||
.. _google web engine:
|
||||
|
||||
Google WEB
|
||||
==========
|
||||
|
||||
.. automodule:: searx.engines.google
|
||||
:members:
|
||||
|
||||
.. _google images engine:
|
||||
|
||||
Google Images
|
||||
=============
|
||||
|
||||
.. automodule:: searx.engines.google_images
|
||||
:members:
|
||||
|
||||
.. _google videos engine:
|
||||
|
||||
Google Videos
|
||||
=============
|
||||
|
||||
.. automodule:: searx.engines.google_videos
|
||||
:members:
|
||||
|
||||
.. _google news engine:
|
||||
|
||||
Google News
|
||||
===========
|
||||
|
||||
.. automodule:: searx.engines.google_news
|
||||
:members:
|
@ -0,0 +1,8 @@
|
||||
.. _load_engines:
|
||||
|
||||
============
|
||||
Load Engines
|
||||
============
|
||||
|
||||
.. automodule:: searx.engines
|
||||
:members:
|
@ -0,0 +1,9 @@
|
||||
.. _tineye engine:
|
||||
|
||||
======
|
||||
Tineye
|
||||
======
|
||||
|
||||
.. automodule:: searx.engines.tineye
|
||||
:members:
|
||||
|
@ -0,0 +1,8 @@
|
||||
.. _yahoo engine:
|
||||
|
||||
============
|
||||
Yahoo Engine
|
||||
============
|
||||
|
||||
.. automodule:: searx.engines.yahoo
|
||||
:members:
|
@ -0,0 +1,8 @@
|
||||
.. _searx.infopage:
|
||||
|
||||
================
|
||||
Online ``/info``
|
||||
================
|
||||
|
||||
.. automodule:: searx.infopage
|
||||
:members:
|
@ -0,0 +1,8 @@
|
||||
.. _searx.locales:
|
||||
|
||||
=======
|
||||
Locales
|
||||
=======
|
||||
|
||||
.. automodule:: searx.locales
|
||||
:members:
|
@ -0,0 +1,8 @@
|
||||
.. _autodetect search language:
|
||||
|
||||
======================
|
||||
Search language plugin
|
||||
======================
|
||||
|
||||
.. automodule:: searx.plugins.autodetect_search_language
|
||||
:members:
|
@ -0,0 +1,13 @@
|
||||
.. _limiter plugin:
|
||||
|
||||
==============
|
||||
Limiter Plugin
|
||||
==============
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
The :ref:`limiter plugin` requires a :ref:`Redis <settings redis>` database.
|
||||
|
||||
.. automodule:: searx.plugins.limiter
|
||||
:members:
|
||||
|
@ -0,0 +1,9 @@
|
||||
.. _tor check plugin:
|
||||
|
||||
================
|
||||
Tor check plugin
|
||||
================
|
||||
|
||||
.. automodule:: searx.plugins.tor_check
|
||||
:members:
|
||||
|
@ -0,0 +1,8 @@
|
||||
.. _redis db:
|
||||
|
||||
========
|
||||
Redis DB
|
||||
========
|
||||
|
||||
.. automodule:: searx.redisdb
|
||||
:members:
|
@ -0,0 +1,8 @@
|
||||
.. _searx.redis:
|
||||
|
||||
=============
|
||||
Redis Library
|
||||
=============
|
||||
|
||||
.. automodule:: searx.redislib
|
||||
:members:
|
@ -0,0 +1,38 @@
|
||||
.. _searx.search:
|
||||
|
||||
======
|
||||
Search
|
||||
======
|
||||
|
||||
.. autoclass:: searx.search.EngineRef
|
||||
:members:
|
||||
|
||||
.. autoclass:: searx.search.SearchQuery
|
||||
:members:
|
||||
|
||||
.. autoclass:: searx.search.Search
|
||||
|
||||
.. attribute:: search_query
|
||||
:type: searx.search.SearchQuery
|
||||
|
||||
.. attribute:: result_container
|
||||
:type: searx.results.ResultContainer
|
||||
|
||||
.. automethod:: search() -> searx.results.ResultContainer
|
||||
|
||||
.. autoclass:: searx.search.SearchWithPlugins
|
||||
:members:
|
||||
|
||||
.. attribute:: search_query
|
||||
:type: searx.search.SearchQuery
|
||||
|
||||
.. attribute:: result_container
|
||||
:type: searx.results.ResultContainer
|
||||
|
||||
.. attribute:: ordered_plugin_list
|
||||
:type: typing.List
|
||||
|
||||
.. attribute:: request
|
||||
:type: flask.request
|
||||
|
||||
.. automethod:: search() -> searx.results.ResultContainer
|
@ -0,0 +1,8 @@
|
||||
.. _searx.utils:
|
||||
|
||||
=================================
|
||||
Utility functions for the engines
|
||||
=================================
|
||||
|
||||
.. automodule:: searx.utils
|
||||
:members:
|
@ -0,0 +1,15 @@
|
||||
================
|
||||
User information
|
||||
================
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 3
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
|
||||
.. _search-syntax:
|
||||
|
||||
.. include:: search-syntax.md
|
||||
:parser: myst_parser.sphinx_
|
||||
|
@ -0,0 +1,30 @@
|
||||
.. _searx_utils:
|
||||
.. _toolboxing:
|
||||
|
||||
==================
|
||||
DevOps tooling box
|
||||
==================
|
||||
|
||||
In the folder :origin:`utils/` we maintain some tools useful for administrators
|
||||
and developers.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
searxng.sh
|
||||
lxc.sh
|
||||
|
||||
Common command environments
|
||||
===========================
|
||||
|
||||
The scripts in our tooling box often dispose of common environments:
|
||||
|
||||
``FORCE_TIMEOUT`` : environment
|
||||
Sets timeout for interactive prompts. If you want to run a script in batch
|
||||
job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
|
||||
SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite
|
||||
<lxc-searxng.env>` use::
|
||||
|
||||
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all
|
||||
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
|
@ -0,0 +1,196 @@
|
||||
|
||||
.. _snap: https://snapcraft.io
|
||||
.. _snapcraft LXD: https://snapcraft.io/lxd
|
||||
.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/
|
||||
.. _LXC: https://linuxcontainers.org/lxc/introduction/
|
||||
.. _LXD: https://linuxcontainers.org/lxd/introduction/
|
||||
.. _`LXD@github`: https://github.com/lxc/lxd
|
||||
|
||||
.. _archlinux: https://www.archlinux.org/
|
||||
|
||||
.. _lxc.sh:
|
||||
|
||||
================
|
||||
``utils/lxc.sh``
|
||||
================
|
||||
|
||||
.. sidebar:: further reading
|
||||
|
||||
- snap_, `snapcraft LXD`_
|
||||
- LXC_, LXD_
|
||||
- `LXC/LXD Image Server`_
|
||||
- `LXD@github`_
|
||||
|
||||
With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
|
||||
containers, what we call the: *lxc suite*. The *SearXNG suite*
|
||||
(:origin:`lxc-searxng.env <utils/lxc-searxng.env>`) is loaded by default, every time
|
||||
you start the ``lxc.sh`` script (*you do not need to care about*).
|
||||
|
||||
Before you can start with containers, you need to install and initiate LXD_
|
||||
once::
|
||||
|
||||
$ snap install lxd
|
||||
$ lxd init --auto
|
||||
|
||||
To make use of the containers from the *SearXNG suite*, you have to build the
|
||||
:ref:`LXC suite containers <lxc.sh help>` initial. But be warned, **this might
|
||||
take some time**::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh build
|
||||
|
||||
A cup of coffee later, your LXC suite is build up and you can run whatever task
|
||||
you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh
|
||||
help>`.
|
||||
|
||||
.. hint::
|
||||
|
||||
If you see any problems with the internet connectivity of your
|
||||
containers read section :ref:`internet connectivity docker`.
|
||||
|
||||
If you do not want to build all containers, **you can build just one**::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh build searxng-archlinux
|
||||
|
||||
*Good to know ...*
|
||||
|
||||
Each container shares the root folder of the repository and the command
|
||||
``utils/lxc.sh cmd`` **handles relative path names transparent**, compare output
|
||||
of::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile
|
||||
...
|
||||
|
||||
In the containers, you can run what ever you want, e.g. to start a bash use::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
|
||||
INFO: [searxng-archlinux] bash
|
||||
[root@searxng-archlinux SearXNG]#
|
||||
|
||||
If there comes the time you want to **get rid off all** the containers and
|
||||
**clean up local images** just type::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh remove
|
||||
$ sudo -H ./utils/lxc.sh remove images
|
||||
|
||||
.. _internet connectivity docker:
|
||||
|
||||
Internet Connectivity & Docker
|
||||
==============================
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
- `Docker blocking network of existing LXC containers <https://github.com/docker/for-linux/issues/103>`__
|
||||
- `Docker and IPtables (fralef.me) <https://fralef.me/docker-and-iptables.html>`__
|
||||
- `Docker and iptables (docker.com) <https://docs.docker.com/network/iptables/#docker-on-a-router/>`__
|
||||
|
||||
There is a conflict in the ``iptables`` setup of Docker & LXC. If you have
|
||||
docker installed, you may find that the internet connectivity of your LXD
|
||||
containers no longer work.
|
||||
|
||||
Whenever docker is started (reboot) it sets the iptables policy for the
|
||||
``FORWARD`` chain to ``DROP`` `[ref]
|
||||
<https://docs.docker.com/network/iptables/#docker-on-a-router>`__::
|
||||
|
||||
$ sudo -H iptables-save | grep FORWARD
|
||||
:FORWARD ACCEPT [7048:7851230]
|
||||
:FORWARD DROP [7048:7851230]
|
||||
|
||||
A handy solution of this problem might be to reset the policy for the
|
||||
``FORWARD`` chain after the network has been initialized. For this create a
|
||||
file in the ``if-up`` section of the network (``/etc/network/if-up.d/iptable``)
|
||||
and insert the following lines::
|
||||
|
||||
#!/bin/sh
|
||||
iptables -F FORWARD
|
||||
iptables -P FORWARD ACCEPT
|
||||
|
||||
Don't forget to set the execution bit::
|
||||
|
||||
sudo chmod ugo+x /etc/network/if-up.d/iptable
|
||||
|
||||
Reboot your system and check the iptables rules::
|
||||
|
||||
$ sudo -H iptables-save | grep FORWARD
|
||||
:FORWARD ACCEPT [7048:7851230]
|
||||
:FORWARD ACCEPT [7048:7851230]
|
||||
|
||||
|
||||
.. _lxc.sh install suite:
|
||||
|
||||
Install suite
|
||||
=============
|
||||
|
||||
To install the complete :ref:`SearXNG suite (includes searx, morty & filtron)
|
||||
<lxc-searxng.env>` into all LXC_ use::
|
||||
|
||||
$ sudo -H ./utils/lxc.sh install suite
|
||||
|
||||
The command above installs a SearXNG suite (see :ref:`installation scripts`).
|
||||
To :ref:`install a nginx <installation nginx>` reverse proxy (or alternatively
|
||||
use :ref:`apache <installation apache>`)::
|
||||
|
||||
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
|
||||
|
||||
To get the IP (URL) of the SearXNG service in the containers use ``show suite``
|
||||
command. To test instances from containers just open the URLs in your
|
||||
WEB-Browser::
|
||||
|
||||
$ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL
|
||||
|
||||
[searxng-ubu2110] SEARXNG_URL : http://n.n.n.147/searxng
|
||||
[searxng-ubu2004] SEARXNG_URL : http://n.n.n.246/searxng
|
||||
[searxnggfedora35] SEARXNG_URL : http://n.n.n.140/searxng
|
||||
[searxng-archlinux] SEARXNG_URL : http://n.n.n.165/searxng
|
||||
|
||||
|
||||
Running commands
|
||||
================
|
||||
|
||||
**Inside containers, you can use make or run scripts** from the
|
||||
:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the
|
||||
Makefile target ``test`` in the archlinux_ container::
|
||||
|
||||
sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost
|
||||
sudo -H ./utils/lxc.sh cmd searxng-archlinux make test
|
||||
|
||||
|
||||
Setup SearXNG buildhost
|
||||
=======================
|
||||
|
||||
You can **install the SearXNG buildhost environment** into one or all containers.
|
||||
The installation procedure to set up a :ref:`build host<buildhosts>` takes its
|
||||
time. Installation in all containers will take more time (time for another cup
|
||||
of coffee).::
|
||||
|
||||
sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost
|
||||
|
||||
To build (live) documentation inside a archlinux_ container::
|
||||
|
||||
sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live
|
||||
...
|
||||
[I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
|
||||
|
||||
To get IP of the container and the port number *live docs* is listening::
|
||||
|
||||
$ sudo ./utils/lxc.sh show suite | grep docs.live
|
||||
...
|
||||
[searxng-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/
|
||||
|
||||
|
||||
.. _lxc.sh help:
|
||||
|
||||
Overview
|
||||
========
|
||||
|
||||
The ``--help`` output of the script is largely self-explanatory:
|
||||
|
||||
.. program-output:: ../utils/lxc.sh --help
|
||||
|
||||
|
||||
.. _lxc-searxng.env:
|
||||
|
||||
SearXNG suite
|
||||
=============
|
||||
|
||||
.. literalinclude:: ../../utils/lxc-searxng.env
|
||||
:language: bash
|