From b8c98c4c0d716c1e1c3a7f26788a012a41e59dbe Mon Sep 17 00:00:00 2001 From: Allen <64094914+allendema@users.noreply.github.com> Date: Sun, 2 Jan 2022 23:05:49 +0100 Subject: [PATCH 1/3] [enh] Add autocompleter from Brave Raw response example: https://search.brave.com/api/suggest?q=how%20to:%20with%20j Headers are needed in order to get a 200 response, thus Searx user-agent is used. Other URL param could be '&rich=false' or '&rich=true'. Cherry-pick: https://github.com/allendema/searx/commit/71786bf9cb6fbb175a054692e6951e77769aac1b --- searx/autocomplete.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index b8d272c32..6c69ff7c5 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -36,6 +36,20 @@ def get(*args, **kwargs): return http_get(*args, **kwargs) +def brave(query, lang): + # brave search autocompleter + url = 'https://search.brave.com/api/suggest?{query}' + resp = get(url.format(query=urlencode({'q': query}))) + + results = [] + + if resp.ok: + data = loads(resp.text) + for item in data[1]: + results.append(item) + return results + + def dbpedia(query, lang): # dbpedia autocompleter, no HTTPS autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' @@ -128,6 +142,7 @@ backends = { 'swisscows': swisscows, 'qwant': qwant, 'wikipedia': wikipedia, + 'brave': brave, } From 9c5bac4c4392542d9c48d61dd3fc9c96e4f514f0 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Jan 2022 17:25:24 +0100 Subject: [PATCH 2/3] [pylint] searx/autocomplete.py Fix remarks from pylint, BTW set SPDX-License-Identifier. Signed-off-by: Markus Heiser --- searx/autocomplete.py | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 6c69ff7c5..00ec98bbc 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -1,33 +1,23 @@ -''' -searx is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""This module implements functions needed for the autocompleter. -searx is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. +""" -You should have received a copy of the GNU Affero General Public License -along with searx. If not, see < http://www.gnu.org/licenses/ >. - -(C) 2013- by Adam Tauber, -''' - - -from lxml import etree from json import loads from urllib.parse import urlencode +from lxml import etree from httpx import HTTPError - from searx import settings from searx.data import ENGINES_LANGUAGES from searx.network import get as http_get from searx.exceptions import SearxEngineResponseException +# a fetch_supported_languages() for XPath engines isn't available right now +# _brave = ENGINES_LANGUAGES['brave'].keys() + def get(*args, **kwargs): if 'timeout' not in kwargs: @@ -36,10 +26,15 @@ def get(*args, **kwargs): return http_get(*args, **kwargs) -def brave(query, lang): +def brave(query, _lang): # brave search autocompleter - url = 'https://search.brave.com/api/suggest?{query}' - resp = get(url.format(query=urlencode({'q': query}))) + url = 'https://search.brave.com/api/suggest?' + url += urlencode({'q': query}) + country = 'all' + # if lang in _brave: + # country = lang + kwargs = {'cookies': {'country': country}} + resp = get(url, **kwargs) results = [] @@ -50,7 +45,7 @@ def brave(query, lang): return results -def dbpedia(query, lang): +def dbpedia(query, _lang): # dbpedia autocompleter, no HTTPS autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' @@ -65,7 +60,7 @@ def dbpedia(query, lang): return results -def duckduckgo(query, lang): +def duckduckgo(query, _lang): # duckduckgo autocompleter url = 'https://ac.duckduckgo.com/ac/?{0}&type=list' @@ -99,7 +94,7 @@ def startpage(query, lang): return [e['text'] for e in data.get('suggestions', []) if 'text' in e] -def swisscows(query, lang): +def swisscows(query, _lang): # swisscows autocompleter url = 'https://swisscows.ch/api/suggest?{query}&itemsCount=5' From e9588b70a6304afb8ac1de73479d7ade1e01b155 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 Jan 2022 17:22:13 +0100 Subject: [PATCH 3/3] [fix] brave autocompleter: charset_normalizer issues Use httpx.Response.json() to avoid charset_normalizer issues: DEBUG charset_normalizer : override steps (5) and chunk_size (512) as content does not fit (153 byte(s) given) parameters. INFO charset_normalizer : ascii passed initial chaos probing. Mean measured chaos is 0.000000 % DEBUG charset_normalizer : ascii should target any language(s) of ['Latin Based'] INFO charset_normalizer : ascii is most likely the one. Stopping the process. [1] https://www.python-httpx.org/api/#response Signed-off-by: Markus Heiser --- searx/autocomplete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 00ec98bbc..8a67f8139 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -39,7 +39,7 @@ def brave(query, _lang): results = [] if resp.ok: - data = loads(resp.text) + data = resp.json() for item in data[1]: results.append(item) return results