From 1933577c8e0a7635894439e278ccec5e3b284568 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 May 2021 13:20:48 +0200 Subject: [PATCH 1/3] [enh] XPath engine - add ISO 639-1 {lang} replacement to search-URL BTW: remove obsolte params['query'] and not needed paging condition. Signed-off-by: Markus Heiser --- searx/engines/xpath.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 920adf4c6..8c2b2d0a2 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -37,8 +37,15 @@ Search URL of the engine, replacements are: ``{pageno}``: Page number if engine supports pagging :py:obj:`paging` +``{lang}``: + ISO 639-1 language code (en, de, fr ..) """ +lang_all='en' +'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is +selected. +''' + soft_max_redirects = 0 '''Maximum redirects, soft limit. Record an error but don't stop the engine''' @@ -77,14 +84,15 @@ def request(query, params): '''Build request parameters (see :ref:`engine request`). ''' - query = urlencode({'q': query})[2:] - - fargs = {'query': query} - if paging and search_url.find('{pageno}') >= 0: - fargs['pageno'] = (params['pageno'] - 1) * page_size + first_page_num - + lang = lang_all + if params['language'] != 'all': + lang = params['language'][:2] + fargs = { + 'query': urlencode({'q': query})[2:], + 'lang': lang, + 'pageno': (params['pageno'] - 1) * page_size + first_page_num + } params['url'] = search_url.format(**fargs) - params['query'] = query params['soft_max_redirects'] = soft_max_redirects logger.debug("query_url --> %s", params['url']) From 6bfe3fd033fa6f82669ff3907fb55d3e57b432b0 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 May 2021 16:49:30 +0200 Subject: [PATCH 2/3] [enh] XPath engine - add time range support Signed-off-by: Markus Heiser --- searx/engines/xpath.py | 47 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 8c2b2d0a2..41d3e17ff 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -39,6 +39,12 @@ Search URL of the engine, replacements are: ``{lang}``: ISO 639-1 language code (en, de, fr ..) + +``{time_range}``: + :py:obj:`URL parameter ` if engine :py:obj:`supports time + range `. The value for the parameter is taken from + :py:obj:`time_range_map`. + """ lang_all='en' @@ -80,6 +86,37 @@ number, but an offset.''' first_page_num = 1 '''Number of the first page (usually 0 or 1).''' +time_range_support = False +'''Engine supports search time range.''' + +time_range_url = '&hours={time_range_val}' +'''Time range URL parameter in the in :py:obj:`search_url`. If no time range is +requested by the user, the URL paramter is an empty string. The +``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`. + +.. code:: yaml + + time_range_url : '&days={time_range_val}' +''' + +time_range_map = { + 'day': 24, + 'week': 24*7, + 'month': 24*30, + 'year': 24*365, +} +'''Maps time range value from user to ``{time_range_val}`` in +:py:obj:`time_range_url`. + +.. code:: yaml + + time_range_map: + day: 1 + week: 7 + month: 30 + year: 365 +''' + def request(query, params): '''Build request parameters (see :ref:`engine request`). @@ -87,11 +124,19 @@ def request(query, params): lang = lang_all if params['language'] != 'all': lang = params['language'][:2] + + time_range = '' + if params.get('time_range'): + time_range_val = time_range_map.get(params.get('time_range')) + time_range = time_range_url.format(time_range_val=time_range_val) + fargs = { 'query': urlencode({'q': query})[2:], 'lang': lang, - 'pageno': (params['pageno'] - 1) * page_size + first_page_num + 'pageno': (params['pageno'] - 1) * page_size + first_page_num, + 'time_range' : time_range, } + params['url'] = search_url.format(**fargs) params['soft_max_redirects'] = soft_max_redirects logger.debug("query_url --> %s", params['url']) From 84a943f867f6a17e760e849d840e329142fff2e5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 May 2021 22:26:18 +0200 Subject: [PATCH 3/3] [enh] XPath engine - add time safe-search support Signed-off-by: Markus Heiser --- searx/engines/xpath.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 41d3e17ff..e14f982b9 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -29,7 +29,11 @@ logger = logger.getChild('XPath engine') search_url = None """ -Search URL of the engine, replacements are: +Search URL of the engine. Example:: + + https://example.org/?search={query}&page={pageno}{time_range}{safe_search} + +Replacements are: ``{query}``: Search terms from user. @@ -45,6 +49,15 @@ Search URL of the engine, replacements are: range `. The value for the parameter is taken from :py:obj:`time_range_map`. +``{safe_search}``: + Safe-search :py:obj:`URL parameter ` if engine + :py:obj:`supports safe-search `. The ``{safe_search}`` + replacement is taken from the :py:obj:`safes_search_map`. Filter results:: + + 0: none, 1: moderate, 2:strict + + If not supported, the URL paramter is an empty string. + """ lang_all='en' @@ -117,6 +130,26 @@ time_range_map = { year: 365 ''' +safe_search_support = False +'''Engine supports safe-search.''' + +safe_search_map = { + 0: '&filter=none', + 1: '&filter=moderate', + 2: '&filter=strict' +} +'''Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`. + +.. code:: yaml + + safesearch: true + safes_search_map: + 0: '&filter=none' + 1: '&filter=moderate' + 2: '&filter=strict' + +''' + def request(query, params): '''Build request parameters (see :ref:`engine request`). @@ -130,11 +163,16 @@ def request(query, params): time_range_val = time_range_map.get(params.get('time_range')) time_range = time_range_url.format(time_range_val=time_range_val) + safe_search = '' + if params['safesearch']: + safe_search = safe_search_map[params['safesearch']] + fargs = { 'query': urlencode({'q': query})[2:], 'lang': lang, 'pageno': (params['pageno'] - 1) * page_size + first_page_num, 'time_range' : time_range, + 'safe_search' : safe_search, } params['url'] = search_url.format(**fargs)