mirror of https://github.com/searxng/searxng
Merge a9d1d1be70
into ac430a9eaf
commit
fb6e212574
@ -1,265 +1,483 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# pylint: disable=missing-module-docstring, global-statement
|
||||
|
||||
import asyncio
|
||||
# lint: pylint
|
||||
# pyright: basic
|
||||
# pylint: disable=redefined-outer-name
|
||||
# ^^ because there is the raise_for_httperror function and the raise_for_httperror parameter.
|
||||
"""HTTP for SearXNG.
|
||||
|
||||
In httpx and similar libraries, a client (also named session) contains a pool of HTTP connections.
|
||||
The client reuses these HTTP connections and automatically recreates them when the server at the other
|
||||
end closes the connections. Whatever the library, each client uses only one proxy (eventually none) and only
|
||||
one local IP address.
|
||||
|
||||
SearXNG's primary use case is an engine sending one (or more) outgoing HTTP request(s). The admin can configure
|
||||
an engine to use multiple proxies and/or IP addresses: SearXNG sends the outgoing HTTP requests through these
|
||||
different proxies/IP addresses ( = HTTP clients ) on a rotational basis.
|
||||
|
||||
In addition, when SearXNG runs an engine request, there is a hard timeout: the engine runtime must not exceed
|
||||
a defined value.
|
||||
|
||||
Moreover, an engine can ask SearXNG to retry a failed HTTP request.
|
||||
|
||||
However, we want to keep the engine codes simple and keep the complexity either in the configuration or the
|
||||
core component components (here, in this module).
|
||||
|
||||
To answer the above requirements, the `searx.network` module introduces three components:
|
||||
* HTTPClient and TorHTTPClient are two classes that wrap one or multiple httpx.Client
|
||||
* NetworkManager, a set of named Network. Each Network
|
||||
* holds the configuration defined in settings.yml
|
||||
* creates NetworkContext fed with an HTTPClient (or TorHTTPClient).
|
||||
This is where the rotation between the proxies and IP addresses happens.
|
||||
* NetworkContext to provide a runtime context for the engines. The constructor needs a global timeout
|
||||
and an HTTPClient factory. NetworkContext is an abstract class with three implementations,
|
||||
one for each retry policy.
|
||||
|
||||
It is only possible to send an HTTP request with a NetworkContext
|
||||
(otherwise, SearXNG raises a NetworkContextNotFound exception).
|
||||
Two helpers set a NetworkContext for the current thread:
|
||||
|
||||
* The decorator `@networkcontext_decorator`, the intended usage is an external script (see searxng_extra)
|
||||
* The context manager `networkcontext_manager`, for the generic use case.
|
||||
|
||||
Inside the thread, the caller can use `searx.network.get`, `searx.network.post` and similar functions without
|
||||
caring about the HTTP client. However, if the caller creates a new thread, it must initialize a new NetworkContext.
|
||||
A NetworkContext is most probably thread-safe, but this has not been tested.
|
||||
|
||||
The overall architecture:
|
||||
* searx.network.network.NETWORKS contains all the networks.
|
||||
The method `NetworkManager.get(network_name)` returns an initialized Network.
|
||||
* searx.network.network.Network defines a network (a set of proxies, local IP address, etc...).
|
||||
They are defined in settings.yml.
|
||||
The method `Network.get_context()` creates a new NetworkContext.
|
||||
* searx.network.context contains three different implementations of NetworkContext. One for each retry policy.
|
||||
* searx.network.client.HTTPClient and searx.network.client.TorHTTPClient implement wrappers around httpx.Client.
|
||||
"""
|
||||
import threading
|
||||
import concurrent.futures
|
||||
from queue import SimpleQueue
|
||||
from types import MethodType
|
||||
from timeit import default_timer
|
||||
from typing import Iterable, NamedTuple, Tuple, List, Dict, Union
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Optional, Union
|
||||
|
||||
import httpx
|
||||
import anyio
|
||||
|
||||
from .network import get_network, initialize, check_network_configuration # pylint:disable=cyclic-import
|
||||
from .client import get_loop
|
||||
from .raise_for_httperror import raise_for_httperror
|
||||
|
||||
|
||||
THREADLOCAL = threading.local()
|
||||
"""Thread-local data is data for thread specific values."""
|
||||
|
||||
|
||||
def reset_time_for_thread():
|
||||
THREADLOCAL.total_time = 0
|
||||
|
||||
from searx.network.client import NOTSET, _NotSetClass
|
||||
from searx.network.context import NetworkContext, P, R
|
||||
from searx.network.network import NETWORKS
|
||||
from searx.network.raise_for_httperror import raise_for_httperror
|
||||
|
||||
def get_time_for_thread():
|
||||
"""returns thread's total time or None"""
|
||||
return THREADLOCAL.__dict__.get('total_time')
|
||||
__all__ = [
|
||||
"NETWORKS",
|
||||
"NetworkContextNotFound",
|
||||
"networkcontext_manager",
|
||||
"networkcontext_decorator",
|
||||
"raise_for_httperror",
|
||||
"request",
|
||||
"get",
|
||||
"options",
|
||||
"head",
|
||||
"post",
|
||||
"put",
|
||||
"patch",
|
||||
"delete",
|
||||
]
|
||||
|
||||
|
||||
def set_timeout_for_thread(timeout, start_time=None):
|
||||
THREADLOCAL.timeout = timeout
|
||||
THREADLOCAL.start_time = start_time
|
||||
_THREADLOCAL = threading.local()
|
||||
"""Thread-local that contains only one field: network_context."""
|
||||
|
||||
_NETWORK_CONTEXT_KEY = 'network_context'
|
||||
"""Key to access _THREADLOCAL"""
|
||||
|
||||
def set_context_network_name(network_name):
|
||||
THREADLOCAL.network = get_network(network_name)
|
||||
DEFAULT_MAX_REDIRECTS = httpx._config.DEFAULT_MAX_REDIRECTS # pylint: disable=protected-access
|
||||
|
||||
|
||||
def get_context_network():
|
||||
"""If set return thread's network.
|
||||
class NetworkContextNotFound(Exception):
|
||||
"""A NetworkContext is expected to exist for the current thread.
|
||||
|
||||
If unset, return value from :py:obj:`get_network`.
|
||||
Use searx.network.networkcontext_manager or searx.network.networkcontext_decorator
|
||||
to set a NetworkContext
|
||||
"""
|
||||
return THREADLOCAL.__dict__.get('network') or get_network()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _record_http_time():
|
||||
# pylint: disable=too-many-branches
|
||||
time_before_request = default_timer()
|
||||
start_time = getattr(THREADLOCAL, 'start_time', time_before_request)
|
||||
try:
|
||||
yield start_time
|
||||
finally:
|
||||
# update total_time.
|
||||
# See get_time_for_thread() and reset_time_for_thread()
|
||||
if hasattr(THREADLOCAL, 'total_time'):
|
||||
time_after_request = default_timer()
|
||||
THREADLOCAL.total_time += time_after_request - time_before_request
|
||||
|
||||
|
||||
def _get_timeout(start_time, kwargs):
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
# timeout (httpx)
|
||||
if 'timeout' in kwargs:
|
||||
timeout = kwargs['timeout']
|
||||
else:
|
||||
timeout = getattr(THREADLOCAL, 'timeout', None)
|
||||
if timeout is not None:
|
||||
kwargs['timeout'] = timeout
|
||||
|
||||
# 2 minutes timeout for the requests without timeout
|
||||
timeout = timeout or 120
|
||||
|
||||
# adjust actual timeout
|
||||
timeout += 0.2 # overhead
|
||||
if start_time:
|
||||
timeout -= default_timer() - start_time
|
||||
|
||||
return timeout
|
||||
|
||||
|
||||
def request(method, url, **kwargs):
|
||||
"""same as requests/requests/api.py request(...)"""
|
||||
with _record_http_time() as start_time:
|
||||
network = get_context_network()
|
||||
timeout = _get_timeout(start_time, kwargs)
|
||||
future = asyncio.run_coroutine_threadsafe(network.request(method, url, **kwargs), get_loop())
|
||||
try:
|
||||
return future.result(timeout)
|
||||
except concurrent.futures.TimeoutError as e:
|
||||
raise httpx.TimeoutException('Timeout', request=None) from e
|
||||
|
||||
|
||||
def multi_requests(request_list: List["Request"]) -> List[Union[httpx.Response, Exception]]:
|
||||
"""send multiple HTTP requests in parallel. Wait for all requests to finish."""
|
||||
with _record_http_time() as start_time:
|
||||
# send the requests
|
||||
network = get_context_network()
|
||||
loop = get_loop()
|
||||
future_list = []
|
||||
for request_desc in request_list:
|
||||
timeout = _get_timeout(start_time, request_desc.kwargs)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
network.request(request_desc.method, request_desc.url, **request_desc.kwargs), loop
|
||||
)
|
||||
future_list.append((future, timeout))
|
||||
|
||||
# read the responses
|
||||
responses = []
|
||||
for future, timeout in future_list:
|
||||
try:
|
||||
responses.append(future.result(timeout))
|
||||
except concurrent.futures.TimeoutError:
|
||||
responses.append(httpx.TimeoutException('Timeout', request=None))
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
responses.append(e)
|
||||
return responses
|
||||
|
||||
|
||||
class Request(NamedTuple):
|
||||
"""Request description for the multi_requests function"""
|
||||
|
||||
method: str
|
||||
url: str
|
||||
kwargs: Dict[str, str] = {}
|
||||
|
||||
@staticmethod
|
||||
def get(url, **kwargs):
|
||||
return Request('GET', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def options(url, **kwargs):
|
||||
return Request('OPTIONS', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def head(url, **kwargs):
|
||||
return Request('HEAD', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def post(url, **kwargs):
|
||||
return Request('POST', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def put(url, **kwargs):
|
||||
return Request('PUT', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def patch(url, **kwargs):
|
||||
return Request('PATCH', url, kwargs)
|
||||
|
||||
@staticmethod
|
||||
def delete(url, **kwargs):
|
||||
return Request('DELETE', url, kwargs)
|
||||
|
||||
|
||||
def get(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return request('get', url, **kwargs)
|
||||
|
||||
|
||||
def options(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return request('options', url, **kwargs)
|
||||
|
||||
|
||||
def head(url, **kwargs):
|
||||
kwargs.setdefault('allow_redirects', False)
|
||||
return request('head', url, **kwargs)
|
||||
|
||||
|
||||
def post(url, data=None, **kwargs):
|
||||
return request('post', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def put(url, data=None, **kwargs):
|
||||
return request('put', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def patch(url, data=None, **kwargs):
|
||||
return request('patch', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def delete(url, **kwargs):
|
||||
return request('delete', url, **kwargs)
|
||||
|
||||
|
||||
async def stream_chunk_to_queue(network, queue, method, url, **kwargs):
|
||||
def networkcontext_manager(
|
||||
network_name: Optional[str] = None, timeout: Optional[float] = None, start_time: Optional[float] = None
|
||||
):
|
||||
"""Context manager to set a NetworkContext for the current thread
|
||||
|
||||
The timeout is for the whole function and is infinite by default (None).
|
||||
The timeout is counted from the current time or start_time if different from None.
|
||||
|
||||
Example of usage:
|
||||
|
||||
```python
|
||||
from time import sleep
|
||||
from searx.network import networkcontext_manager, get
|
||||
|
||||
def search(query):
|
||||
# the timeout is automatically set to 2.0 seconds (the remaining time for the NetworkContext)
|
||||
# 2.0 because the timeout for the NetworkContext is 3.0 and one second has elllapsed with sleep(1.0)
|
||||
auckland_time = get("http://worldtimeapi.org/api/timezone/Pacific/Auckland").json()
|
||||
# the timeout is automatically set to 2.0 - (runtime of the previous HTTP request)
|
||||
ip_time = get("http://worldtimeapi.org/api/ip").json()
|
||||
return auckland_time, ip_time
|
||||
|
||||
# "worldtimeapi" is network defined in settings.yml
|
||||
# network_context.call might call multiple times the search function,
|
||||
# however the timeout will be respected.
|
||||
with networkcontext_manager('worldtimeapi', timeout=3.0) as network_context:
|
||||
sleep(1.0)
|
||||
auckland_time, ip_time = network_context.call(search(query))
|
||||
print("Auckland time: ", auckland_time["datetime"])
|
||||
print("My time: ", ip_time["datetime"])
|
||||
print("HTTP runtime:", network_context.get_http_runtime())
|
||||
```
|
||||
"""
|
||||
network = NETWORKS.get(network_name)
|
||||
network_context = network.get_context(timeout=timeout, start_time=start_time)
|
||||
setattr(_THREADLOCAL, _NETWORK_CONTEXT_KEY, network_context)
|
||||
try:
|
||||
async with await network.stream(method, url, **kwargs) as response:
|
||||
queue.put(response)
|
||||
# aiter_raw: access the raw bytes on the response without applying any HTTP content decoding
|
||||
# https://www.python-httpx.org/quickstart/#streaming-responses
|
||||
async for chunk in response.aiter_raw(65536):
|
||||
if len(chunk) > 0:
|
||||
queue.put(chunk)
|
||||
except (httpx.StreamClosed, anyio.ClosedResourceError):
|
||||
# the response was queued before the exception.
|
||||
# the exception was raised on aiter_raw.
|
||||
# we do nothing here: in the finally block, None will be queued
|
||||
# so stream(method, url, **kwargs) generator can stop
|
||||
pass
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
# broad except to avoid this scenario:
|
||||
# exception in network.stream(method, url, **kwargs)
|
||||
# -> the exception is not catch here
|
||||
# -> queue None (in finally)
|
||||
# -> the function below steam(method, url, **kwargs) has nothing to return
|
||||
queue.put(e)
|
||||
yield network_context
|
||||
finally:
|
||||
queue.put(None)
|
||||
|
||||
delattr(_THREADLOCAL, _NETWORK_CONTEXT_KEY)
|
||||
del network_context
|
||||
|
||||
def _stream_generator(method, url, **kwargs):
|
||||
queue = SimpleQueue()
|
||||
network = get_context_network()
|
||||
future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(network, queue, method, url, **kwargs), get_loop())
|
||||
|
||||
# yield chunks
|
||||
obj_or_exception = queue.get()
|
||||
while obj_or_exception is not None:
|
||||
if isinstance(obj_or_exception, Exception):
|
||||
raise obj_or_exception
|
||||
yield obj_or_exception
|
||||
obj_or_exception = queue.get()
|
||||
future.result()
|
||||
def networkcontext_decorator(
|
||||
network_name: Optional[str] = None, timeout: Optional[float] = None, start_time: Optional[float] = None
|
||||
):
|
||||
"""Set the NetworkContext, then call the wrapped function using searx.network.context.NetworkContext.call
|
||||
|
||||
The timeout is for the whole function and is infinite by default (None).
|
||||
The timeout is counted from the current time or start_time if different from None
|
||||
|
||||
def _close_response_method(self):
|
||||
asyncio.run_coroutine_threadsafe(self.aclose(), get_loop())
|
||||
# reach the end of _self.generator ( _stream_generator ) to an avoid memory leak.
|
||||
# it makes sure that :
|
||||
# * the httpx response is closed (see the stream_chunk_to_queue function)
|
||||
# * to call future.result() in _stream_generator
|
||||
for _ in self._generator: # pylint: disable=protected-access
|
||||
continue
|
||||
Intended usage: to provide a NetworkContext for scripts in searxng_extra.
|
||||
|
||||
Example of usage:
|
||||
|
||||
def stream(method, url, **kwargs) -> Tuple[httpx.Response, Iterable[bytes]]:
|
||||
"""Replace httpx.stream.
|
||||
```python
|
||||
from time import sleep
|
||||
from searx import network
|
||||
|
||||
Usage:
|
||||
response, stream = poolrequests.stream(...)
|
||||
for chunk in stream:
|
||||
...
|
||||
@network.networkcontext_decorator(timeout=3.0)
|
||||
def main()
|
||||
sleep(1.0)
|
||||
# the timeout is automatically set to 2.0 (the remaining time for the NetworkContext).
|
||||
my_ip = network.get("https://ifconfig.me/ip").text
|
||||
print(my_ip)
|
||||
|
||||
httpx.Client.stream requires to write the httpx.HTTPTransport version of the
|
||||
the httpx.AsyncHTTPTransport declared above.
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
```
|
||||
"""
|
||||
generator = _stream_generator(method, url, **kwargs)
|
||||
|
||||
# yield response
|
||||
response = next(generator) # pylint: disable=stop-iteration-return
|
||||
if isinstance(response, Exception):
|
||||
raise response
|
||||
|
||||
response._generator = generator # pylint: disable=protected-access
|
||||
response.close = MethodType(_close_response_method, response)
|
||||
|
||||
return response, generator
|
||||
def func_outer(func: Callable[P, R]):
|
||||
@wraps(func)
|
||||
def func_inner(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
with networkcontext_manager(network_name, timeout, start_time) as network_context:
|
||||
return network_context.call(func, *args, **kwargs)
|
||||
|
||||
return func_inner
|
||||
|
||||
return func_outer
|
||||
|
||||
|
||||
def request(
|
||||
method: str,
|
||||
url: str,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
content: Optional[httpx._types.RequestContent] = None,
|
||||
data: Optional[httpx._types.RequestData] = None,
|
||||
files: Optional[httpx._types.RequestFiles] = None,
|
||||
json: Optional[Any] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.request ( https://www.python-httpx.org/api/ ) with some differences:
|
||||
|
||||
* proxies:
|
||||
it is not available and has to be defined in the Network configuration (in settings.yml)
|
||||
* cert:
|
||||
it is not available and is always None.
|
||||
* trust_env:
|
||||
it is not available and is always True.
|
||||
* timeout:
|
||||
the implementation uses the lowest timeout between this parameter and remaining time for the NetworkContext.
|
||||
* allow_redirects:
|
||||
it replaces the follow_redirects parameter to be compatible with the requests API.
|
||||
* raise_for_httperror:
|
||||
when True, this function calls searx.network.raise_for_httperror.raise_for_httperror.
|
||||
|
||||
Some parameters from httpx.Client ( https://www.python-httpx.org/api/#client) are available:
|
||||
|
||||
* max_redirects:
|
||||
Set to None to use the value from the Network configuration.
|
||||
The maximum number of redirect responses that should be followed.
|
||||
* verify:
|
||||
Set to None to use the value from the Network configuration.
|
||||
* limits:
|
||||
it has to be defined in the Network configuration (in settings.yml)
|
||||
* default_encoding:
|
||||
this parameter is not available and is always "utf-8".
|
||||
|
||||
This function requires a NetworkContext provided by either networkcontext_decorator or networkcontext_manager.
|
||||
|
||||
The implementation uses one or more httpx.Client
|
||||
"""
|
||||
# pylint: disable=too-many-arguments
|
||||
network_context: Optional[NetworkContext] = getattr(_THREADLOCAL, _NETWORK_CONTEXT_KEY, None)
|
||||
if network_context is None:
|
||||
raise NetworkContextNotFound()
|
||||
http_client = network_context._get_http_client() # pylint: disable=protected-access
|
||||
return http_client.request(
|
||||
method,
|
||||
url,
|
||||
params=params,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
timeout=timeout,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def get(
|
||||
url: str,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = True,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.get, see the request method for the details.
|
||||
|
||||
allow_redirects is by default True (httpx default value is False).
|
||||
"""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"GET",
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def options(
|
||||
url: str,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.options, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"OPTIONS",
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def head(
|
||||
url: str,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.head, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"HEAD",
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def post(
|
||||
url: str,
|
||||
content: Optional[httpx._types.RequestContent] = None,
|
||||
data: Optional[httpx._types.RequestData] = None,
|
||||
files: Optional[httpx._types.RequestFiles] = None,
|
||||
json: Optional[Any] = None,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.post, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"POST",
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def put(
|
||||
url: str,
|
||||
content: Optional[httpx._types.RequestContent] = None,
|
||||
data: Optional[httpx._types.RequestData] = None,
|
||||
files: Optional[httpx._types.RequestFiles] = None,
|
||||
json: Optional[Any] = None,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.put, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"PUT",
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def patch(
|
||||
url: str,
|
||||
content: Optional[httpx._types.RequestContent] = None,
|
||||
data: Optional[httpx._types.RequestData] = None,
|
||||
files: Optional[httpx._types.RequestFiles] = None,
|
||||
json: Optional[Any] = None,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.patch, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"PATCH",
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
||||
|
||||
def delete(
|
||||
url: str,
|
||||
params: Optional[httpx._types.QueryParamTypes] = None,
|
||||
headers: Optional[httpx._types.HeaderTypes] = None,
|
||||
cookies: Optional[httpx._types.CookieTypes] = None,
|
||||
auth: Optional[httpx._types.AuthTypes] = None,
|
||||
allow_redirects: bool = False,
|
||||
max_redirects: Union[_NotSetClass, int] = NOTSET,
|
||||
verify: Union[_NotSetClass, httpx._types.VerifyTypes] = NOTSET,
|
||||
timeout: httpx._types.TimeoutTypes = None,
|
||||
raise_for_httperror: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""Similar to httpx.delete, see the request method for the details."""
|
||||
# pylint: disable=too-many-arguments
|
||||
return request(
|
||||
"DELETE",
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
allow_redirects=allow_redirects,
|
||||
max_redirects=max_redirects,
|
||||
verify=verify,
|
||||
timeout=timeout,
|
||||
raise_for_httperror=raise_for_httperror,
|
||||
)
|
||||
|
@ -0,0 +1,418 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
# pyright: basic
|
||||
"""This module implements various NetworkContext which deals with
|
||||
|
||||
* retry strategies: what to do when an HTTP request fails and retries>0
|
||||
* record HTTP runtime
|
||||
* timeout: In engines, the user query starts at one point in time,
|
||||
the engine timeout is the starting point plus a defined value.
|
||||
NetworkContext sends HTTP requests following the request timeout and the engine timeouts.
|
||||
|
||||
Example of usage:
|
||||
|
||||
```
|
||||
context = NetworkContextRetryFunction(...) # or another implementation
|
||||
|
||||
def my_engine():
|
||||
http_client = context.get_http_client()
|
||||
ip_ifconfig = http_client.request("GET", "https://ifconfig.me/")
|
||||
print("ip from ifconfig.me ", ip_ifconfig)
|
||||
ip_myip = http_client.request("GET", "https://api.myip.com").json()["ip"]
|
||||
print("ip from api.myip.com", ip_myip)
|
||||
assert ip_ifconfig == ip_myip
|
||||
# ^^ always true with NetworkContextRetryFunction and NetworkContextRetrySameHTTPClient
|
||||
|
||||
result = context.call(my_engine)
|
||||
print('HTTP runtime:', context.get_total_time())
|
||||
```
|
||||
|
||||
Note in the code above NetworkContextRetryFunction is instanced directly for the sake of simplicity.
|
||||
NetworkContext are actually instanciated using Network.get_context(...)
|
||||
|
||||
Various implementations define what to do when there is an exception in the function `my_engine`:
|
||||
|
||||
* `NetworkContextRetryFunction` gets another HTTP client and tries the whole function again.
|
||||
* `NetworkContextRetryDifferentHTTPClient` gets another HTTP client and tries the query again.
|
||||
* `NetworkContextRetrySameHTTPClient` tries the query again with the same HTTP client.
|
||||
"""
|
||||
import functools
|
||||
import ssl
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from timeit import default_timer
|
||||
from typing import Callable, Optional, final
|
||||
|
||||
try:
|
||||
from typing import ParamSpec, TypeVar
|
||||
except ImportError:
|
||||
# to support Python < 3.10
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import httpx
|
||||
|
||||
from searx.network.client import ABCHTTPClient, SoftRetryHTTPException
|
||||
|
||||
P = ParamSpec('P')
|
||||
R = TypeVar('R')
|
||||
HTTPCLIENTFACTORY = Callable[[], ABCHTTPClient]
|
||||
|
||||
DEFAULT_TIMEOUT = 120.0
|
||||
|
||||
|
||||
## NetworkContext
|
||||
|
||||
|
||||
class NetworkContext(ABC):
|
||||
"""Abstract implementation: the call must defined in concrete classes.
|
||||
|
||||
Lifetime: one engine request or initialization of an engine.
|
||||
"""
|
||||
|
||||
__slots__ = ('_retries', '_http_client_factory', '_http_client', '_start_time', '_http_time', '_timeout')
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
retries: int,
|
||||
http_client_factory: HTTPCLIENTFACTORY,
|
||||
start_time: Optional[float],
|
||||
timeout: Optional[float],
|
||||
):
|
||||
self._retries: int = retries
|
||||
# wrap http_client_factory here, so we can forget about this wrapping
|
||||
self._http_client_factory = _TimeHTTPClientWrapper.wrap_factory(http_client_factory, self)
|
||||
self._http_client: Optional[ABCHTTPClient] = None
|
||||
self._start_time: float = start_time or default_timer()
|
||||
self._http_time: float = 0.0
|
||||
self._timeout: Optional[float] = timeout
|
||||
|
||||
@abstractmethod
|
||||
def call(self, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R:
|
||||
"""Call func within the network context.
|
||||
|
||||
The retry policy might call func multiple times.
|
||||
|
||||
Within the function self.get_http_client() returns an HTTP client to use.
|
||||
|
||||
The retry policy might send multiple times the same HTTP request
|
||||
until it works or the retry count falls to zero.
|
||||
"""
|
||||
|
||||
@final
|
||||
def request(self, *args, **kwargs):
|
||||
"""Convenient method to wrap a call to request inside the call method.
|
||||
|
||||
Use a new HTTP client to wrap a call to the request method using self.call
|
||||
"""
|
||||
|
||||
def local_request(*args, **kwargs):
|
||||
return self._get_http_client().request(*args, **kwargs)
|
||||
|
||||
return self.call(local_request, *args, **kwargs)
|
||||
|
||||
@final
|
||||
def stream(self, *args, **kwargs):
|
||||
"""Convenient method to wrap a call to stream inside the call method.
|
||||
|
||||
Use a new HTTP client to wrap a call to the stream method using self.call
|
||||
"""
|
||||
|
||||
def local_stream(*args, **kwargs):
|
||||
return self._get_http_client().stream(*args, **kwargs)
|
||||
|
||||
return self.call(local_stream, *args, **kwargs)
|
||||
|
||||
@final
|
||||
def get_http_runtime(self) -> Optional[float]:
|
||||
"""Return the amount of time spent on HTTP requests"""
|
||||
return self._http_time
|
||||
|
||||
@final
|
||||
def get_remaining_time(self, _override_timeout: Optional[float] = None) -> float:
|
||||
"""Return the remaining time for the context.
|
||||
|
||||
_override_timeout is not intended to be used outside this module.
|
||||
"""
|
||||
timeout = _override_timeout or self._timeout or DEFAULT_TIMEOUT
|
||||
timeout += 0.2 # overhead
|
||||
timeout -= default_timer() - self._start_time
|
||||
return timeout
|
||||
|
||||
@final
|
||||
def _get_http_client(self) -> ABCHTTPClient:
|
||||
"""Return the HTTP client to use for this context."""
|
||||
if self._http_client is None:
|
||||
raise ValueError("HTTP client has not been set")
|
||||
return self._http_client
|
||||
|
||||
@final
|
||||
def _set_http_client(self):
|
||||
"""Ask the NetworkContext to use another HTTP client using the factory.
|
||||
|
||||
Use the method _get_new_client_from_factory() to call the factory,
|
||||
so the NetworkContext implementations can wrap the ABCHTTPClient.
|
||||
"""
|
||||
self._http_client = self._get_new_client_from_factory()
|
||||
|
||||
@final
|
||||
def _reset_http_client(self):
|
||||
self._http_client = None
|
||||
|
||||
def _get_new_client_from_factory(self):
|
||||
return self._http_client_factory()
|
||||
|
||||
@contextmanager
|
||||
def _record_http_time(self):
|
||||
"""This decorator records the code's runtime and adds it to self.total_time"""
|
||||
time_before_request = default_timer()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._http_time += default_timer() - time_before_request
|
||||
|
||||
def __repr__(self):
|
||||
common_attributes = (
|
||||
f"{self.__class__.__name__}"
|
||||
+ f" retries={self._retries!r} timeout={self._timeout!r} http_client={self._http_client!r}"
|
||||
)
|
||||
# get the original factory : see the __init__ method of this class and _TimeHTTPClientWrapper.wrap_factory
|
||||
factory = self._http_client_factory.__wrapped__
|
||||
# break the abstraction safely: get back the Network object through the bound method
|
||||
# see Network.get_context
|
||||
bound_instance = getattr(factory, "__self__", None)
|
||||
if bound_instance is not None and hasattr(bound_instance, 'get_context'):
|
||||
# bound_instance has a "get_context" attribute: this is most likely a Network
|
||||
# searx.network.network.Network is not imported to avoid circular import
|
||||
return f"<{common_attributes} network_context={factory.__self__!r}>"
|
||||
# fallback : this instance was not created using Network.get_context
|
||||
return f"<{common_attributes} http_client_factory={factory!r}>"
|
||||
|
||||
|
||||
## Measure time and deal with timeout
|
||||
|
||||
|
||||
class _TimeHTTPClientWrapper(ABCHTTPClient):
|
||||
"""Wrap an ABCHTTPClient:
|
||||
* to record the HTTP runtime
|
||||
* to override the timeout to make sure the total time does not exceed the timeout set on the NetworkContext
|
||||
"""
|
||||
|
||||
__slots__ = ('http_client', 'network_context')
|
||||
|
||||
@staticmethod
|
||||
def wrap_factory(http_client_factory: HTTPCLIENTFACTORY, network_context: NetworkContext):
|
||||
"""Return a factory which wraps the result of http_client_factory with _TimeHTTPClientWrapper instance."""
|
||||
functools.wraps(http_client_factory)
|
||||
|
||||
def wrapped_factory():
|
||||
return _TimeHTTPClientWrapper(http_client_factory(), network_context)
|
||||
|
||||
wrapped_factory.__wrapped__ = http_client_factory
|
||||
return wrapped_factory
|
||||
|
||||
def __init__(self, http_client: ABCHTTPClient, network_context: NetworkContext) -> None:
|
||||
self.http_client = http_client
|
||||
self.network_context = network_context
|
||||
|
||||
def send(self, stream, method, url, **kwargs) -> httpx.Response:
|
||||
"""Send the HTTP request using self.http_client
|
||||
|
||||
Inaccurate with stream: the method must record HTTP time with the close method of httpx.Response.
|
||||
It is not a problem since stream are used only for the image proxy.
|
||||
"""
|
||||
with self.network_context._record_http_time(): # pylint: disable=protected-access
|
||||
timeout = self._extract_timeout(kwargs)
|
||||
return self.http_client.send(stream, method, url, timeout=timeout, **kwargs)
|
||||
|
||||
def close(self):
|
||||
return self.http_client.close()
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
return self.http_client.is_closed
|
||||
|
||||
def _extract_timeout(self, kwargs):
|
||||
"""Extract the timeout parameter and adjust it according to the remaining time"""
|
||||
timeout = kwargs.pop('timeout', None)
|
||||
return self.network_context.get_remaining_time(timeout)
|
||||
|
||||
|
||||
## NetworkContextRetryFunction
|
||||
|
||||
|
||||
class NetworkContextRetryFunction(NetworkContext):
|
||||
"""When an HTTP request fails, this NetworkContext tries again
|
||||
the whole function with another HTTP client.
|
||||
|
||||
This guarantees that func has the same HTTP client all along.
|
||||
"""
|
||||
|
||||
def call(self, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R:
|
||||
try:
|
||||
# if retries == 1, this method can call `func` twice,
|
||||
# so the exit condition is self._retries must be equal or above zero
|
||||
# to allow two iteration
|
||||
while self._retries >= 0 and self.get_remaining_time() > 0:
|
||||
self._set_http_client()
|
||||
try:
|
||||
return func(*args, **kwargs) # type: ignore
|
||||
except SoftRetryHTTPException as e:
|
||||
if self._retries <= 0:
|
||||
return e.response
|
||||
if e.response:
|
||||
e.response.close()
|
||||
except (ssl.SSLError, httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
if self._retries <= 1:
|
||||
# raise the exception only there is no more try
|
||||
raise e
|
||||
self._retries -= 1
|
||||
if self.get_remaining_time() <= 0:
|
||||
raise httpx.TimeoutException("Timeout")
|
||||
raise httpx.HTTPError("Internal error: this should not happen")
|
||||
finally:
|
||||
self._reset_http_client()
|
||||
|
||||
def _get_new_client_from_factory(self):
|
||||
return _RetryFunctionHTTPClient(super()._get_new_client_from_factory(), self)
|
||||
|
||||
|
||||
class _RetryFunctionHTTPClient(ABCHTTPClient):
|
||||
"""Companion class of NetworkContextRetryFunction
|
||||
|
||||
Do one thing: if the retries count of the NetworkContext is zero and there is a SoftRetryHTTPException,
|
||||
then the send method catch this exception and returns the HTTP response.
|
||||
This make sure the SoftRetryHTTPException exception is not seen outside the searx.network module.
|
||||
"""
|
||||
|
||||
def __init__(self, http_client: ABCHTTPClient, network_context: NetworkContextRetryFunction):
|
||||
self.http_client = http_client
|
||||
self.network_context = network_context
|
||||
|
||||
def send(self, stream: bool, method: str, url: str, **kwargs) -> httpx.Response:
|
||||
try:
|
||||
return self.http_client.send(stream, method, url, **kwargs)
|
||||
except SoftRetryHTTPException as e:
|
||||
if self.network_context._retries <= 0: # pylint: disable=protected-access
|
||||
return e.response
|
||||
if e.response:
|
||||
e.response.close()
|
||||
raise e
|
||||
|
||||
def close(self):
|
||||
return self.http_client.close()
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
return self.http_client.is_closed
|
||||
|
||||
|
||||
## NetworkContextRetrySameHTTPClient
|
||||
|
||||
|
||||
class NetworkContextRetrySameHTTPClient(NetworkContext):
|
||||
"""When an HTTP request fails, this NetworkContext tries again
|
||||
the same HTTP request with the same HTTP client
|
||||
|
||||
The implementation wraps the provided ABCHTTPClient with _RetrySameHTTPClient."""
|
||||
|
||||
def call(self, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R:
|
||||
try:
|
||||
self._set_http_client()
|
||||
return func(*args, **kwargs) # type: ignore
|
||||
finally:
|
||||
self._reset_http_client()
|
||||
|
||||
def _get_new_client_from_factory(self):
|
||||
return _RetrySameHTTPClient(super()._get_new_client_from_factory(), self)
|
||||
|
||||
|
||||
class _RetrySameHTTPClient(ABCHTTPClient):
|
||||
"""Companion class of NetworkContextRetrySameHTTPClient"""
|
||||
|
||||
def __init__(self, http_client: ABCHTTPClient, network_content: NetworkContextRetrySameHTTPClient):
|
||||
self.http_client = http_client
|
||||
self.network_context = network_content
|
||||
|
||||
def send(self, stream: bool, method: str, url: str, **kwargs) -> httpx.Response:
|
||||
retries = self.network_context._retries # pylint: disable=protected-access
|
||||
# if retries == 1, this method can send two HTTP requets,
|
||||
# so the exit condition is self._retries must be equal or above zero
|
||||
# to allow two iteration
|
||||
while retries >= 0 and self.network_context.get_remaining_time() > 0:
|
||||
try:
|
||||
return self.http_client.send(stream, method, url, **kwargs)
|
||||
except SoftRetryHTTPException as e:
|
||||
if retries <= 0:
|
||||
return e.response
|
||||
if e.response:
|
||||
e.response.close()
|
||||
except (ssl.SSLError, httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
if retries <= 0:
|
||||
raise e
|
||||
retries -= 1
|
||||
if self.network_context.get_remaining_time() <= 0:
|
||||
raise httpx.TimeoutException("Timeout")
|
||||
raise httpx.HTTPError("Internal error: this should not happen")
|
||||
|
||||
def close(self):
|
||||
return self.http_client.close()
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
return self.http_client.is_closed
|
||||
|
||||
|
||||
## NetworkContextRetryDifferentHTTPClient
|
||||
|
||||
|
||||
class NetworkContextRetryDifferentHTTPClient(NetworkContext):
|
||||
"""When a HTTP request fails, this NetworkContext tries again
|
||||
the same HTTP request with a different HTTP client
|
||||
|
||||
The implementation wraps the provided ABCHTTPClient with _RetryDifferentHTTPClient."""
|
||||
|
||||
def call(self, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R:
|
||||
self._set_http_client()
|
||||
try:
|
||||
return func(*args, **kwargs) # type: ignore
|
||||
finally:
|
||||
self._reset_http_client()
|
||||
|
||||
def _get_new_client_from_factory(self):
|
||||
return _RetryDifferentHTTPClient(self)
|
||||
|
||||
|
||||
class _RetryDifferentHTTPClient(ABCHTTPClient):
|
||||
"""Companion class of NetworkContextRetryDifferentHTTPClient"""
|
||||
|
||||
def __init__(self, network_context: NetworkContextRetryDifferentHTTPClient) -> None:
|
||||
self.network_context = network_context
|
||||
|
||||
def send(self, stream: bool, method: str, url: str, **kwargs) -> httpx.Response:
|
||||
retries = self.network_context._retries # pylint: disable=protected-access
|
||||
# if retries == 1, this method can send two HTTP requets,
|
||||
# so the exit condition is self._retries must be equal or above zero
|
||||
# to allow two iteration
|
||||
while retries >= 0 and self.network_context.get_remaining_time() > 0:
|
||||
http_client = self.network_context._http_client_factory() # pylint: disable=protected-access
|
||||
try:
|
||||
return http_client.send(stream, method, url, **kwargs)
|
||||
except SoftRetryHTTPException as e:
|
||||
if retries <= 0:
|
||||
return e.response
|
||||
if e.response:
|
||||
e.response.close()
|
||||
except (ssl.SSLError, httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
if retries <= 0:
|
||||
raise e
|
||||
retries -= 1
|
||||
if self.network_context.get_remaining_time() <= 0:
|
||||
raise httpx.TimeoutException("Timeout")
|
||||
raise httpx.HTTPError("Internal error: this should not happen")
|
||||
|
||||
def close(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
raise NotImplementedError()
|
@ -0,0 +1,97 @@
|
||||
server:
|
||||
secret_key: "user_settings_secret"
|
||||
outgoing:
|
||||
networks:
|
||||
http00:
|
||||
enable_http: false
|
||||
enable_http2: false
|
||||
http10:
|
||||
enable_http: true
|
||||
enable_http2: false
|
||||
http01:
|
||||
enable_http: false
|
||||
enable_http2: true
|
||||
http11:
|
||||
enable_http: true
|
||||
enable_http2: true
|
||||
sock5h:
|
||||
proxies: socks5h://127.0.0.1:4000
|
||||
sock5:
|
||||
proxies: socks5://127.0.0.1:4000
|
||||
sock4:
|
||||
proxies: socks4://127.0.0.1:4000
|
||||
engines:
|
||||
#
|
||||
- name: enginea
|
||||
shortcut: a
|
||||
engine: dummy
|
||||
network:
|
||||
proxies: http://127.0.0.1:8000
|
||||
- name: engineb
|
||||
shortcut: b
|
||||
engine: dummy
|
||||
proxies: http://127.0.0.1:8000
|
||||
- name: enginec
|
||||
shortcut: c
|
||||
engine: dummy
|
||||
network:
|
||||
proxies:
|
||||
all://: http://127.0.0.1:8000
|
||||
- name: engined
|
||||
shortcut: d
|
||||
engine: dummy
|
||||
network:
|
||||
proxies:
|
||||
all://:
|
||||
- http://127.0.0.1:8000
|
||||
- http://127.0.0.1:9000
|
||||
- name: enginee
|
||||
shortcut: e
|
||||
engine: dummy
|
||||
network:
|
||||
proxies:
|
||||
all://:
|
||||
- http://127.0.0.1:8000
|
||||
- http://127.0.0.1:9000
|
||||
example.com://:
|
||||
- http://127.0.0.1:6000
|
||||
- http://127.0.0.1:7000
|
||||
- name: enginef
|
||||
shortcut: f
|
||||
engine: dummy
|
||||
network:
|
||||
proxies:
|
||||
- http://127.0.0.1:8000
|
||||
- http://127.0.0.1:9000
|
||||
- name: engineg
|
||||
shortcut: g
|
||||
engine: dummy
|
||||
network:
|
||||
local_addresses:
|
||||
- 192.168.0.1
|
||||
- 192.168.0.200
|
||||
- name: engineh
|
||||
shortcut: h
|
||||
engine: dummy
|
||||
network:
|
||||
local_addresses: 192.168.0.2
|
||||
- name: ip2
|
||||
shortcut: ip2
|
||||
engine: dummy
|
||||
network:
|
||||
local_addresses: 192.168.0.1/30
|
||||
- name: enginei
|
||||
shortcut: i
|
||||
engine: dummy
|
||||
network:
|
||||
retry_strategy: engine
|
||||
- name: enginej
|
||||
shortcut: j
|
||||
engine: dummy
|
||||
network:
|
||||
retry_strategy: SAME_HTTP_CLIENT
|
||||
- name: enginek
|
||||
shortcut: k
|
||||
engine: dummy
|
||||
network:
|
||||
retry_strategy: DIFFERENT_HTTP_CLIENT
|
@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# pylint: disable=missing-class-docstring
|
||||
# pylint: disable=protected-access
|
||||
"""Test for settings related to the network configuration
|
||||
"""
|
||||
|
||||
from os.path import dirname, join, abspath
|
||||
from unittest.mock import patch
|
||||
|
||||
from searx import settings_loader, settings_defaults, engines
|
||||
from searx.network import network
|
||||
from tests import SearxTestCase
|
||||
|
||||
|
||||
test_dir = abspath(dirname(__file__))
|
||||
|
||||
|
||||
class TestDefaultSettings(SearxTestCase):
|
||||
def test_load(self):
|
||||
# to do later : write more tests (thank you pylint for stoping the linting of t o d o...)
|
||||
# for now, make sure the code does not crash
|
||||
with patch.dict(settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'network_settings.yml')}):
|
||||
settings, _ = settings_loader.load_settings()
|
||||
settings_defaults.apply_schema(settings, settings_defaults.SCHEMA, [])
|
||||
engines.load_engines(settings["engines"])
|
||||
network_manager = network.NetworkManager()
|
||||
network_manager.initialize_from_settings(settings["engines"], settings["outgoing"], check=True)
|
||||
|
||||
network_enginea = network_manager.get("enginea")
|
||||
http_client = network_enginea._get_http_client()
|
||||
|
||||
repr_network = "<Network logger_name='enginea'>"
|
||||
|
||||
self.assertEqual(repr(network_enginea), repr_network)
|
||||
self.assertTrue(repr_network in repr(http_client))
|
Loading…
Reference in New Issue