diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index 8eb69630..9670aa49 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -5,12 +5,20 @@ To monitor rate limits and protect privacy the IP addresses are getting stored with a hash so the limiter plugin knows who to block. A redis database is -needed to store the hash values. +needed to store the hash values. + +It is also possible to bypass the limiter for a specific IP address or subnet +using the `whitelist_ip` and `whitelist_subnet` settings. Enable the plugin in ``settings.yml``: - ``server.limiter: true`` +- ``server.limiter.whitelist_ip: ['127.0.0.1']`` +- ``server.limiter_whitelist_subnet: ['192.168.0.0/24']`` - ``redis.url: ...`` check the value, see :ref:`settings redis` + + + """ import ipaddress @@ -41,14 +49,24 @@ WHITELISTED_IPS = get_setting('server.limiter_whitelist_ip', default=[]) WHITELISTED_SUBNET = get_setting('server.limiter_whitelist_subnet', default=[]) -def is_whitelist_ip(ip): +def is_whitelist_ip(ip: str) -> bool: ''' Check if the given IP address belongs to the whitelisted list of IP addresses or subnets. ''' - return ip in WHITELISTED_IPS or any(ipaddress.ip_address(ip) in - ipaddress.ip_network(subnet) - for subnet in WHITELISTED_SUBNET) + # if ip is empty use the source ip + if ip == '': + ip = request.remote_addr + logger.debug("checking whitelist rules for: %s", ip) + whitelisted = False + try: + whitelisted = ip in WHITELISTED_IPS or any( + ipaddress.ip_address(ip) in ipaddress.ip_network(subnet) for subnet in WHITELISTED_SUBNET + ) + except ValueError as e: + logger.error("Error while checking ratelimiter whitelist: %s", e) + + return whitelisted def is_accepted_request() -> bool: diff --git a/searx/settings.yml b/searx/settings.yml index 6f80b4c5..4ee59a4a 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -72,6 +72,11 @@ server: base_url: false # Possible values: false or "https://example.org/location". limiter: false # rate limit the number of request on the instance, block some bots + ## If you enabled the rate limiter you can add ips or subnet exceptions, + ## uncomment any of the lines below to add you exceptions + # limiter_whitelist_ip: ['127.0.0.1'] # disables the rate limiter for localhost + # limiter_whitelist_subnet: ['192.168.0.0/24'] # disable the rate limiter for an example home subnet + # If your instance owns a /etc/searxng/settings.yml file, then set the following # values there. diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 7baa23ca..ae4a4415 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -174,6 +174,8 @@ SCHEMA = { 'port': SettingsValue((int, str), 8888, 'SEARXNG_PORT'), 'bind_address': SettingsValue(str, '127.0.0.1', 'SEARXNG_BIND_ADDRESS'), 'limiter': SettingsValue(bool, False), + 'limiter_whitelist_ip': SettingsValue(list, []), + 'limiter_whitelist_subnet': SettingsValue(list, []), 'secret_key': SettingsValue(str, environ_name='SEARXNG_SECRET'), 'base_url': SettingsValue((False, str), False, 'SEARXNG_BASE_URL'), 'image_proxy': SettingsValue(bool, False), diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 28df835e..1beab5c8 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -from searx import plugins +from searx import plugins, redisdb from mock import Mock +from mock import patch from tests import SearxTestCase @@ -152,3 +153,61 @@ class HashPluginTest(SearxTestCase): '18980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5' 'fa9ad8e6f57f50028a8ff' in search.result_container.answers['hash']['answer'] ) + + +@patch.object(redisdb, '_CLIENT', new=True) +class LimiterPluginTest(SearxTestCase): + def test_whitelist(self): + store = plugins.PluginStore() + limiter_settings = { + 'server': { + 'limiter': True, + } + } + + app_mock = Mock(before_request=lambda x: True) + plugin = plugins.load_and_initialize_plugin('searx.plugins.limiter', False, (app_mock, limiter_settings)) + store.register(plugin) + self.assertTrue(len(store.plugins) == 1) + + def test_whitelist_case(case): + plugins.limiter.WHITELISTED_SUBNET = case[1]['whitelist_subnet'] + plugins.limiter.WHITELISTED_IPS = case[1]['whitelist_ip'] + ret = store.call(store.plugins, 'is_whitelist_ip', case[0]) + self.assertEqual(ret, case[2]) + + test_cases = [] + # default test case with no whitelist + test_cases.append( + ( + '192.0.43.22', # x_forwarded_for + {'whitelist_ip': [], 'whitelist_subnet': []}, + False, # expected return value if request is accepted + ) + ) + + # not an ip + test_cases.append(('192.0.43.22', {'whitelist_ip': 'not an ip', 'whitelist_subnet': []}, False)) + + # not a subnet + test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': 'not a subnet'}, False)) + + # test single ip + test_cases.append(('192.0.43.22', {'whitelist_ip': '192.0.43.22', 'whitelist_subnet': []}, True)) + + # test ip in list + test_cases.append(('192.0.43.22', {'whitelist_ip': ['192.0.43.22'], 'whitelist_subnet': []}, True)) + + # test ok single subnet + test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/16']}, True)) + + # test ok subnet in list + test_cases.append( + ('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/16', '192.0.0.1/24']}, True) + ) + + # test ko subnet + test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/24']}, False)) + + for case in test_cases: + test_whitelist_case(case)