limiter_whitelisting: add tests, update doc

searx/plugins/limiter.py

@@ -5,12 +5,20 @@
 
 To monitor rate limits and protect privacy the IP addresses are getting stored
 with a hash so the limiter plugin knows who to block.  A redis database is
-needed to store the hash values.
+needed to store the hash values. 
+
+It is also possible to bypass the limiter for a specific IP address or subnet 
+using the `whitelist_ip` and `whitelist_subnet` settings.
 
 Enable the plugin in ``settings.yml``:
 
 - ``server.limiter: true``
+- ``server.limiter.whitelist_ip: ['127.0.0.1']``
+- ``server.limiter_whitelist_subnet: ['192.168.0.0/24']``
 - ``redis.url: ...`` check the value, see :ref:`settings redis`
+
+
+
 """
 
 import ipaddress
@@ -41,14 +49,24 @@ WHITELISTED_IPS = get_setting('server.limiter_whitelist_ip', default=[])
 WHITELISTED_SUBNET = get_setting('server.limiter_whitelist_subnet', default=[])
 
 
-def is_whitelist_ip(ip):
+def is_whitelist_ip(ip: str) -> bool:
     '''
     Check if the given IP address belongs to the whitelisted list
     of IP addresses or subnets.
     '''
-    return ip in WHITELISTED_IPS or any(ipaddress.ip_address(ip) in
+    # if ip is empty use the source ip
-                                        ipaddress.ip_network(subnet)
+    if ip == '':
-                                        for subnet in WHITELISTED_SUBNET)
+        ip = request.remote_addr
+    logger.debug("checking whitelist rules for: %s", ip)
+    whitelisted = False
+    try:
+        whitelisted = ip in WHITELISTED_IPS or any(
+            ipaddress.ip_address(ip) in ipaddress.ip_network(subnet) for subnet in WHITELISTED_SUBNET
+        )
+    except ValueError as e:
+        logger.error("Error while checking ratelimiter whitelist: %s", e)
+
+    return whitelisted
 
 
 def is_accepted_request() -> bool:

searx/settings.yml

@@ -72,6 +72,11 @@ server:
   base_url: false  # Possible values: false or "https://example.org/location".
   limiter: false  # rate limit the number of request on the instance, block some bots
 
+  ## If you enabled the rate limiter you can add ips or subnet exceptions,
+  ## uncomment any of the lines below to add you exceptions
+  # limiter_whitelist_ip: ['127.0.0.1'] # disables the rate limiter for localhost
+  # limiter_whitelist_subnet: ['192.168.0.0/24'] # disable the rate limiter for an example home subnet
+
   # If your instance owns a /etc/searxng/settings.yml file, then set the following
   # values there.
 

searx/settings_defaults.py

@@ -174,6 +174,8 @@ SCHEMA = {
         'port': SettingsValue((int, str), 8888, 'SEARXNG_PORT'),
         'bind_address': SettingsValue(str, '127.0.0.1', 'SEARXNG_BIND_ADDRESS'),
         'limiter': SettingsValue(bool, False),
+        'limiter_whitelist_ip': SettingsValue(list, []),
+        'limiter_whitelist_subnet': SettingsValue(list, []),
         'secret_key': SettingsValue(str, environ_name='SEARXNG_SECRET'),
         'base_url': SettingsValue((False, str), False, 'SEARXNG_BASE_URL'),
         'image_proxy': SettingsValue(bool, False),

tests/unit/test_plugins.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 
-from searx import plugins
+from searx import plugins, redisdb
 from mock import Mock
+from mock import patch
 from tests import SearxTestCase
 
 
@@ -152,3 +153,61 @@ class HashPluginTest(SearxTestCase):
             '18980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5'
             'fa9ad8e6f57f50028a8ff' in search.result_container.answers['hash']['answer']
         )
+
+
+@patch.object(redisdb, '_CLIENT', new=True)
+class LimiterPluginTest(SearxTestCase):
+    def test_whitelist(self):
+        store = plugins.PluginStore()
+        limiter_settings = {
+            'server': {
+                'limiter': True,
+            }
+        }
+
+        app_mock = Mock(before_request=lambda x: True)
+        plugin = plugins.load_and_initialize_plugin('searx.plugins.limiter', False, (app_mock, limiter_settings))
+        store.register(plugin)
+        self.assertTrue(len(store.plugins) == 1)
+
+        def test_whitelist_case(case):
+            plugins.limiter.WHITELISTED_SUBNET = case[1]['whitelist_subnet']
+            plugins.limiter.WHITELISTED_IPS = case[1]['whitelist_ip']
+            ret = store.call(store.plugins, 'is_whitelist_ip', case[0])
+            self.assertEqual(ret, case[2])
+
+        test_cases = []
+        # default test case with no whitelist
+        test_cases.append(
+            (
+                '192.0.43.22',  # x_forwarded_for
+                {'whitelist_ip': [], 'whitelist_subnet': []},
+                False,  # expected return value if request is accepted
+            )
+        )
+
+        # not an ip
+        test_cases.append(('192.0.43.22', {'whitelist_ip': 'not an ip', 'whitelist_subnet': []}, False))
+
+        # not a subnet
+        test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': 'not a subnet'}, False))
+
+        # test single ip
+        test_cases.append(('192.0.43.22', {'whitelist_ip': '192.0.43.22', 'whitelist_subnet': []}, True))
+
+        # test ip in list
+        test_cases.append(('192.0.43.22', {'whitelist_ip': ['192.0.43.22'], 'whitelist_subnet': []}, True))
+
+        # test ok single subnet
+        test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/16']}, True))
+
+        # test ok subnet in list
+        test_cases.append(
+            ('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/16', '192.0.0.1/24']}, True)
+        )
+
+        # test ko subnet
+        test_cases.append(('192.0.43.22', {'whitelist_ip': [], 'whitelist_subnet': ['192.0.0.0/24']}, False))
+
+        for case in test_cases:
+            test_whitelist_case(case)