.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
limiter.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Bot protection / IP rate limitation. The intention of rate limitation is to
3limit suspicious requests from an IP. The motivation behind this is the fact
4that SearXNG passes through requests from bots and is thus classified as a bot
5itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked
6by the search engine (the origin) in some other way.
7
8To avoid blocking, the requests from bots to SearXNG must also be blocked, this
9is the task of the limiter. To perform this task, the limiter uses the methods
10from the :ref:`botdetection`:
11
12- Analysis of the HTTP header in the request / :ref:`botdetection probe headers`
13 can be easily bypassed.
14
15- Block and pass lists in which IPs are listed / :ref:`botdetection ip_lists`
16 are hard to maintain, since the IPs of bots are not all known and change over
17 the time.
18
19- Detection & dynamically :ref:`botdetection rate limit` of bots based on the
20 behavior of the requests. For dynamically changeable IP lists a Redis
21 database is needed.
22
23The prerequisite for IP based methods is the correct determination of the IP of
24the client. The IP of the client is determined via the X-Forwarded-For_ HTTP
25header.
26
27.. attention::
28
29 A correct setup of the HTTP request headers ``X-Forwarded-For`` and
30 ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
31
32 - `NGINX RequestHeader`_
33 - `Apache RequestHeader`_
34
35.. _X-Forwarded-For:
36 https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
37.. _NGINX RequestHeader:
38 https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
39.. _Apache RequestHeader:
40 https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
41
42Enable Limiter
43==============
44
45To enable the limiter activate:
46
47.. code:: yaml
48
49 server:
50 ...
51 limiter: true # rate limit the number of request on the instance, block some bots
52
53and set the redis-url connection. Check the value, it depends on your redis DB
54(see :ref:`settings redis`), by example:
55
56.. code:: yaml
57
58 redis:
59 url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
60
61
62Configure Limiter
63=================
64
65The methods of :ref:`botdetection` the limiter uses are configured in a local
66file ``/etc/searxng/limiter.toml``. The defaults are shown in limiter.toml_ /
67Don't copy all values to your local configuration, just enable what you need by
68overwriting the defaults. For instance to activate the ``link_token`` method in
69the :ref:`botdetection.ip_limit` you only need to set this option to ``true``:
70
71.. code:: toml
72
73 [botdetection.ip_limit]
74 link_token = true
75
76.. _limiter.toml:
77
78``limiter.toml``
79================
80
81In this file the limiter finds the configuration of the :ref:`botdetection`:
82
83- :ref:`botdetection ip_lists`
84- :ref:`botdetection rate limit`
85- :ref:`botdetection probe headers`
86
87.. kernel-include:: $SOURCEDIR/limiter.toml
88 :code: toml
89
90Implementation
91==============
92
93"""
94
95from __future__ import annotations
96import sys
97
98from pathlib import Path
99from ipaddress import ip_address
100import flask
101import werkzeug
102
103from searx import (
104 logger,
105 redisdb,
106)
107from searx import botdetection
108from searx.extended_types import SXNG_Request, sxng_request
109from searx.botdetection import (
110 config,
111 http_accept,
112 http_accept_encoding,
113 http_accept_language,
114 http_user_agent,
115 http_sec_fetch,
116 ip_limit,
117 ip_lists,
118 get_network,
119 get_real_ip,
120 dump_request,
121)
122
123# the configuration are limiter.toml and "limiter" in settings.yml so, for
124# coherency, the logger is "limiter"
125logger = logger.getChild('limiter')
126
127CFG: config.Config = None # type: ignore
128_INSTALLED = False
129
130LIMITER_CFG_SCHEMA = Path(__file__).parent / "limiter.toml"
131"""Base configuration (schema) of the botdetection."""
132
133CFG_DEPRECATED = {
134 # "dummy.old.foo": "config 'dummy.old.foo' exists only for tests. Don't use it in your real project config."
135}
136
137
138def get_cfg() -> config.Config:
139 global CFG # pylint: disable=global-statement
140
141 if CFG is None:
142 from . import settings_loader # pylint: disable=import-outside-toplevel
143
144 cfg_file = (settings_loader.get_user_cfg_folder() or Path("/etc/searxng")) / "limiter.toml"
145 CFG = config.Config.from_toml(LIMITER_CFG_SCHEMA, cfg_file, CFG_DEPRECATED)
146 return CFG
147
148
149def filter_request(request: SXNG_Request) -> werkzeug.Response | None:
150 # pylint: disable=too-many-return-statements
151
152 cfg = get_cfg()
153 real_ip = ip_address(get_real_ip(request))
154 network = get_network(real_ip, cfg)
155
156 if request.path == '/healthz':
157 return None
158
159 # link-local
160
161 if network.is_link_local:
162 return None
163
164 # block- & pass- lists
165 #
166 # 1. The IP of the request is first checked against the pass-list; if the IP
167 # matches an entry in the list, the request is not blocked.
168 # 2. If no matching entry is found in the pass-list, then a check is made against
169 # the block list; if the IP matches an entry in the list, the request is
170 # blocked.
171 # 3. If the IP is not in either list, the request is not blocked.
172
173 match, msg = ip_lists.pass_ip(real_ip, cfg)
174 if match:
175 logger.warning("PASS %s: matched PASSLIST - %s", network.compressed, msg)
176 return None
177
178 match, msg = ip_lists.block_ip(real_ip, cfg)
179 if match:
180 logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
181 return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
182
183 # methods applied on all requests
184
185 for func in [
186 http_user_agent,
187 ]:
188 val = func.filter_request(network, request, cfg)
189 if val is not None:
190 logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
191 return val
192
193 # methods applied on /search requests
194
195 if request.path == '/search':
196
197 for func in [
198 http_accept,
199 http_accept_encoding,
200 http_accept_language,
201 http_user_agent,
202 http_sec_fetch,
203 ip_limit,
204 ]:
205 val = func.filter_request(network, request, cfg)
206 if val is not None:
207 logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
208 return val
209
210 logger.debug(f"OK {network}: %s", dump_request(sxng_request))
211 return None
212
213
215 """See :py:obj:`flask.Flask.before_request`"""
216 return filter_request(sxng_request)
217
218
220 """Returns ``True`` if limiter is active and a redis DB is available."""
221 return _INSTALLED
222
223
224def initialize(app: flask.Flask, settings):
225 """Install the limiter"""
226 global _INSTALLED # pylint: disable=global-statement
227
228 # even if the limiter is not activated, the botdetection must be activated
229 # (e.g. the self_info plugin uses the botdetection to get client IP)
230
231 cfg = get_cfg()
232 redis_client = redisdb.client()
233 botdetection.init(cfg, redis_client)
234
235 if not (settings['server']['limiter'] or settings['server']['public_instance']):
236 return
237
238 if not redis_client:
239 logger.error(
240 "The limiter requires Redis, please consult the documentation: "
241 "https://docs.searxng.org/admin/searx.limiter.html"
242 )
243 if settings['server']['public_instance']:
244 sys.exit(1)
245 return
246
247 _INSTALLED = True
248
249 if settings['server']['public_instance']:
250 # overwrite limiter.toml setting
251 cfg.set('botdetection.ip_limit.link_token', True)
252
253 app.before_request(pre_request)
config.Config get_cfg()
Definition limiter.py:138
initialize(flask.Flask app, settings)
Definition limiter.py:224
werkzeug.Response|None filter_request(SXNG_Request request)
Definition limiter.py:149