.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
_helpers.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2# pylint: disable=missing-module-docstring, invalid-name
3from __future__ import annotations
4
5from ipaddress import (
6 IPv4Network,
7 IPv6Network,
8 IPv4Address,
9 IPv6Address,
10 ip_network,
11 ip_address,
12)
13import flask
14import werkzeug
15
16from searx import logger
17from searx.extended_types import SXNG_Request
18
19from . import config
20
21logger = logger.getChild('botdetection')
22
23
24def dump_request(request: SXNG_Request):
25 return (
26 request.path
27 + " || X-Forwarded-For: %s" % request.headers.get('X-Forwarded-For')
28 + " || X-Real-IP: %s" % request.headers.get('X-Real-IP')
29 + " || form: %s" % request.form
30 + " || Accept: %s" % request.headers.get('Accept')
31 + " || Accept-Language: %s" % request.headers.get('Accept-Language')
32 + " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding')
33 + " || Content-Type: %s" % request.headers.get('Content-Type')
34 + " || Content-Length: %s" % request.headers.get('Content-Length')
35 + " || Connection: %s" % request.headers.get('Connection')
36 + " || User-Agent: %s" % request.headers.get('User-Agent')
37 )
38
39
40def too_many_requests(network: IPv4Network | IPv6Network, log_msg: str) -> werkzeug.Response | None:
41 """Returns a HTTP 429 response object and writes a ERROR message to the
42 'botdetection' logger. This function is used in part by the filter methods
43 to return the default ``Too Many Requests`` response.
44
45 """
46
47 logger.debug("BLOCK %s: %s", network.compressed, log_msg)
48 return flask.make_response(('Too Many Requests', 429))
49
50
51def get_network(real_ip: IPv4Address | IPv6Address, cfg: config.Config) -> IPv4Network | IPv6Network:
52 """Returns the (client) network of whether the real_ip is part of."""
53
54 if real_ip.version == 6:
55 prefix = cfg['real_ip.ipv6_prefix']
56 else:
57 prefix = cfg['real_ip.ipv4_prefix']
58 network = ip_network(f"{real_ip}/{prefix}", strict=False)
59 # logger.debug("get_network(): %s", network.compressed)
60 return network
61
62
63_logged_errors = []
64
65
67 if err_msg not in _logged_errors:
68 logger.error(err_msg)
69 _logged_errors.append(err_msg)
70
71
72def get_real_ip(request: SXNG_Request) -> str:
73 """Returns real IP of the request. Since not all proxies set all the HTTP
74 headers and incoming headers can be faked it may happen that the IP cannot
75 be determined correctly.
76
77 .. sidebar:: :py:obj:`flask.Request.remote_addr`
78
79 SearXNG uses Werkzeug's ProxyFix_ (with it default ``x_for=1``).
80
81 This function tries to get the remote IP in the order listed below,
82 additional some tests are done and if inconsistencies or errors are
83 detected, they are logged.
84
85 The remote IP of the request is taken from (first match):
86
87 - X-Forwarded-For_ header
88 - `X-real-IP header <https://github.com/searxng/searxng/issues/1237#issuecomment-1147564516>`__
89 - :py:obj:`flask.Request.remote_addr`
90
91 .. _ProxyFix:
92 https://werkzeug.palletsprojects.com/middleware/proxy_fix/
93
94 .. _X-Forwarded-For:
95 https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
96
97 """
98
99 forwarded_for = request.headers.get("X-Forwarded-For")
100 real_ip = request.headers.get('X-Real-IP')
101 remote_addr = request.remote_addr
102 # logger.debug(
103 # "X-Forwarded-For: %s || X-Real-IP: %s || request.remote_addr: %s", forwarded_for, real_ip, remote_addr
104 # )
105
106 if not forwarded_for:
107 _log_error_only_once("X-Forwarded-For header is not set!")
108 else:
109 from . import cfg # pylint: disable=import-outside-toplevel, cyclic-import
110
111 forwarded_for = [x.strip() for x in forwarded_for.split(',')]
112 x_for: int = cfg['real_ip.x_for'] # type: ignore
113 forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
114
115 if not real_ip:
116 _log_error_only_once("X-Real-IP header is not set!")
117
118 if forwarded_for and real_ip and forwarded_for != real_ip:
119 logger.warning("IP from X-Real-IP (%s) is not equal to IP from X-Forwarded-For (%s)", real_ip, forwarded_for)
120
121 if forwarded_for and remote_addr and forwarded_for != remote_addr:
122 logger.warning(
123 "IP from WSGI environment (%s) is not equal to IP from X-Forwarded-For (%s)", remote_addr, forwarded_for
124 )
125
126 if real_ip and remote_addr and real_ip != remote_addr:
127 logger.warning("IP from WSGI environment (%s) is not equal to IP from X-Real-IP (%s)", remote_addr, real_ip)
128
129 request_ip = ip_address(forwarded_for or real_ip or remote_addr or '0.0.0.0')
130 if request_ip.version == 6 and request_ip.ipv4_mapped:
131 request_ip = request_ip.ipv4_mapped
132
133 # logger.debug("get_real_ip() -> %s", request_ip)
134 return str(request_ip)
IPv4Network|IPv6Network get_network(IPv4Address|IPv6Address real_ip, config.Config cfg)
Definition _helpers.py:51
werkzeug.Response|None too_many_requests(IPv4Network|IPv6Network network, str log_msg)
Definition _helpers.py:40
str get_real_ip(SXNG_Request request)
Definition _helpers.py:72
dump_request(SXNG_Request request)
Definition _helpers.py:24