.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
http_user_agent.py
Go to the documentation of this file.
1
# SPDX-License-Identifier: AGPL-3.0-or-later
2
"""
3
Method ``http_user_agent``
4
--------------------------
5
6
The ``http_user_agent`` method evaluates a request as the request of a bot if
7
the User-Agent_ header is unset or matches the regular expression
8
:py:obj:`USER_AGENT`.
9
10
.. _User-Agent:
11
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
12
13
"""
14
15
from
__future__
import
annotations
16
import
re
17
from
ipaddress
import
(
18
IPv4Network,
19
IPv6Network,
20
)
21
22
import
werkzeug
23
import
flask
24
25
from
.
import
config
26
from
._helpers
import
too_many_requests
27
28
29
USER_AGENT = (
30
r'('
31
+
r'unknown'
32
+
r'|[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp'
33
+
r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy'
34
+
r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot'
35
+
r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot'
36
+
r'|ZmEu|BLEXBot|bitlybot|HeadlessChrome'
37
# unmaintained Farside instances
38
+
r'|'
39
+ re.escape(
r'Mozilla/5.0 (compatible; Farside/0.1.0; +https://farside.link)'
)
40
# other bots and client to block
41
+
'|.*PetalBot.*'
42
+
r')'
43
)
44
"""Regular expression that matches to User-Agent_ from known *bots*"""
45
46
_regexp =
None
47
48
49
def
regexp_user_agent
():
50
global
_regexp
# pylint: disable=global-statement
51
if
not
_regexp:
52
_regexp = re.compile(USER_AGENT)
53
return
_regexp
54
55
56
def
filter_request
(
57
network: IPv4Network | IPv6Network,
58
request: flask.Request,
59
cfg:
config.Config
,
# pylint: disable=unused-argument
60
) -> werkzeug.Response |
None
:
61
62
user_agent = request.headers.get(
'User-Agent'
,
'unknown'
)
63
if
regexp_user_agent
().match(user_agent):
64
return
too_many_requests(network, f
"bot detected, HTTP header User-Agent: {user_agent}"
)
65
return
None
searx.botdetection.config.Config
Definition
config.py:67
searx.botdetection.http_user_agent.filter_request
werkzeug.Response|None filter_request(IPv4Network|IPv6Network network, flask.Request request, config.Config cfg)
Definition
http_user_agent.py:60
searx.botdetection.http_user_agent.regexp_user_agent
regexp_user_agent()
Definition
http_user_agent.py:49
searxng
searx
botdetection
http_user_agent.py
Generated on
for .oO SearXNG Developer Documentation Oo. by
1.14.0