.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
http_user_agent.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""
3Method ``http_user_agent``
4--------------------------
5
6The ``http_user_agent`` method evaluates a request as the request of a bot if
7the User-Agent_ header is unset or matches the regular expression
8:py:obj:`USER_AGENT`.
9
10.. _User-Agent:
11 https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
12
13"""
14
15from __future__ import annotations
16import re
17from ipaddress import (
18 IPv4Network,
19 IPv6Network,
20)
21
22import werkzeug
23import flask
24
25from . import config
26from ._helpers import too_many_requests
27
28
29USER_AGENT = (
30 r'('
31 + r'unknown'
32 + r'|[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp'
33 + r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy'
34 + r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot'
35 + r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot'
36 + r'|ZmEu|BLEXBot|bitlybot|HeadlessChrome'
37 # unmaintained Farside instances
38 + r'|'
39 + re.escape(r'Mozilla/5.0 (compatible; Farside/0.1.0; +https://farside.link)')
40 # other bots and client to block
41 + '|.*PetalBot.*'
42 + r')'
43)
44"""Regular expression that matches to User-Agent_ from known *bots*"""
45
46_regexp = None
47
48
50 global _regexp # pylint: disable=global-statement
51 if not _regexp:
52 _regexp = re.compile(USER_AGENT)
53 return _regexp
54
55
57 network: IPv4Network | IPv6Network,
58 request: flask.Request,
59 cfg: config.Config, # pylint: disable=unused-argument
60) -> werkzeug.Response | None:
61
62 user_agent = request.headers.get('User-Agent', 'unknown')
63 if regexp_user_agent().match(user_agent):
64 return too_many_requests(network, f"bot detected, HTTP header User-Agent: {user_agent}")
65 return None
werkzeug.Response|None filter_request(IPv4Network|IPv6Network network, flask.Request request, config.Config cfg)