.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
error_recorder.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2# pylint: disable=missing-module-docstring, invalid-name
3
4import typing as t
5
6import inspect
7from json import JSONDecodeError
8from urllib.parse import urlparse
9from httpx import HTTPError, HTTPStatusError
10from searx.exceptions import (
11 SearxXPathSyntaxException,
12 SearxEngineXPathException,
13 SearxEngineAPIException,
14 SearxEngineAccessDeniedException,
15)
16from searx import searx_parent_dir, settings
17from searx.engines import engines
18
19
20errors_per_engines: dict[str, t.Any] = {}
21
22LogParametersType = tuple[str, ...]
23
24
25class ErrorContext: # pylint: disable=missing-class-docstring
26
27 def __init__( # pylint: disable=too-many-arguments
28 self,
29 filename: str,
30 function: str,
31 line_no: int,
32 code: str,
33 exception_classname: str,
34 log_message: str,
35 log_parameters: LogParametersType,
36 secondary: bool,
37 ):
38 self.filename: str = filename
39 self.function: str = function
40 self.line_no: int = line_no
41 self.code: str = code
42 self.exception_classname: str = exception_classname
43 self.log_message: str = log_message
44 self.log_parameters: LogParametersType = log_parameters
45 self.secondary: bool = secondary
46
47 def __eq__(self, o) -> bool: # pylint: disable=invalid-name
48 if not isinstance(o, ErrorContext):
49 return False
50 return (
51 self.filename == o.filename
52 and self.function == o.function
53 and self.line_no == o.line_no
54 and self.code == o.code
55 and self.exception_classname == o.exception_classname
56 and self.log_message == o.log_message
57 and self.log_parameters == o.log_parameters
58 and self.secondary == o.secondary
59 )
60
61 def __hash__(self):
62 return hash(
63 (
64 self.filename,
65 self.function,
66 self.line_no,
67 self.code,
69 self.log_message,
70 self.log_parameters,
71 self.secondary,
72 )
73 )
74
75 def __repr__(self):
76 return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) {!r}".format(
77 self.filename,
78 self.line_no,
79 self.code,
81 self.log_message,
82 self.log_parameters,
83 self.secondary,
84 )
85
86
87def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
88 errors_for_engine = errors_per_engines.setdefault(engine_name, {})
89 errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
90 engines[engine_name].logger.warning('%s', str(error_context))
91
92
93def get_trace(traces):
94 for trace in reversed(traces):
95 split_filename: list[str] = trace.filename.split('/')
96 if '/'.join(split_filename[-3:-1]) == 'searx/engines':
97 return trace
98 if '/'.join(split_filename[-4:-1]) == 'searx/search/processors':
99 return trace
100 return traces[-1]
101
102
103def get_hostname(exc: HTTPError) -> str | None:
104 url = exc.request.url
105 if url is None and exc.response is not None:
106 url = exc.response.url
107 return urlparse(url).netloc
108
109
111 exc: HTTPError,
112) -> tuple[str | None, str | None, str | None]:
113 url = None
114 status_code = None
115 reason = None
116 hostname = None
117 if hasattr(exc, '_request') and exc._request is not None: # pylint: disable=protected-access
118 # exc.request is property that raise an RuntimeException
119 # if exc._request is not defined.
120 url = exc.request.url
121 if url is None and hasattr(exc, 'response') and exc.response is not None:
122 url = exc.response.url
123 if url is not None:
124 hostname = url.host
125 if isinstance(exc, HTTPStatusError):
126 status_code = str(exc.response.status_code)
127 reason = exc.response.reason_phrase
128 return (status_code, reason, hostname)
129
130
131def get_messages(exc, filename) -> tuple[str, ...]: # pylint: disable=too-many-return-statements
132 if isinstance(exc, JSONDecodeError):
133 return (exc.msg,)
134 if isinstance(exc, TypeError):
135 return (str(exc),)
136 if isinstance(exc, ValueError) and 'lxml' in filename:
137 return (str(exc),)
138 if isinstance(exc, HTTPError):
140 if isinstance(exc, SearxXPathSyntaxException):
141 return (exc.xpath_str, exc.message)
142 if isinstance(exc, SearxEngineXPathException):
143 return (exc.xpath_str, exc.message)
144 if isinstance(exc, SearxEngineAPIException):
145 return (str(exc.args[0]),)
146 if isinstance(exc, SearxEngineAccessDeniedException):
147 return (exc.message,)
148 return ()
149
150
151def get_exception_classname(exc: BaseException) -> str:
152 exc_class = exc.__class__
153 exc_name = exc_class.__qualname__
154 exc_module = exc_class.__module__
155 if exc_module is None or exc_module == str.__class__.__module__:
156 return exc_name
157 return exc_module + '.' + exc_name
158
159
161 framerecords, exception_classname, log_message, log_parameters: LogParametersType, secondary: bool
162) -> ErrorContext:
163 searx_frame = get_trace(framerecords)
164 filename = searx_frame.filename
165 if filename.startswith(searx_parent_dir):
166 filename = filename[len(searx_parent_dir) + 1 :]
167 function = searx_frame.function
168 line_no = searx_frame.lineno
169 code = searx_frame.code_context[0].strip()
170 del framerecords
171 return ErrorContext(filename, function, line_no, code, exception_classname, log_message, log_parameters, secondary)
172
173
174def count_exception(engine_name: str, exc: BaseException, secondary: bool = False) -> None:
175 if not settings['general']['enable_metrics']:
176 return
177 framerecords = inspect.trace()
178 try:
179 exception_classname = get_exception_classname(exc)
180 log_parameters = get_messages(exc, framerecords[-1][1])
181 error_context = get_error_context(framerecords, exception_classname, None, log_parameters, secondary)
182 add_error_context(engine_name, error_context)
183 finally:
184 del framerecords
185
186
188 engine_name: str,
189 log_message: str,
190 log_parameters: LogParametersType | None = None,
191 secondary: bool = False,
192) -> None:
193 if not settings['general']['enable_metrics']:
194 return
195 framerecords = list(reversed(inspect.stack()[1:]))
196 try:
197 error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary)
198 add_error_context(engine_name, error_context)
199 finally:
200 del framerecords
__init__(self, str filename, str function, int line_no, str code, str exception_classname, str log_message, LogParametersType log_parameters, bool secondary)
::1337x
Definition 1337x.py:1
ErrorContext get_error_context(framerecords, exception_classname, log_message, LogParametersType log_parameters, bool secondary)
None count_error(str engine_name, str log_message, LogParametersType|None log_parameters=None, bool secondary=False)
str|None get_hostname(HTTPError exc)
tuple[str|None, str|None, str|None] get_request_exception_messages(HTTPError exc)
None add_error_context(str engine_name, ErrorContext error_context)
None count_exception(str engine_name, BaseException exc, bool secondary=False)
str get_exception_classname(BaseException exc)
tuple[str,...] get_messages(exc, filename)