.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
searx.metrics Namespace Reference

Namespaces

namespace  error_recorder
namespace  models

Functions

 histogram_observe_time (*args)
 histogram_observe (duration, *args)
 histogram (*args, raise_on_not_found=True)
 counter_inc (*str args)
 counter_add (int value, *str args)
 counter (*args)
None initialize (list[str]|None engine_names=None, bool enabled=True)
 get_engine_errors (engline_name_list)
 get_reliabilities (engline_name_list, checker_results)
 get_engines_stats (list[str] engine_name_list)
 openmetrics (engine_stats, engine_reliabilities)

Variables

list __all__
dict ENDPOINTS = {'search'}
HistogramStorage histogram_storage = None
CounterStorage counter_storage = None

Function Documentation

◆ counter()

searx.metrics.counter ( * args)

Definition at line 66 of file __init__.py.

66def counter(*args):
67 return counter_storage.get(*args)
68
69

Referenced by get_engine_errors(), get_engines_stats(), and get_reliabilities().

Here is the caller graph for this function:

◆ counter_add()

searx.metrics.counter_add ( int value,
*str args )

Definition at line 62 of file __init__.py.

62def counter_add(value: int, *args: str):
63 counter_storage.add(value, *args)
64
65

◆ counter_inc()

searx.metrics.counter_inc ( *str args)

Definition at line 58 of file __init__.py.

58def counter_inc(*args: str):
59 counter_storage.add(1, *args)
60
61

◆ get_engine_errors()

searx.metrics.get_engine_errors ( engline_name_list)

Definition at line 111 of file __init__.py.

111def get_engine_errors(engline_name_list):
112 result = {}
113 engine_names = list(errors_per_engines.keys())
114 engine_names.sort()
115 for engine_name in engine_names:
116 if engine_name not in engline_name_list:
117 continue
118
119 error_stats = errors_per_engines[engine_name]
120 sent_search_count = max(counter('engine', engine_name, 'search', 'count', 'sent'), 1)
121 sorted_context_count_list = sorted(error_stats.items(), key=lambda context_count: context_count[1])
122 r = []
123 for context, count in sorted_context_count_list:
124 percentage = round(20 * count / sent_search_count) * 5
125 r.append(
126 {
127 'filename': context.filename,
128 'function': context.function,
129 'line_no': context.line_no,
130 'code': context.code,
131 'exception_classname': context.exception_classname,
132 'log_message': context.log_message,
133 'log_parameters': context.log_parameters,
134 'secondary': context.secondary,
135 'percentage': percentage,
136 }
137 )
138 result[engine_name] = sorted(r, reverse=True, key=lambda d: d['percentage'])
139 return result
140
141

References counter().

Referenced by get_reliabilities().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_engines_stats()

searx.metrics.get_engines_stats ( list[str] engine_name_list)

Definition at line 175 of file __init__.py.

175def get_engines_stats(engine_name_list: list[str]):
176 assert counter_storage is not None
177 assert histogram_storage is not None
178
179 list_time = []
180 max_time_total = max_result_count = None
181
182 for engine_name in engine_name_list:
183
184 sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
185 if sent_count == 0:
186 continue
187
188 result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
189 result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
190 successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
191
192 time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
193 max_time_total = max(time_total or 0, max_time_total or 0)
194 max_result_count = max(result_count or 0, max_result_count or 0)
195
196 stats = {
197 'name': engine_name,
198 'total': None,
199 'total_p80': None,
200 'total_p95': None,
201 'http': None,
202 'http_p80': None,
203 'http_p95': None,
204 'processing': None,
205 'processing_p80': None,
206 'processing_p95': None,
207 'score': 0,
208 'score_per_result': 0,
209 'result_count': result_count,
210 }
211
212 if successful_count and result_count_sum:
213 score = counter('engine', engine_name, 'score')
214
215 stats['score'] = score
216 stats['score_per_result'] = score / float(result_count_sum)
217
218 time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
219 time_http_p80 = time_http_p95 = 0
220
221 if time_http is not None:
222
223 time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
224 time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
225
226 stats['http'] = round(time_http, 1)
227 stats['http_p80'] = round(time_http_p80, 1)
228 stats['http_p95'] = round(time_http_p95, 1)
229
230 if time_total is not None:
231
232 time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
233 time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
234
235 stats['total'] = round(time_total, 1)
236 stats['total_p80'] = round(time_total_p80, 1)
237 stats['total_p95'] = round(time_total_p95, 1)
238
239 stats['processing'] = round(time_total - (time_http or 0), 1)
240 stats['processing_p80'] = round(time_total_p80 - time_http_p80, 1)
241 stats['processing_p95'] = round(time_total_p95 - time_http_p95, 1)
242
243 list_time.append(stats)
244
245 return {
246 'time': list_time,
247 'max_time': math.ceil(max_time_total or 0),
248 'max_result_count': math.ceil(max_result_count or 0),
249 }
250
251

References counter(), and histogram().

Here is the call graph for this function:

◆ get_reliabilities()

searx.metrics.get_reliabilities ( engline_name_list,
checker_results )

Definition at line 142 of file __init__.py.

142def get_reliabilities(engline_name_list, checker_results):
143 reliabilities = {}
144
145 engine_errors = get_engine_errors(engline_name_list)
146
147 for engine_name in engline_name_list:
148 checker_result = checker_results.get(engine_name, {})
149 checker_success = checker_result.get('success', True)
150 errors = engine_errors.get(engine_name) or []
151 sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
152
153 if sent_count == 0:
154 # no request
155 reliability = None
156 elif checker_success and not errors:
157 reliability = 100
158 elif 'simple' in checker_result.get('errors', {}):
159 # the basic (simple) test doesn't work: the engine is broken according to the checker
160 # even if there is no exception
161 reliability = 0
162 else:
163 # pylint: disable=consider-using-generator
164 reliability = 100 - sum([error['percentage'] for error in errors if not error.get('secondary')])
165
166 reliabilities[engine_name] = {
167 'reliability': reliability,
168 'sent_count': sent_count,
169 'errors': errors,
170 'checker': checker_result.get('errors', {}),
171 }
172 return reliabilities
173
174

References counter(), and get_engine_errors().

Here is the call graph for this function:

◆ histogram()

searx.metrics.histogram ( * args,
raise_on_not_found = True )

Definition at line 51 of file __init__.py.

51def histogram(*args, raise_on_not_found=True):
52 h = histogram_storage.get(*args)
53 if raise_on_not_found and h is None:
54 raise ValueError("histogram " + repr((*args,)) + " doesn't not exist")
55 return h
56
57

Referenced by get_engines_stats().

Here is the caller graph for this function:

◆ histogram_observe()

searx.metrics.histogram_observe ( duration,
* args )

Definition at line 47 of file __init__.py.

47def histogram_observe(duration, *args):
48 histogram_storage.get(*args).observe(duration)
49
50

◆ histogram_observe_time()

searx.metrics.histogram_observe_time ( * args)

Definition at line 36 of file __init__.py.

36def histogram_observe_time(*args):
37 h = histogram_storage.get(*args)
38 before = default_timer()
39 yield before
40 duration = default_timer() - before
41 if h:
42 h.observe(duration)
43 else:
44 raise ValueError("histogram " + repr((*args,)) + " doesn't not exist")
45
46

◆ initialize()

None searx.metrics.initialize ( list[str] | None engine_names = None,
bool enabled = True )
Initialize metrics

Definition at line 70 of file __init__.py.

70def initialize(engine_names: list[str] | None = None, enabled: bool = True) -> None:
71 """
72 Initialize metrics
73 """
74 global counter_storage, histogram_storage # pylint: disable=global-statement
75
76 if enabled:
77 counter_storage = CounterStorage()
78 histogram_storage = HistogramStorage()
79 else:
80 counter_storage = VoidCounterStorage()
81 histogram_storage = HistogramStorage(histogram_class=VoidHistogram)
82
83 # max_timeout = max of all the engine.timeout
84 max_timeout = 2
85 for engine_name in engine_names or engines:
86 if engine_name in engines:
87 max_timeout = max(max_timeout, engines[engine_name].timeout)
88
89 # histogram configuration
90 histogram_width = 0.1
91 histogram_size = int(1.5 * max_timeout / histogram_width)
92
93 # engines
94 for engine_name in engine_names or engines:
95 # search count
96 counter_storage.configure('engine', engine_name, 'search', 'count', 'sent')
97 counter_storage.configure('engine', engine_name, 'search', 'count', 'successful')
98 # global counter of errors
99 counter_storage.configure('engine', engine_name, 'search', 'count', 'error')
100 # score of the engine
101 counter_storage.configure('engine', engine_name, 'score')
102 # result count per requests
103 histogram_storage.configure(1, 100, 'engine', engine_name, 'result', 'count')
104 # time doing HTTP requests
105 histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'http')
106 # total time
107 # .time.request and ...response times may overlap .time.http time.
108 histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'total')
109
110

◆ openmetrics()

searx.metrics.openmetrics ( engine_stats,
engine_reliabilities )

Definition at line 252 of file __init__.py.

252def openmetrics(engine_stats, engine_reliabilities):
253 metrics = [
254 OpenMetricsFamily(
255 key="searxng_engines_response_time_total_seconds",
256 type_hint="gauge",
257 help_hint="The average total response time of the engine",
258 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
259 data=[engine['total'] or 0 for engine in engine_stats['time']],
260 ),
261 OpenMetricsFamily(
262 key="searxng_engines_response_time_processing_seconds",
263 type_hint="gauge",
264 help_hint="The average processing response time of the engine",
265 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
266 data=[engine['processing'] or 0 for engine in engine_stats['time']],
267 ),
268 OpenMetricsFamily(
269 key="searxng_engines_response_time_http_seconds",
270 type_hint="gauge",
271 help_hint="The average HTTP response time of the engine",
272 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
273 data=[engine['http'] or 0 for engine in engine_stats['time']],
274 ),
275 OpenMetricsFamily(
276 key="searxng_engines_result_count_total",
277 type_hint="counter",
278 help_hint="The total amount of results returned by the engine",
279 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
280 data=[engine['result_count'] or 0 for engine in engine_stats['time']],
281 ),
282 OpenMetricsFamily(
283 key="searxng_engines_request_count_total",
284 type_hint="counter",
285 help_hint="The total amount of user requests made to this engine",
286 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
287 data=[
288 engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) or 0
289 for engine in engine_stats['time']
290 ],
291 ),
292 OpenMetricsFamily(
293 key="searxng_engines_reliability_total",
294 type_hint="counter",
295 help_hint="The overall reliability of the engine",
296 data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
297 data=[
298 engine_reliabilities.get(engine['name'], {}).get('reliability', 0) or 0
299 for engine in engine_stats['time']
300 ],
301 ),
302 ]
303 return "".join([str(metric) for metric in metrics])

Variable Documentation

◆ __all__

list searx.metrics.__all__
private
Initial value:
1= [
2 "initialize",
3 "get_engines_stats",
4 "get_engine_errors",
5 "histogram",
6 "histogram_observe",
7 "histogram_observe_time",
8 "counter",
9 "counter_inc",
10 "counter_add",
11 "count_error",
12 "count_exception",
13]

Definition at line 13 of file __init__.py.

◆ counter_storage

CounterStorage searx.metrics.counter_storage = None

Definition at line 32 of file __init__.py.

◆ ENDPOINTS

dict searx.metrics.ENDPOINTS = {'search'}

Definition at line 28 of file __init__.py.

◆ histogram_storage

HistogramStorage searx.metrics.histogram_storage = None

Definition at line 31 of file __init__.py.