.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
searx.network.network Namespace Reference

Classes

class  Network

Functions

"Network" get_network (str|None name=None)
 check_network_configuration ()
None initialize (list[dict[str, t.Any]] settings_engines=None, dict[str, t.Any] settings_outgoing=None)
 done ()

Variables

 logger = logger.getChild('network')
str DEFAULT_NAME = '__DEFAULT__'
dict NETWORKS = {}
dict PROXY_PATTERN_MAPPING
dict ADDRESS_MAPPING = {'ipv4': '0.0.0.0', 'ipv6': '::'}

Function Documentation

◆ check_network_configuration()

searx.network.network.check_network_configuration ( )

Definition at line 316 of file network.py.

316def check_network_configuration():
317 async def check():
318 exception_count = 0
319 for network in NETWORKS.values():
320 if network.using_tor_proxy:
321 try:
322 await network.get_client()
323 except Exception: # pylint: disable=broad-except
324 network._logger.exception('Error') # pylint: disable=protected-access
325 exception_count += 1
326 return exception_count
327
328 future = asyncio.run_coroutine_threadsafe(check(), get_loop())
329 exception_count = future.result()
330 if exception_count > 0:
331 raise RuntimeError("Invalid network configuration")
332
333

◆ done()

searx.network.network.done ( )
Close all HTTP client

Avoid a warning at exit
See https://github.com/encode/httpx/pull/2026

Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register

Definition at line 422 of file network.py.

422def done():
423 """Close all HTTP client
424
425 Avoid a warning at exit
426 See https://github.com/encode/httpx/pull/2026
427
428 Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
429 So Network.aclose is called here using atexit.register
430 """
431 try:
432 loop = get_loop()
433 if loop:
434 future = asyncio.run_coroutine_threadsafe(Network.aclose_all(), loop)
435 # wait 3 seconds to close the HTTP clients
436 future.result(3)
437 finally:
438 NETWORKS.clear()
439
440

Referenced by initialize().

Here is the caller graph for this function:

◆ get_network()

"Network" searx.network.network.get_network ( str | None name = None)

Definition at line 312 of file network.py.

312def get_network(name: str | None = None) -> "Network":
313 return NETWORKS.get(name or DEFAULT_NAME) # pyright: ignore[reportReturnType]
314
315

◆ initialize()

None searx.network.network.initialize ( list[dict[str, t.Any]] settings_engines = None,
dict[str, t.Any] settings_outgoing = None )

Definition at line 334 of file network.py.

337) -> None:
338 # pylint: disable=import-outside-toplevel)
339 from searx.engines import engines
340 from searx import settings
341
342 # pylint: enable=import-outside-toplevel)
343
344 settings_engines = settings_engines or settings['engines']
345 settings_outgoing = settings_outgoing or settings['outgoing']
346
347 # default parameters for AsyncHTTPTransport
348 # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
349 default_params: dict[str, t.Any] = {
350 'enable_http': False,
351 'verify': settings_outgoing['verify'],
352 'enable_http2': settings_outgoing['enable_http2'],
353 'max_connections': settings_outgoing['pool_connections'],
354 'max_keepalive_connections': settings_outgoing['pool_maxsize'],
355 'keepalive_expiry': settings_outgoing['keepalive_expiry'],
356 'local_addresses': settings_outgoing['source_ips'],
357 'using_tor_proxy': settings_outgoing['using_tor_proxy'],
358 'proxies': settings_outgoing['proxies'],
359 'max_redirects': settings_outgoing['max_redirects'],
360 'retries': settings_outgoing['retries'],
361 'retry_on_http_error': None,
362 }
363
364 def new_network(params: dict[str, t.Any], logger_name: str | None = None):
365 nonlocal default_params
366 result = {}
367 result.update(default_params) # pyright: ignore[reportUnknownMemberType]
368 result.update(params) # pyright: ignore[reportUnknownMemberType]
369 if logger_name:
370 result['logger_name'] = logger_name
371 return Network(**result) # type: ignore
372
373 def iter_networks():
374 nonlocal settings_engines
375 for engine_spec in settings_engines:
376 engine_name = engine_spec['name']
377 engine = engines.get(engine_name)
378 if engine is None:
379 continue
380 network = getattr(engine, 'network', None)
381 yield engine_name, engine, network
382
383 if NETWORKS:
384 done()
385 NETWORKS.clear()
386 NETWORKS[DEFAULT_NAME] = new_network({}, logger_name='default')
387 NETWORKS['ipv4'] = new_network({'local_addresses': '0.0.0.0'}, logger_name='ipv4')
388 NETWORKS['ipv6'] = new_network({'local_addresses': '::'}, logger_name='ipv6')
389
390 # define networks from outgoing.networks
391 for network_name, network in settings_outgoing['networks'].items():
392 NETWORKS[network_name] = new_network(network, logger_name=network_name)
393
394 # define networks from engines.[i].network (except references)
395 for engine_name, engine, network in iter_networks():
396 if network is None:
397 network = {}
398 for attribute_name, attribute_value in default_params.items():
399 if hasattr(engine, attribute_name):
400 network[attribute_name] = getattr(engine, attribute_name)
401 else:
402 network[attribute_name] = attribute_value
403 NETWORKS[engine_name] = new_network(network, logger_name=engine_name)
404 elif isinstance(network, dict):
405 NETWORKS[engine_name] = new_network(network, logger_name=engine_name)
406
407 # define networks from engines.[i].network (references)
408 for engine_name, engine, network in iter_networks():
409 if isinstance(network, str):
410 NETWORKS[engine_name] = NETWORKS[network]
411
412 # the /image_proxy endpoint has a dedicated network.
413 # same parameters than the default network, but HTTP/2 is disabled.
414 # It decreases the CPU load average, and the total time is more or less the same
415 if 'image_proxy' not in NETWORKS:
416 image_proxy_params = default_params.copy()
417 image_proxy_params['enable_http2'] = False
418 NETWORKS['image_proxy'] = new_network(image_proxy_params, logger_name='image_proxy')
419
420
421@atexit.register
::1337x
Definition 1337x.py:1

References done().

Here is the call graph for this function:

Variable Documentation

◆ ADDRESS_MAPPING

dict searx.network.network.ADDRESS_MAPPING = {'ipv4': '0.0.0.0', 'ipv6': '::'}

Definition at line 37 of file network.py.

◆ DEFAULT_NAME

str searx.network.network.DEFAULT_NAME = '__DEFAULT__'

Definition at line 21 of file network.py.

◆ logger

searx.network.network.logger = logger.getChild('network')

Definition at line 20 of file network.py.

◆ NETWORKS

dict searx.network.network.NETWORKS = {}

Definition at line 22 of file network.py.

◆ PROXY_PATTERN_MAPPING

dict searx.network.network.PROXY_PATTERN_MAPPING
Initial value:
1= {
2 'http': 'http://',
3 'https': 'https://',
4 'socks4': 'socks4://',
5 'socks5': 'socks5://',
6 'socks5h': 'socks5h://',
7 'http:': 'http://',
8 'https:': 'https://',
9 'socks4:': 'socks4://',
10 'socks5:': 'socks5://',
11 'socks5h:': 'socks5h://',
12}

Definition at line 24 of file network.py.