searx.network.network Namespace Reference

Classes
class	Network

Functions
	get_network (name=None)
	check_network_configuration ()
	initialize (settings_engines=None, settings_outgoing=None)
	done ()

Variables
	logger = logger.getChild('network')
str	DEFAULT_NAME = '__DEFAULT__'
dict	NETWORKS = {}
dict	PROXY_PATTERN_MAPPING
dict	ADDRESS_MAPPING = {'ipv4': '0.0.0.0', 'ipv6': '::'}

Function Documentation

check_network_configuration()

searx.network.network.check_network_configuration

(

)

Definition at line 311 of file network.py.

def check_network_configuration():
    async def check():
        exception_count = 0
        for network in NETWORKS.values():
            if network.using_tor_proxy:
                try:
                    await network.get_client()
                except Exception:  # pylint: disable=broad-except
                    network._logger.exception('Error')  # pylint: disable=protected-access
                    exception_count += 1
        return exception_count
 
    future = asyncio.run_coroutine_threadsafe(check(), get_loop())
    exception_count = future.result()
    if exception_count > 0:
        raise RuntimeError("Invalid network configuration")
 
 

done()

searx.network.network.done

(

)

Close all HTTP client

Avoid a warning at exit
See https://github.com/encode/httpx/pull/2026

Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register

Definition at line 414 of file network.py.

def done():
    """Close all HTTP client
 
    Avoid a warning at exit
    See https://github.com/encode/httpx/pull/2026
 
    Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
    So Network.aclose is called here using atexit.register
    """
    try:
        loop = get_loop()
        if loop:
            future = asyncio.run_coroutine_threadsafe(Network.aclose_all(), loop)
            # wait 3 seconds to close the HTTP clients
            future.result(3)
    finally:
        NETWORKS.clear()
 
 

Referenced by initialize().

Here is the caller graph for this function:

get_network()

searx.network.network.get_network

(

name = None

)

Definition at line 307 of file network.py.

def get_network(name=None):
    return NETWORKS.get(name or DEFAULT_NAME)
 
 

initialize()

searx.network.network.initialize	(		settings_engines = None,
			settings_outgoing = None )

Definition at line 329 of file network.py.

def initialize(settings_engines=None, settings_outgoing=None):
    # pylint: disable=import-outside-toplevel)
    from searx.engines import engines
    from searx import settings
 
    # pylint: enable=import-outside-toplevel)
 
    settings_engines = settings_engines or settings['engines']
    settings_outgoing = settings_outgoing or settings['outgoing']
 
    # default parameters for AsyncHTTPTransport
    # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121  # pylint: disable=line-too-long
    default_params = {
        'enable_http': False,
        'verify': settings_outgoing['verify'],
        'enable_http2': settings_outgoing['enable_http2'],
        'max_connections': settings_outgoing['pool_connections'],
        'max_keepalive_connections': settings_outgoing['pool_maxsize'],
        'keepalive_expiry': settings_outgoing['keepalive_expiry'],
        'local_addresses': settings_outgoing['source_ips'],
        'using_tor_proxy': settings_outgoing['using_tor_proxy'],
        'proxies': settings_outgoing['proxies'],
        'max_redirects': settings_outgoing['max_redirects'],
        'retries': settings_outgoing['retries'],
        'retry_on_http_error': None,
    }
 
    def new_network(params, logger_name=None):
        nonlocal default_params
        result = {}
        result.update(default_params)
        result.update(params)
        if logger_name:
            result['logger_name'] = logger_name
        return Network(**result)
 
    def iter_networks():
        nonlocal settings_engines
        for engine_spec in settings_engines:
            engine_name = engine_spec['name']
            engine = engines.get(engine_name)
            if engine is None:
                continue
            network = getattr(engine, 'network', None)
            yield engine_name, engine, network
 
    if NETWORKS:
        done()
    NETWORKS.clear()
    NETWORKS[DEFAULT_NAME] = new_network({}, logger_name='default')
    NETWORKS['ipv4'] = new_network({'local_addresses': '0.0.0.0'}, logger_name='ipv4')
    NETWORKS['ipv6'] = new_network({'local_addresses': '::'}, logger_name='ipv6')
 
    # define networks from outgoing.networks
    for network_name, network in settings_outgoing['networks'].items():
        NETWORKS[network_name] = new_network(network, logger_name=network_name)
 
    # define networks from engines.[i].network (except references)
    for engine_name, engine, network in iter_networks():
        if network is None:
            network = {}
            for attribute_name, attribute_value in default_params.items():
                if hasattr(engine, attribute_name):
                    network[attribute_name] = getattr(engine, attribute_name)
                else:
                    network[attribute_name] = attribute_value
            NETWORKS[engine_name] = new_network(network, logger_name=engine_name)
        elif isinstance(network, dict):
            NETWORKS[engine_name] = new_network(network, logger_name=engine_name)
 
    # define networks from engines.[i].network (references)
    for engine_name, engine, network in iter_networks():
        if isinstance(network, str):
            NETWORKS[engine_name] = NETWORKS[network]
 
    # the /image_proxy endpoint has a dedicated network.
    # same parameters than the default network, but HTTP/2 is disabled.
    # It decreases the CPU load average, and the total time is more or less the same
    if 'image_proxy' not in NETWORKS:
        image_proxy_params = default_params.copy()
        image_proxy_params['enable_http2'] = False
        NETWORKS['image_proxy'] = new_network(image_proxy_params, logger_name='image_proxy')
 
 
@atexit.register

References done().

Here is the call graph for this function:

Variable Documentation

ADDRESS_MAPPING

dict searx.network.network.ADDRESS_MAPPING = {'ipv4': '0.0.0.0', 'ipv6': '::'}

Definition at line 38 of file network.py.

DEFAULT_NAME

str searx.network.network.DEFAULT_NAME = '__DEFAULT__'

Definition at line 22 of file network.py.

logger

searx.network.network.logger = logger.getChild('network')

Definition at line 21 of file network.py.

NETWORKS

dict searx.network.network.NETWORKS = {}

Definition at line 23 of file network.py.

PROXY_PATTERN_MAPPING

dict searx.network.network.PROXY_PATTERN_MAPPING

Initial value:

=  {
    'http': 'http://',
    'https': 'https://',
    'socks4': 'socks4://',
    'socks5': 'socks5://',
    'socks5h': 'socks5h://',
    'http:': 'http://',
    'https:': 'https://',
    'socks4:': 'socks4://',
    'socks5:': 'socks5://',
    'socks5h:': 'socks5h://',
}

Definition at line 25 of file network.py.