Public Member Functions
	__init__ (self, enable_http=True, verify=True, enable_http2=False, max_connections=None, max_keepalive_connections=None, keepalive_expiry=None, proxies=None, using_tor_proxy=False, local_addresses=None, retries=0, retry_on_http_error=None, max_redirects=30, logger_name=None)

	check_parameters (self)

	iter_ipaddresses (self)

	get_ipaddress_cycle (self)

	iter_proxies (self)

	get_proxy_cycles (self)

	log_response (self, httpx.Response response)

	get_client (self, verify=None, max_redirects=None)

	aclose (self)

	is_valid_response (self, response)

	call_client (self, stream, method, url, **kwargs)

	request (self, method, url, **kwargs)

	stream (self, method, url, **kwargs)

	aclose_all (cls)

Static Public Member Functions
bool	check_tor_proxy (httpx.AsyncClient client, proxies)

	extract_kwargs_clients (kwargs)

	extract_do_raise_for_httperror (kwargs)

	patch_response (response, do_raise_for_httperror)

Public Attributes
	enable_http

	verify

	enable_http2

	max_connections

	max_keepalive_connections

	keepalive_expiry

	proxies

	using_tor_proxy

	local_addresses

	retries

	retry_on_http_error

	max_redirects

Protected Attributes
	_local_addresses_cycle

	_proxies_cycle

	_clients

	_logger

Static Protected Attributes
dict	_TOR_CHECK_RESULT = {}

Static Private Attributes
tuple	__slots__

Detailed Description

Definition at line 38 of file network.py.

Constructor & Destructor Documentation

◆ init()

searx.network.network.Network.__init__	(	self,
		enable_http = True,
		verify = True,
		enable_http2 = False,
		max_connections = None,
		max_keepalive_connections = None,
		keepalive_expiry = None,
		proxies = None,
		using_tor_proxy = False,
		local_addresses = None,
		retries = 0,
		retry_on_http_error = None,
		max_redirects = 30,
		logger_name = None )

Definition at line 61 of file network.py.

    ):
 
        self.enable_http = enable_http
        self.verify = verify
        self.enable_http2 = enable_http2
        self.max_connections = max_connections
        self.max_keepalive_connections = max_keepalive_connections
        self.keepalive_expiry = keepalive_expiry
        self.proxies = proxies
        self.using_tor_proxy = using_tor_proxy
        self.local_addresses = local_addresses
        self.retries = retries
        self.retry_on_http_error = retry_on_http_error
        self.max_redirects = max_redirects
        self._local_addresses_cycle = self.get_ipaddress_cycle()
        self._proxies_cycle = self.get_proxy_cycles()
        self._clients = {}
        self._logger = logger.getChild(logger_name) if logger_name else logger
        self.check_parameters()
 

Member Function Documentation

◆ aclose()

searx.network.network.Network.aclose ( self )

Definition at line 207 of file network.py.

    async def aclose(self):
        async def close_client(client):
            try:
                await client.aclose()
            except httpx.HTTPError:
                pass
 
        await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
 

References searx.network.network.Network._clients.

◆ aclose_all()

searx.network.network.Network.aclose_all ( cls )

Definition at line 295 of file network.py.

    async def aclose_all(cls):
        await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)

◆ call_client()

searx.network.network.Network.call_client	(		self,
			stream,
			method,
			url,
		**	kwargs )

Definition at line 259 of file network.py.

    async def call_client(self, stream, method, url, **kwargs):
        retries = self.retries
        was_disconnected = False
        do_raise_for_httperror = Network.extract_do_raise_for_httperror(kwargs)
        kwargs_clients = Network.extract_kwargs_clients(kwargs)
        while retries >= 0:  # pragma: no cover
            client = await self.get_client(**kwargs_clients)
            try:
                if stream:
                    response = client.stream(method, url, **kwargs)
                else:
                    response = await client.request(method, url, **kwargs)
                if self.is_valid_response(response) or retries <= 0:
                    return Network.patch_response(response, do_raise_for_httperror)
            except httpx.RemoteProtocolError as e:
                if not was_disconnected:
                    # the server has closed the connection:
                    # try again without decreasing the retries variable & with a new HTTP client
                    was_disconnected = True
                    await client.aclose()
                    self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
                    continue
                if retries <= 0:
                    raise e
            except (httpx.RequestError, httpx.HTTPStatusError) as e:
                if retries <= 0:
                    raise e
            retries -= 1
 

References searx.network.network.Network._logger, searx.network.network.Network.get_client(), searx.network.network.Network.is_valid_response(), and searx.network.network.Network.retries.

Referenced by searx.network.network.Network.request(), and searx.network.network.Network.stream().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_parameters()

searx.network.network.Network.check_parameters ( self )

Definition at line 97 of file network.py.

    def check_parameters(self):
        for address in self.iter_ipaddresses():
            if '/' in address:
                ipaddress.ip_network(address, False)
            else:
                ipaddress.ip_address(address)
 
        if self.proxies is not None and not isinstance(self.proxies, (str, dict)):
            raise ValueError('proxies type has to be str, dict or None')
 

References searx.network.network.Network.iter_ipaddresses(), searx.enginelib.Engine.proxies, and searx.network.network.Network.proxies.

Here is the call graph for this function:

◆ check_tor_proxy()

bool searx.network.network.Network.check_tor_proxy	(	httpx.AsyncClient	client,
			proxies )

static

Definition at line 160 of file network.py.

    async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
        if proxies in Network._TOR_CHECK_RESULT:
            return Network._TOR_CHECK_RESULT[proxies]
 
        result = True
        # ignore client._transport because it is not used with all://
        for transport in client._mounts.values():  # pylint: disable=protected-access
            if isinstance(transport, AsyncHTTPTransportNoHttp):
                continue
            if getattr(transport, "_pool") and getattr(
                transport._pool, "_rdns", False  # pylint: disable=protected-access
            ):
                continue
            return False
        response = await client.get("https://check.torproject.org/api/ip", timeout=60)
        if not response.json()["IsTor"]:
            result = False
        Network._TOR_CHECK_RESULT[proxies] = result
        return result
 

Referenced by searx.network.network.Network.get_client().

Here is the caller graph for this function:

◆ extract_do_raise_for_httperror()

searx.network.network.Network.extract_do_raise_for_httperror ( kwargs )

static

Definition at line 229 of file network.py.

    def extract_do_raise_for_httperror(kwargs):
        do_raise_for_httperror = True
        if 'raise_for_httperror' in kwargs:
            do_raise_for_httperror = kwargs['raise_for_httperror']
            del kwargs['raise_for_httperror']
        return do_raise_for_httperror
 

◆ extract_kwargs_clients()

searx.network.network.Network.extract_kwargs_clients ( kwargs )

static

Definition at line 217 of file network.py.

    def extract_kwargs_clients(kwargs):
        kwargs_clients = {}
        if 'verify' in kwargs:
            kwargs_clients['verify'] = kwargs.pop('verify')
        if 'max_redirects' in kwargs:
            kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
        if 'allow_redirects' in kwargs:
            # see https://github.com/encode/httpx/pull/1808
            kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
        return kwargs_clients
 

◆ get_client()

searx.network.network.Network.get_client	(	self,
		verify = None,
		max_redirects = None )

Definition at line 180 of file network.py.

    async def get_client(self, verify=None, max_redirects=None):
        verify = self.verify if verify is None else verify
        max_redirects = self.max_redirects if max_redirects is None else max_redirects
        local_address = next(self._local_addresses_cycle)
        proxies = next(self._proxies_cycle)  # is a tuple so it can be part of the key
        key = (verify, max_redirects, local_address, proxies)
        hook_log_response = self.log_response if searx_debug else None
        if key not in self._clients or self._clients[key].is_closed:
            client = new_client(
                self.enable_http,
                verify,
                self.enable_http2,
                self.max_connections,
                self.max_keepalive_connections,
                self.keepalive_expiry,
                dict(proxies),
                local_address,
                0,
                max_redirects,
                hook_log_response,
            )
            if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
                await client.aclose()
                raise httpx.ProxyError('Network configuration problem: not using Tor')
            self._clients[key] = client
        return self._clients[key]
 

Referenced by searx.network.network.Network.call_client().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ get_ipaddress_cycle()

searx.network.network.Network.get_ipaddress_cycle ( self )

Definition at line 115 of file network.py.

    def get_ipaddress_cycle(self):
        while True:
            count = 0
            for address in self.iter_ipaddresses():
                if '/' in address:
                    for a in ipaddress.ip_network(address, False).hosts():
                        yield str(a)
                        count += 1
                else:
                    a = ipaddress.ip_address(address)
                    yield str(a)
                    count += 1
            if count == 0:
                yield None
 

References searx.network.network.Network.iter_ipaddresses().

Here is the call graph for this function:

◆ get_proxy_cycles()

searx.network.network.Network.get_proxy_cycles ( self )

Definition at line 143 of file network.py.

    def get_proxy_cycles(self):
        proxy_settings = {}
        for pattern, proxy_urls in self.iter_proxies():
            proxy_settings[pattern] = cycle(proxy_urls)
        while True:
            # pylint: disable=stop-iteration-return
            yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
 

References searx.network.network.Network.iter_proxies().

Here is the call graph for this function:

◆ is_valid_response()

searx.network.network.Network.is_valid_response	(		self,
			response )

Definition at line 249 of file network.py.

    def is_valid_response(self, response):
        # pylint: disable=too-many-boolean-expressions
        if (
            (self.retry_on_http_error is True and 400 <= response.status_code <= 599)
            or (isinstance(self.retry_on_http_error, list) and response.status_code in self.retry_on_http_error)
            or (isinstance(self.retry_on_http_error, int) and response.status_code == self.retry_on_http_error)
        ):
            return False
        return True
 

References searx.network.network.Network.retry_on_http_error.

Referenced by searx.network.network.Network.call_client().

Here is the caller graph for this function:

◆ iter_ipaddresses()

searx.network.network.Network.iter_ipaddresses ( self )

Definition at line 107 of file network.py.

    def iter_ipaddresses(self):
        local_addresses = self.local_addresses
        if not local_addresses:
            return
        if isinstance(local_addresses, str):
            local_addresses = [local_addresses]
        yield from local_addresses
 

References searx.network.network.Network.local_addresses.

Referenced by searx.network.network.Network.check_parameters(), and searx.network.network.Network.get_ipaddress_cycle().

Here is the caller graph for this function:

◆ iter_proxies()

searx.network.network.Network.iter_proxies ( self )

Definition at line 130 of file network.py.

    def iter_proxies(self):
        if not self.proxies:
            return
        # https://www.python-httpx.org/compatibility/#proxy-keys
        if isinstance(self.proxies, str):
            yield 'all://', [self.proxies]
        else:
            for pattern, proxy_url in self.proxies.items():
                pattern = PROXY_PATTERN_MAPPING.get(pattern, pattern)
                if isinstance(proxy_url, str):
                    proxy_url = [proxy_url]
                yield pattern, proxy_url
 

References searx.enginelib.Engine.proxies, and searx.network.network.Network.proxies.

Referenced by searx.network.network.Network.get_proxy_cycles().

Here is the caller graph for this function:

◆ log_response()

searx.network.network.Network.log_response	(		self,
		httpx.Response	response )

Definition at line 151 of file network.py.

    async def log_response(self, response: httpx.Response):
        request = response.request
        status = f"{response.status_code} {response.reason_phrase}"
        response_line = f"{response.http_version} {status}"
        content_type = response.headers.get("Content-Type")
        content_type = f' ({content_type})' if content_type else ''
        self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
 

References searx.network.network.Network._logger.

Referenced by searx.network.network.Network.get_client().

Here is the caller graph for this function:

◆ patch_response()

searx.network.network.Network.patch_response	(		response,
			do_raise_for_httperror )

static

Definition at line 237 of file network.py.

    def patch_response(response, do_raise_for_httperror):
        if isinstance(response, httpx.Response):
            # requests compatibility (response is not streamed)
            # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
            response.ok = not response.is_error
 
            # raise an exception
            if do_raise_for_httperror:
                raise_for_httperror(response)
 
        return response
 

◆ request()

searx.network.network.Network.request	(		self,
			method,
			url,
		**	kwargs )

Definition at line 288 of file network.py.

    async def request(self, method, url, **kwargs):
        return await self.call_client(False, method, url, **kwargs)
 

References searx.network.network.Network.call_client().

Here is the call graph for this function:

◆ stream()

searx.network.network.Network.stream	(		self,
			method,
			url,
		**	kwargs )

Definition at line 291 of file network.py.

    async def stream(self, method, url, **kwargs):
        return await self.call_client(True, method, url, **kwargs)
 

References searx.network.network.Network.call_client().

Referenced by searx.webutils.CSVWriter.writerow().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ slots

tuple searx.network.network.Network.__slots__

staticprivate

Initial value:

=  (
        'enable_http',
        'verify',
        'enable_http2',
        'max_connections',
        'max_keepalive_connections',
        'keepalive_expiry',
        'local_addresses',
        'proxies',
        'using_tor_proxy',
        'max_redirects',
        'retries',
        'retry_on_http_error',
        '_local_addresses_cycle',
        '_proxies_cycle',
        '_clients',
        '_logger',
    )

Definition at line 40 of file network.py.

◆ _clients

searx.network.network.Network._clients

protected

Definition at line 93 of file network.py.

Referenced by searx.network.network.Network.aclose(), and searx.network.network.Network.get_client().

◆ _local_addresses_cycle

searx.network.network.Network._local_addresses_cycle

protected

Definition at line 91 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ _logger

searx.network.network.Network._logger

protected

Definition at line 94 of file network.py.

Referenced by searx.network.network.Network.call_client(), and searx.network.network.Network.log_response().

◆ _proxies_cycle

searx.network.network.Network._proxies_cycle

protected

Definition at line 92 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ _TOR_CHECK_RESULT

dict searx.network.network.Network._TOR_CHECK_RESULT = {}

staticprotected

Definition at line 59 of file network.py.

◆ enable_http

searx.network.network.Network.enable_http

Definition at line 79 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ enable_http2

searx.network.network.Network.enable_http2

Definition at line 81 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ keepalive_expiry

searx.network.network.Network.keepalive_expiry

Definition at line 84 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ local_addresses

searx.network.network.Network.local_addresses

Definition at line 87 of file network.py.

Referenced by searx.network.network.Network.iter_ipaddresses().

◆ max_connections

searx.network.network.Network.max_connections

Definition at line 82 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ max_keepalive_connections

searx.network.network.Network.max_keepalive_connections

Definition at line 83 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ max_redirects

searx.network.network.Network.max_redirects

Definition at line 90 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ proxies

searx.network.network.Network.proxies

Definition at line 85 of file network.py.

Referenced by searx.network.network.Network.check_parameters(), and searx.network.network.Network.iter_proxies().

◆ retries

searx.network.network.Network.retries

Definition at line 88 of file network.py.

Referenced by searx.network.network.Network.call_client().

◆ retry_on_http_error

searx.network.network.Network.retry_on_http_error

Definition at line 89 of file network.py.

Referenced by searx.network.network.Network.is_valid_response().

◆ using_tor_proxy

searx.network.network.Network.using_tor_proxy

Definition at line 86 of file network.py.

Referenced by searx.network.network.Network.get_client().

◆ verify

searx.network.network.Network.verify

Definition at line 80 of file network.py.

Referenced by searx.network.network.Network.get_client().

The documentation for this class was generated from the following file:

/home/andrew/Documents/code/public/searxng/searx/network/network.py

Public Member Functions

Static Public Member Functions

Public Attributes

Protected Attributes

Static Protected Attributes

Static Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ aclose()

◆ aclose_all()

◆ call_client()

◆ check_parameters()

◆ check_tor_proxy()

◆ extract_do_raise_for_httperror()

◆ extract_kwargs_clients()

◆ get_client()

◆ get_ipaddress_cycle()

◆ get_proxy_cycles()

◆ is_valid_response()

◆ iter_ipaddresses()

◆ iter_proxies()

◆ log_response()

◆ patch_response()

◆ request()

◆ stream()

Member Data Documentation

◆ __slots__

◆ _clients

◆ _local_addresses_cycle

◆ _logger

◆ _proxies_cycle

◆ _TOR_CHECK_RESULT

◆ enable_http

◆ enable_http2

◆ keepalive_expiry

◆ local_addresses

◆ max_connections

◆ max_keepalive_connections

◆ max_redirects

◆ proxies

◆ retries

◆ retry_on_http_error

◆ using_tor_proxy

◆ verify

◆ init()

◆ slots