Public Member Functions
	__init__ (self, bool enable_http=True, bool verify=True, bool enable_http2=False, int max_connections=None, int max_keepalive_connections=None, float keepalive_expiry=None, str\|dict[str, str]\|None proxies=None, bool using_tor_proxy=False, str\|list[str]\|None local_addresses=None, int retries=0, bool retry_on_http_error=False, int max_redirects=30, str logger_name=None)
	check_parameters (self)
Generator[str]	iter_ipaddresses (self)
	get_ipaddress_cycle (self)
Generator[tuple[str, list[str]]]	iter_proxies (self)
Generator[tuple[tuple[str, str],...], str, str]	get_proxy_cycles (self)
	log_response (self, httpx.Response response)
httpx.AsyncClient	get_client (self, bool\|None verify=None, int\|None max_redirects=None)
	aclose (self)
SXNG_Response	patch_response (self, httpx.Response response, bool do_raise_for_httperror)
	is_valid_response (self, httpx.Response response)
SXNG_Response	call_client (self, bool stream, str method, str url, **t.Any kwargs)
SXNG_Response	request (self, str method, str url, **t.Any kwargs)
	stream (self, str method, str url, **kwargs)
	aclose_all (cls)

Static Public Member Functions
bool	check_tor_proxy (httpx.AsyncClient client, proxies)
dict[str, t.Any]	extract_kwargs_clients (dict[str, t.Any] kwargs)
	extract_do_raise_for_httperror (dict[str, t.Any] kwargs)

Public Attributes
	enable_http = enable_http
	verify = verify
	enable_http2 = enable_http2
	max_connections = max_connections
	max_keepalive_connections = max_keepalive_connections
	keepalive_expiry = keepalive_expiry
	proxies = proxies
	using_tor_proxy = using_tor_proxy
	local_addresses = local_addresses
	retries = retries
	retry_on_http_error = retry_on_http_error
	max_redirects = max_redirects

Protected Attributes
	_local_addresses_cycle = self.get_ipaddress_cycle()
Generator[tuple[tuple[str, str],...], str, str]	_proxies_cycle = self.get_proxy_cycles()
dict	_clients = {}
	_logger = logger.getChild(logger_name) if logger_name else logger

Static Protected Attributes
dict	_TOR_CHECK_RESULT = {}

Static Private Attributes
tuple	__slots__

Detailed Description

Definition at line 45 of file network.py.

Constructor & Destructor Documentation

◆ init()

searx.network.network.Network.__init__	(		self,
		bool	enable_http = True,
		bool	verify = True,
		bool	enable_http2 = False,
		int	max_connections = None,
		int	max_keepalive_connections = None,
		float	keepalive_expiry = None,
		str \| dict[str, str] \| None	proxies = None,
		bool	using_tor_proxy = False,
		str \| list[str] \| None	local_addresses = None,
		int	retries = 0,
		bool	retry_on_http_error = False,
		int	max_redirects = 30,
		str	logger_name = None )

Definition at line 68 of file network.py.

    ):
 
        self.enable_http = enable_http
        self.verify = verify
        self.enable_http2 = enable_http2
        self.max_connections = max_connections
        self.max_keepalive_connections = max_keepalive_connections
        self.keepalive_expiry = keepalive_expiry
        self.proxies = proxies
        self.using_tor_proxy = using_tor_proxy
        self.local_addresses = local_addresses
        self.retries = retries
        self.retry_on_http_error = retry_on_http_error
        self.max_redirects = max_redirects
        self._local_addresses_cycle = self.get_ipaddress_cycle()
        self._proxies_cycle = self.get_proxy_cycles()
        self._clients = {}
        self._logger = logger.getChild(logger_name) if logger_name else logger
        self.check_parameters()
 

Member Function Documentation

◆ aclose()

searx.network.network.Network.aclose ( self )

Definition at line 217 of file network.py.

    async def aclose(self):
        async def close_client(client):
            try:
                await client.aclose()
            except httpx.HTTPError:
                pass
 
        await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
 

References _clients.

◆ aclose_all()

searx.network.network.Network.aclose_all ( cls )

Definition at line 310 of file network.py.

    async def aclose_all(cls):
        await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)

◆ call_client()

SXNG_Response searx.network.network.Network.call_client	(		self,
		bool	stream,
		str	method,
		str	url,
		**t.Any	kwargs )

Definition at line 272 of file network.py.

    async def call_client(self, stream: bool, method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
        retries = self.retries
        was_disconnected = False
        do_raise_for_httperror = Network.extract_do_raise_for_httperror(kwargs)
        kwargs_clients = Network.extract_kwargs_clients(kwargs)
        while retries >= 0:  # pragma: no cover
            client = await self.get_client(**kwargs_clients)
            cookies = kwargs.pop("cookies", None)
            client.cookies = httpx.Cookies(cookies)
            try:
                if stream:
                    return client.stream(method, url, **kwargs)
 
                response = await client.request(method, url, **kwargs)
                if self.is_valid_response(response) or retries <= 0:
                    return self.patch_response(response, do_raise_for_httperror)
            except httpx.RemoteProtocolError as e:
                if not was_disconnected:
                    # the server has closed the connection:
                    # try again without decreasing the retries variable & with a new HTTP client
                    was_disconnected = True
                    await client.aclose()
                    self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
                    continue
                if retries <= 0:
                    raise e
            except (httpx.RequestError, httpx.HTTPStatusError) as e:
                if retries <= 0:
                    raise e
            retries -= 1
 

References _logger, get_client(), is_valid_response(), patch_response(), and retries.

Referenced by request(), and stream().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_parameters()

searx.network.network.Network.check_parameters ( self )

Definition at line 104 of file network.py.

    def check_parameters(self):
        for address in self.iter_ipaddresses():
            if '/' in address:
                ipaddress.ip_network(address, False)
            else:
                ipaddress.ip_address(address)
 
        if self.proxies is not None and not isinstance(self.proxies, (str, dict)):
            raise ValueError('proxies type has to be str, dict or None')
 

References iter_ipaddresses(), searx.enginelib.Engine.proxies, and proxies.

Here is the call graph for this function:

◆ check_tor_proxy()

bool searx.network.network.Network.check_tor_proxy	(	httpx.AsyncClient	client,
			proxies )

static

Definition at line 167 of file network.py.

    async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
        if proxies in Network._TOR_CHECK_RESULT:
            return Network._TOR_CHECK_RESULT[proxies]
 
        result = True
        # ignore client._transport because it is not used with all://
        for transport in client._mounts.values():  # pylint: disable=protected-access
            if isinstance(transport, AsyncHTTPTransportNoHttp):
                continue
            if getattr(transport, "_pool") and getattr(
                # pylint: disable=protected-access
                transport._pool,  # type: ignore
                "_rdns",
                False,
            ):
                continue
            return False
        response = await client.get("https://check.torproject.org/api/ip", timeout=60)
        if not response.json()["IsTor"]:
            result = False
        Network._TOR_CHECK_RESULT[proxies] = result
        return result
 

Referenced by get_client().

Here is the caller graph for this function:

◆ extract_do_raise_for_httperror()

searx.network.network.Network.extract_do_raise_for_httperror ( dict[str, t.Any] kwargs )

static

Definition at line 239 of file network.py.

    def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
        do_raise_for_httperror = True
        if 'raise_for_httperror' in kwargs:
            do_raise_for_httperror = kwargs['raise_for_httperror']
            del kwargs['raise_for_httperror']
        return do_raise_for_httperror
 

◆ extract_kwargs_clients()

dict[str, t.Any] searx.network.network.Network.extract_kwargs_clients ( dict[str, t.Any] kwargs )

static

Definition at line 227 of file network.py.

    def extract_kwargs_clients(kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
        kwargs_clients: dict[str, t.Any] = {}
        if 'verify' in kwargs:
            kwargs_clients['verify'] = kwargs.pop('verify')
        if 'max_redirects' in kwargs:
            kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
        if 'allow_redirects' in kwargs:
            # see https://github.com/encode/httpx/pull/1808
            kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
        return kwargs_clients
 

◆ get_client()

httpx.AsyncClient searx.network.network.Network.get_client	(		self,
		bool \| None	verify = None,
		int \| None	max_redirects = None )

Definition at line 190 of file network.py.

    async def get_client(self, verify: bool | None = None, max_redirects: int | None = None) -> httpx.AsyncClient:
        verify = self.verify if verify is None else verify
        max_redirects = self.max_redirects if max_redirects is None else max_redirects
        local_address = next(self._local_addresses_cycle)
        proxies = next(self._proxies_cycle)  # is a tuple so it can be part of the key
        key = (verify, max_redirects, local_address, proxies)
        hook_log_response = self.log_response if sxng_debug else None
        if key not in self._clients or self._clients[key].is_closed:
            client = new_client(
                self.enable_http,
                verify,
                self.enable_http2,
                self.max_connections,
                self.max_keepalive_connections,
                self.keepalive_expiry,
                dict(proxies),
                local_address,
                0,
                max_redirects,
                hook_log_response,
            )
            if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
                await client.aclose()
                raise httpx.ProxyError('Network configuration problem: not using Tor')
            self._clients[key] = client
        return self._clients[key]
 

References _clients, _local_addresses_cycle, _proxies_cycle, check_tor_proxy(), enable_http, enable_http2, keepalive_expiry, log_response(), max_connections, max_keepalive_connections, max_redirects, using_tor_proxy, and verify.

Referenced by call_client().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ get_ipaddress_cycle()

searx.network.network.Network.get_ipaddress_cycle ( self )

Definition at line 122 of file network.py.

    def get_ipaddress_cycle(self):
        while True:
            count = 0
            for address in self.iter_ipaddresses():
                if '/' in address:
                    for a in ipaddress.ip_network(address, False).hosts():
                        yield str(a)
                        count += 1
                else:
                    a = ipaddress.ip_address(address)
                    yield str(a)
                    count += 1
            if count == 0:
                yield None
 

References iter_ipaddresses().

Here is the call graph for this function:

◆ get_proxy_cycles()

Generator[tuple[tuple[str, str], ...], str, str] searx.network.network.Network.get_proxy_cycles ( self )

Definition at line 150 of file network.py.

    def get_proxy_cycles(self) -> Generator[tuple[tuple[str, str], ...], str, str]:  # not sure type is correct
        proxy_settings: dict[str, t.Any] = {}
        for pattern, proxy_urls in self.iter_proxies():
            proxy_settings[pattern] = cycle(proxy_urls)
        while True:
            # pylint: disable=stop-iteration-return
            yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
 

References iter_proxies().

Here is the call graph for this function:

◆ is_valid_response()

searx.network.network.Network.is_valid_response	(		self,
		httpx.Response	response )

Definition at line 262 of file network.py.

    def is_valid_response(self, response: httpx.Response):
        # pylint: disable=too-many-boolean-expressions
        if (
            (self.retry_on_http_error is True and 400 <= response.status_code <= 599)
            or (isinstance(self.retry_on_http_error, list) and response.status_code in self.retry_on_http_error)
            or (isinstance(self.retry_on_http_error, int) and response.status_code == self.retry_on_http_error)
        ):
            return False
        return True
 

References retry_on_http_error.

Referenced by call_client().

Here is the caller graph for this function:

◆ iter_ipaddresses()

Generator[str] searx.network.network.Network.iter_ipaddresses ( self )

Definition at line 114 of file network.py.

    def iter_ipaddresses(self) -> Generator[str]:
        local_addresses = self.local_addresses
        if not local_addresses:
            return
        if isinstance(local_addresses, str):
            local_addresses = [local_addresses]
        yield from local_addresses
 

References local_addresses.

Referenced by check_parameters(), and get_ipaddress_cycle().

Here is the caller graph for this function:

◆ iter_proxies()

Generator[tuple[str, list[str]]] searx.network.network.Network.iter_proxies ( self )

Definition at line 137 of file network.py.

    def iter_proxies(self) -> Generator[tuple[str, list[str]]]:
        if not self.proxies:
            return
        # https://www.python-httpx.org/compatibility/#proxy-keys
        if isinstance(self.proxies, str):
            yield 'all://', [self.proxies]
        else:
            for pattern, proxy_url in self.proxies.items():
                pattern: str = PROXY_PATTERN_MAPPING.get(pattern, pattern)
                if isinstance(proxy_url, str):
                    proxy_url = [proxy_url]
                yield pattern, proxy_url
 

References searx.enginelib.Engine.proxies, and proxies.

Referenced by get_proxy_cycles().

Here is the caller graph for this function:

◆ log_response()

searx.network.network.Network.log_response	(		self,
		httpx.Response	response )

Definition at line 158 of file network.py.

    async def log_response(self, response: httpx.Response):
        request = response.request
        status = f"{response.status_code} {response.reason_phrase}"
        response_line = f"{response.http_version} {status}"
        content_type = response.headers.get("Content-Type")
        content_type = f' ({content_type})' if content_type else ''
        self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
 

References _logger.

Referenced by get_client().

Here is the caller graph for this function:

◆ patch_response()

SXNG_Response searx.network.network.Network.patch_response	(		self,
		httpx.Response	response,
		bool	do_raise_for_httperror )

Definition at line 246 of file network.py.

    def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
        if isinstance(response, httpx.Response):
            response = t.cast(SXNG_Response, response)
            # requests compatibility (response is not streamed)
            # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
            response.ok = not response.is_error
 
            # raise an exception
            if do_raise_for_httperror:
                try:
                    raise_for_httperror(response)
                except:
                    self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}")
                    raise
        return response
 

References _logger.

Referenced by call_client().

Here is the caller graph for this function:

◆ request()

SXNG_Response searx.network.network.Network.request	(		self,
		str	method,
		str	url,
		**t.Any	kwargs )

Definition at line 303 of file network.py.

    async def request(self, method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
        return await self.call_client(False, method, url, **kwargs)
 

References call_client().

Here is the call graph for this function:

◆ stream()

searx.network.network.Network.stream	(		self,
		str	method,
		str	url,
		**	kwargs )

Definition at line 306 of file network.py.

    async def stream(self, method: str, url: str, **kwargs):
        return await self.call_client(True, method, url, **kwargs)
 

References call_client().

Here is the call graph for this function:

Member Data Documentation

◆ slots

tuple searx.network.network.Network.__slots__

staticprivate

Initial value:

=  (
        'enable_http',
        'verify',
        'enable_http2',
        'max_connections',
        'max_keepalive_connections',
        'keepalive_expiry',
        'local_addresses',
        'proxies',
        'using_tor_proxy',
        'max_redirects',
        'retries',
        'retry_on_http_error',
        '_local_addresses_cycle',
        '_proxies_cycle',
        '_clients',
        '_logger',
    )

Definition at line 47 of file network.py.

◆ _clients

dict searx.network.network.Network._clients = {}

protected

Definition at line 100 of file network.py.

Referenced by aclose(), and get_client().

◆ _local_addresses_cycle

searx.network.network.Network._local_addresses_cycle = self.get_ipaddress_cycle()

protected

Definition at line 98 of file network.py.

Referenced by get_client().

◆ _logger

searx.network.network.Network._logger = logger.getChild(logger_name) if logger_name else logger

protected

Definition at line 101 of file network.py.

Referenced by call_client(), log_response(), and patch_response().

◆ _proxies_cycle

Generator[tuple[tuple[str, str], ...], str, str] searx.network.network.Network._proxies_cycle = self.get_proxy_cycles()

protected

Definition at line 99 of file network.py.

Referenced by get_client().

◆ _TOR_CHECK_RESULT

dict searx.network.network.Network._TOR_CHECK_RESULT = {}

staticprotected

Definition at line 66 of file network.py.

◆ enable_http

searx.network.network.Network.enable_http = enable_http

Definition at line 86 of file network.py.

Referenced by get_client().

◆ enable_http2

searx.network.network.Network.enable_http2 = enable_http2

Definition at line 88 of file network.py.

Referenced by get_client().

◆ keepalive_expiry

searx.network.network.Network.keepalive_expiry = keepalive_expiry

Definition at line 91 of file network.py.

Referenced by get_client().

◆ local_addresses

searx.network.network.Network.local_addresses = local_addresses

Definition at line 94 of file network.py.

Referenced by iter_ipaddresses().

◆ max_connections

searx.network.network.Network.max_connections = max_connections

Definition at line 89 of file network.py.

Referenced by get_client().

◆ max_keepalive_connections

searx.network.network.Network.max_keepalive_connections = max_keepalive_connections

Definition at line 90 of file network.py.

Referenced by get_client().

◆ max_redirects

searx.network.network.Network.max_redirects = max_redirects

Definition at line 97 of file network.py.

Referenced by get_client().

◆ proxies

searx.network.network.Network.proxies = proxies

Definition at line 92 of file network.py.

Referenced by check_parameters(), and iter_proxies().

◆ retries

searx.network.network.Network.retries = retries

Definition at line 95 of file network.py.

Referenced by call_client().

◆ retry_on_http_error

searx.network.network.Network.retry_on_http_error = retry_on_http_error

Definition at line 96 of file network.py.

Referenced by is_valid_response().

◆ using_tor_proxy

searx.network.network.Network.using_tor_proxy = using_tor_proxy

Definition at line 93 of file network.py.

Referenced by get_client().

◆ verify

searx.network.network.Network.verify = verify

Definition at line 87 of file network.py.

Referenced by get_client().

The documentation for this class was generated from the following file:

/home/andrew/Documents/code/public/searxng/searx/network/network.py

Public Member Functions

Static Public Member Functions

Public Attributes

Protected Attributes

Static Protected Attributes

Static Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ aclose()

◆ aclose_all()

◆ call_client()

◆ check_parameters()

◆ check_tor_proxy()

◆ extract_do_raise_for_httperror()

◆ extract_kwargs_clients()

◆ get_client()

◆ get_ipaddress_cycle()

◆ get_proxy_cycles()

◆ is_valid_response()

◆ iter_ipaddresses()

◆ iter_proxies()

◆ log_response()

◆ patch_response()

◆ request()

◆ stream()

Member Data Documentation

◆ __slots__

◆ _clients

◆ _local_addresses_cycle

◆ _logger

◆ _proxies_cycle

◆ _TOR_CHECK_RESULT

◆ enable_http

◆ enable_http2

◆ keepalive_expiry

◆ local_addresses

◆ max_connections

◆ max_keepalive_connections

◆ max_redirects

◆ proxies

◆ retries

◆ retry_on_http_error

◆ using_tor_proxy

◆ verify

◆ init()

◆ slots