searx.result_types._base Namespace Reference

Classes
class	LegacyResult
class	MainResult
class	Result

Functions
	_normalize_url_fields ("Result | LegacyResult" result)
	_normalize_text_fields ("MainResult | LegacyResult" result)
	_filter_urls ("Result | LegacyResult" result, "Callable[[Result | LegacyResult, str, str], str | bool]" filter_func)
	_normalize_date_fields ("MainResult | LegacyResult" result)

Variables
list	__all__ = ["Result"]
	WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
	UNKNOWN = object()

Detailed Description

Basic types for the typification of results.

- :py:obj:`Result` base class
- :py:obj:`LegacyResult` for internal use only

----

.. autoclass:: Result
   :members:

.. _LegacyResult:

.. autoclass:: LegacyResult
   :members:

Function Documentation

_filter_urls()

searx.result_types._base._filter_urls ( "Result | LegacyResult" result, "Callable[[Result | LegacyResult, str, str], str | bool]" filter_func )

protected

Definition at line 112 of file _base.py.

):
    # pylint: disable=too-many-branches, too-many-statements
 
    # As soon we need LegacyResult not any longer, we can move this function to
    # method Result.
 
    url_fields = ["url", "iframe_src", "audio_src", "img_src", "thumbnail_src", "thumbnail"]
 
    url_src: str
 
    for field_name in url_fields:
        url_src = getattr(result, field_name, "")
        if not url_src:
            continue
 
        new_url = filter_func(result, field_name, url_src)
        # log.debug("filter_urls: filter_func(result, %s) '%s' -> '%s'", field_name, field_value, new_url)
        if isinstance(new_url, bool):
            if new_url:
                # log.debug("filter_urls: unchanged field %s URL %s", field_name, field_value)
                continue
            log.debug("filter_urls: drop field %s URL %s", field_name, url_src)
            new_url = None
        else:
            log.debug("filter_urls: modify field %s URL %s -> %s", field_name, url_src, new_url)
 
        setattr(result, field_name, new_url)
        if field_name == "url":
            # sync parsed_url with new_url
            if not new_url:
                result.parsed_url = None
            elif isinstance(new_url, str):
                result.parsed_url = urllib.parse.urlparse(new_url)
 
    # "urls": are from infobox
    #
    # As soon we have InfoboxResult, we can move this function to method
    # InfoboxResult.normalize_result_fields
 
    infobox_urls: list[dict[str, str]] = getattr(result, "urls", [])
 
    if infobox_urls:
        # log.debug("filter_urls: infobox_urls .. %s", infobox_urls)
        new_infobox_urls: list[dict[str, str]] = []
 
        for item in infobox_urls:
            url_src = item.get("url", "")
            if not url_src:
                new_infobox_urls.append(item)
                continue
 
            new_url = filter_func(result, "infobox_urls", url_src)
            if isinstance(new_url, bool):
                if new_url:
                    new_infobox_urls.append(item)
                    # log.debug("filter_urls: leave URL in field 'urls' ('infobox_urls') unchanged -> %s", _url)
                    continue
                log.debug("filter_urls: remove URL from field 'urls' ('infobox_urls') URL %s", url_src)
                new_url = None
            if new_url:
                log.debug("filter_urls: modify URL from field 'urls' ('infobox_urls') URL %s -> %s", url_src, new_url)
                item["url"] = new_url
                new_infobox_urls.append(item)
 
        setattr(result, "urls", new_infobox_urls)
 
    # "attributes": are from infobox
    #
    # The infobox has additional subsections for attributes, urls and relatedTopics:
 
    infobox_attributes: list[dict[str, t.Any]] = getattr(result, "attributes", [])
 
    if infobox_attributes:
        # log.debug("filter_urls: infobox_attributes .. %s", infobox_attributes)
        new_infobox_attributes: list[dict[str, str | list[dict[str, str]]]] = []
 
        for item in infobox_attributes:
            image: dict[str, str] = item.get("image", {})
            url_src = image.get("src", "")
            if not url_src:
                new_infobox_attributes.append(item)
                continue
 
            new_url = filter_func(result, "infobox_attributes", url_src)
            if isinstance(new_url, bool):
                if new_url:
                    new_infobox_attributes.append(item)
                    # log.debug("filter_urls: leave URL in field 'image.src' unchanged -> %s", url_src)
                    continue
                log.debug("filter_urls: drop field 'image.src' ('infobox_attributes') URL %s", url_src)
                new_url = None
 
            if new_url:
                log.debug(
                    "filter_urls: modify 'image.src' ('infobox_attributes') URL %s -> %s",
                    url_src,
                    new_url,
                )
                item["image"]["src"] = new_url
                new_infobox_attributes.append(item)
 
        setattr(result, "attributes", new_infobox_attributes)
 
    result.normalize_result_fields()
 
 

Referenced by searx.result_types._base.LegacyResult.filter_urls().

Here is the caller graph for this function:

_normalize_date_fields()

searx.result_types._base._normalize_date_fields ( "MainResult | LegacyResult" result )

protected

Definition at line 220 of file _base.py.

def _normalize_date_fields(result: "MainResult | LegacyResult"):
 
    if result.publishedDate:  # do not try to get a date from an empty string or a None type
        try:  # test if publishedDate >= 1900 (datetime module bug)
            result.pubdate = result.publishedDate.strftime('%Y-%m-%d %H:%M:%S%z')
        except ValueError:
            result.publishedDate = None
 
 

Referenced by searx.result_types._base.LegacyResult.normalize_result_fields(), and searx.result_types._base.MainResult.normalize_result_fields().

Here is the caller graph for this function:

_normalize_text_fields()

searx.result_types._base._normalize_text_fields ( "MainResult | LegacyResult" result )

protected

Definition at line 86 of file _base.py.

def _normalize_text_fields(result: "MainResult | LegacyResult"):
 
    # As soon we need LegacyResult not any longer, we can move this function to
    # method MainResult.normalize_result_fields
 
    # Actually, a type check should not be necessary if the engine is
    # implemented correctly. Historically, however, we have always had a type
    # check here.
 
    if result.title and not isinstance(result.title, str):
        log.debug("result: invalid type of field 'title': %s", str(result))
        result.title = str(result)
    if result.content and not isinstance(result.content, str):
        log.debug("result: invalid type of field 'content': %s", str(result))
        result.content = str(result)
 
    # normalize title and content
    if result.title:
        result.title = WHITESPACE_REGEX.sub(" ", result.title).strip()
    if result.content:
        result.content = WHITESPACE_REGEX.sub(" ", result.content).strip()
    if result.content == result.title:
        # avoid duplicate content between the content and title fields
        result.content = ""
 
 

Referenced by searx.result_types._base.LegacyResult.normalize_result_fields(), and searx.result_types._base.MainResult.normalize_result_fields().

Here is the caller graph for this function:

_normalize_url_fields()

searx.result_types._base._normalize_url_fields ( "Result | LegacyResult" result )

protected

Definition at line 39 of file _base.py.

def _normalize_url_fields(result: "Result | LegacyResult"):
 
    # As soon we need LegacyResult not any longer, we can move this function to
    # method Result.normalize_result_fields
 
    if result.url and not result.parsed_url:
        if not isinstance(result.url, str):
            log.debug('result: invalid URL: %s', str(result))
            result.url = ""
            result.parsed_url = None
        else:
            result.parsed_url = urllib.parse.urlparse(result.url)
 
    if result.parsed_url:
        result.parsed_url = result.parsed_url._replace(
            # if the result has no scheme, use http as default
            scheme=result.parsed_url.scheme or "http",
            path=result.parsed_url.path,
        )
        result.url = result.parsed_url.geturl()
 
    if isinstance(result, LegacyResult) and getattr(result, "infobox", None):
        # As soon we have InfoboxResult, we can move this function to method
        # InfoboxResult.normalize_result_fields
 
        infobox_urls: list[dict[str, str]] = getattr(result, "urls", [])
        for item in infobox_urls:
            _url = item.get("url")
            if not _url:
                continue
            _url = urllib.parse.urlparse(_url)
            item["url"] = _url._replace(
                scheme=_url.scheme or "http",
                # netloc=_url.netloc.replace("www.", ""),
                path=_url.path,
            ).geturl()
 
        infobox_id: str | None = getattr(result, "id", None)
        if infobox_id:
            _url = urllib.parse.urlparse(infobox_id)
            result.id = _url._replace(
                scheme=_url.scheme or "http",
                # netloc=_url.netloc.replace("www.", ""),
                path=_url.path,
            ).geturl()
 
 

Referenced by searx.result_types._base.LegacyResult.normalize_result_fields(), and searx.result_types._base.Result.normalize_result_fields().

Here is the caller graph for this function:

Variable Documentation

__all__

list searx.result_types._base.__all__ = ["Result"]

private

Definition at line 19 of file _base.py.

UNKNOWN

searx.result_types._base.UNKNOWN = object()

Definition at line 36 of file _base.py.

WHITESPACE_REGEX

searx.result_types._base.WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

Definition at line 35 of file _base.py.