searx.results.ResultContainer Class Reference

Collaboration diagram for searx.results.ResultContainer:

Public Member Functions
	__init__ (self)
	extend (self, str|None engine_name, results)
	close (self)
	get_ordered_results (self)
	results_length (self)
int	number_of_results (self)
	add_unresponsive_engine (self, str engine_name, str error_type, bool suspended=False)
	add_timing (self, str engine_name, float engine_time, float page_load_time)
	get_timings (self)

Public Attributes
list	infoboxes = []
set[str]	suggestions = set()
	answers = AnswerSet()
	corrections = set()
dict[str, str|dict]	engine_data = defaultdict(dict)
bool	paging = False
Set[UnresponsiveEngine]	unresponsive_engines = set()
list	timings = []
	redirect_url = None
bool	on_result = lambda _: True

Protected Member Functions
	_merge_infobox (self, infobox)
	_is_valid_url_result (self, result, error_msgs)

Protected Attributes
list	_merged_results = []
list	_number_of_results = []
bool	_closed = False
	_lock = RLock()

Private Member Functions
	__merge_url_result (self, result, position)
	__find_duplicated_http_result (self, result)
	__merge_duplicated_http_result (self, duplicated, result, position)
	__merge_result_no_url (self, result, position)

Static Private Attributes
tuple	__slots__

Detailed Description

docstring for ResultContainer

Definition at line 168 of file results.py.

Constructor & Destructor Documentation

__init__()

searx.results.ResultContainer.__init__

(

self

)

Definition at line 188 of file results.py.

    def __init__(self):
        super().__init__()
        self._merged_results: list[LegacyResult] = []
        self.infoboxes: list[dict] = []
        self.suggestions: set[str] = set()
        self.answers = AnswerSet()
        self.corrections = set()
        self._number_of_results: list[int] = []
        self.engine_data: dict[str, str | dict] = defaultdict(dict)
        self._closed: bool = False
        self.paging: bool = False
        self.unresponsive_engines: Set[UnresponsiveEngine] = set()
        self.timings: List[Timing] = []
        self.redirect_url = None
        self.on_result = lambda _: True
        self._lock = RLock()
 

Member Function Documentation

__find_duplicated_http_result()

searx.results.ResultContainer.__find_duplicated_http_result ( self, result )

private

Definition at line 318 of file results.py.

    def __find_duplicated_http_result(self, result):
        result_template = result.get('template')
        for merged_result in self._merged_results:
            if not merged_result.get('parsed_url'):
                continue
 
            if compare_urls(result['parsed_url'], merged_result['parsed_url']) and result_template == merged_result.get(
                'template'
            ):
                if result_template != 'images.html':
                    # not an image, same template, same url : it's a duplicate
                    return merged_result
 
                # it's an image
                # it's a duplicate if the parsed_url, template and img_src are different
                if result.get('img_src', '') == merged_result.get('img_src', ''):
                    return merged_result
        return None
 

__merge_duplicated_http_result()

searx.results.ResultContainer.__merge_duplicated_http_result ( self, duplicated, result, position )

private

Definition at line 337 of file results.py.

    def __merge_duplicated_http_result(self, duplicated, result, position):
        # use content with more text
        if result_content_len(result.get('content', '')) > result_content_len(duplicated.get('content', '')):
            duplicated['content'] = result['content']
 
        # use title with more text
        if result_content_len(result.get('title', '')) > len(duplicated.get('title', '')):
            duplicated['title'] = result['title']
 
        # merge all result's parameters not found in duplicate
        for key in result.keys():
            if not duplicated.get(key):
                duplicated[key] = result.get(key)
 
        # add the new position
        duplicated['positions'].append(position)
 
        # add engine to list of result-engines
        duplicated['engines'].add(result['engine'])
 
        # use https if possible
        if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
            duplicated['url'] = result['parsed_url'].geturl()
            duplicated['parsed_url'] = result['parsed_url']
 

__merge_result_no_url()

searx.results.ResultContainer.__merge_result_no_url ( self, result, position )

private

Definition at line 362 of file results.py.

    def __merge_result_no_url(self, result, position):
        result['engines'] = set([result['engine']])
        result['positions'] = [position]
        with self._lock:
            self._merged_results.append(result)
 

__merge_url_result()

searx.results.ResultContainer.__merge_url_result ( self, result, position )

private

Definition at line 306 of file results.py.

    def __merge_url_result(self, result, position):
        result['engines'] = set([result['engine']])
        with self._lock:
            duplicated = self.__find_duplicated_http_result(result)
            if duplicated:
                self.__merge_duplicated_http_result(duplicated, result, position)
                return
 
            # if there is no duplicate found, append result
            result['positions'] = [position]
            self._merged_results.append(result)
 

_is_valid_url_result()

searx.results.ResultContainer._is_valid_url_result ( self, result, error_msgs )

protected

Definition at line 286 of file results.py.

    def _is_valid_url_result(self, result, error_msgs):
        if 'url' in result:
            if not isinstance(result['url'], str):
                logger.debug('result: invalid URL: %s', str(result))
                error_msgs.add('invalid URL')
                return False
 
        if 'title' in result and not isinstance(result['title'], str):
            logger.debug('result: invalid title: %s', str(result))
            error_msgs.add('invalid title')
            return False
 
        if 'content' in result:
            if not isinstance(result['content'], str):
                logger.debug('result: invalid content: %s', str(result))
                error_msgs.add('invalid content')
                return False
 
        return True
 

_merge_infobox()

searx.results.ResultContainer._merge_infobox ( self, infobox )

protected

Definition at line 271 of file results.py.

    def _merge_infobox(self, infobox):
        add_infobox = True
        infobox_id = infobox.get('id', None)
        infobox['engines'] = set([infobox['engine']])
        if infobox_id is not None:
            parsed_url_infobox_id = urlparse(infobox_id)
            with self._lock:
                for existingIndex in self.infoboxes:
                    if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
                        merge_two_infoboxes(existingIndex, infobox)
                        add_infobox = False
 
        if add_infobox:
            self.infoboxes.append(infobox)
 

add_timing()

searx.results.ResultContainer.add_timing	(		self,
		str	engine_name,
		float	engine_time,
		float	page_load_time )

Definition at line 471 of file results.py.

    def add_timing(self, engine_name: str, engine_time: float, page_load_time: float):
        with self._lock:
            if self._closed:
                logger.error("call to ResultContainer.add_timing after ResultContainer.close")
                return
            self.timings.append(Timing(engine_name, total=engine_time, load=page_load_time))
 

add_unresponsive_engine()

searx.results.ResultContainer.add_unresponsive_engine	(		self,
		str	engine_name,
		str	error_type,
		bool	suspended = False )

Definition at line 463 of file results.py.

    def add_unresponsive_engine(self, engine_name: str, error_type: str, suspended: bool = False):
        with self._lock:
            if self._closed:
                logger.error("call to ResultContainer.add_unresponsive_engine after ResultContainer.close")
                return
            if engines[engine_name].display_error_messages:
                self.unresponsive_engines.add(UnresponsiveEngine(engine_name, error_type, suspended))
 

close()

searx.results.ResultContainer.close

(

self

)

Definition at line 368 of file results.py.

    def close(self):
        self._closed = True
 
        for result in self._merged_results:
            result['score'] = result_score(result, result.get('priority'))
            # removing html content and whitespace duplications
            if result.get('content'):
                result['content'] = result['content'].strip()
            if result.get('title'):
                result['title'] = ' '.join(result['title'].strip().split())
 
            for result_engine in result['engines']:
                counter_add(result['score'], 'engine', result_engine, 'score')
 
        results = sorted(self._merged_results, key=itemgetter('score'), reverse=True)
 
        # pass 2 : group results by category and template
        gresults = []
        categoryPositions = {}
 
        for res in results:
            if not res.get('url'):
                continue
 
            # do we need to handle more than one category per engine?
            engine = engines[res['engine']]
            res['category'] = engine.categories[0] if len(engine.categories) > 0 else ''
 
            # do we need to handle more than one category per engine?
            category = (
                res['category']
                + ':'
                + res.get('template', '')
                + ':'
                + ('img_src' if 'img_src' in res or 'thumbnail' in res else '')
            )
 
            current = None if category not in categoryPositions else categoryPositions[category]
 
            # group with previous results using the same category
            # if the group can accept more result and is not too far
            # from the current position
            if current is not None and (current['count'] > 0) and (len(gresults) - current['index'] < 20):
                # group with the previous results using
                # the same category with this one
                index = current['index']
                gresults.insert(index, res)
 
                # update every index after the current one
                # (including the current one)
                for k in categoryPositions:  # pylint: disable=consider-using-dict-items
                    v = categoryPositions[k]['index']
                    if v >= index:
                        categoryPositions[k]['index'] = v + 1
 
                # update this category
                current['count'] -= 1
 
            else:
                # same category
                gresults.append(res)
 
                # update categoryIndex
                categoryPositions[category] = {'index': len(gresults), 'count': 8}
 
        # update _merged_results
        self._merged_results = gresults
 

extend()

searx.results.ResultContainer.extend	(		self,
		str | None	engine_name,
			results )

Definition at line 205 of file results.py.

    def extend(self, engine_name: str | None, results):  # pylint: disable=too-many-branches
        if self._closed:
            return
 
        standard_result_count = 0
        error_msgs = set()
 
        for result in list(results):
 
            if isinstance(result, Result):
                result.engine = result.engine or engine_name
                result.normalize_result_fields()
 
                if isinstance(result, BaseAnswer) and self.on_result(result):
                    self.answers.add(result)
                else:
                    # more types need to be implemented in the future ..
                    raise NotImplementedError(f"no handler implemented to process the result of type {result}")
 
            else:
                result['engine'] = result.get('engine') or engine_name or ""
                result = LegacyResult(result)  # for backward compatibility, will be romeved one day
 
                if 'suggestion' in result and self.on_result(result):
                    self.suggestions.add(result['suggestion'])
                elif 'answer' in result and self.on_result(result):
                    warnings.warn(
                        f"answer results from engine {result.engine}"
                        " are without typification / migrate to Answer class.",
                        DeprecationWarning,
                    )
                    self.answers.add(result)
                elif 'correction' in result and self.on_result(result):
                    self.corrections.add(result['correction'])
                elif 'infobox' in result and self.on_result(result):
                    self._merge_infobox(result)
                elif 'number_of_results' in result and self.on_result(result):
                    self._number_of_results.append(result['number_of_results'])
                elif 'engine_data' in result and self.on_result(result):
                    self.engine_data[result.engine][result['key']] = result['engine_data']
                elif result.url:
                    # standard result (url, title, content)
                    if not self._is_valid_url_result(result, error_msgs):
                        continue
                    # normalize the result
                    result.normalize_result_fields()
                    # call on_result call searx.search.SearchWithPlugins._on_result
                    # which calls the plugins
                    if not self.on_result(result):
                        continue
                    self.__merge_url_result(result, standard_result_count + 1)
                    standard_result_count += 1
                elif self.on_result(result):
                    self.__merge_result_no_url(result, standard_result_count + 1)
                    standard_result_count += 1
 
        if len(error_msgs) > 0:
            for msg in error_msgs:
                count_error(engine_name, 'some results are invalids: ' + msg, secondary=True)
 
        if engine_name in engines:
            histogram_observe(standard_result_count, 'engine', engine_name, 'result', 'count')
 
        if not self.paging and engine_name in engines and engines[engine_name].paging:
            self.paging = True
 

get_ordered_results()

searx.results.ResultContainer.get_ordered_results

(

self

)

Definition at line 436 of file results.py.

    def get_ordered_results(self):
        if not self._closed:
            self.close()
        return self._merged_results
 

get_timings()

searx.results.ResultContainer.get_timings

(

self

)

Definition at line 478 of file results.py.

    def get_timings(self):
        with self._lock:
            if not self._closed:
                logger.error("call to ResultContainer.get_timings before ResultContainer.close")
                return []
            return self.timings

number_of_results()

int searx.results.ResultContainer.number_of_results

(

self

)

Returns the average of results number, returns zero if the average
result number is smaller than the actual result count.

Definition at line 445 of file results.py.

    def number_of_results(self) -> int:
        """Returns the average of results number, returns zero if the average
        result number is smaller than the actual result count."""
 
        with self._lock:
            if not self._closed:
                logger.error("call to ResultContainer.number_of_results before ResultContainer.close")
                return 0
 
            resultnum_sum = sum(self._number_of_results)
            if not resultnum_sum or not self._number_of_results:
                return 0
 
            average = int(resultnum_sum / len(self._number_of_results))
            if average < self.results_length():
                average = 0
            return average
 

results_length()

searx.results.ResultContainer.results_length

(

self

)

Definition at line 441 of file results.py.

    def results_length(self):
        return len(self._merged_results)
 

Member Data Documentation

__slots__

tuple searx.results.ResultContainer.__slots__

staticprivate

Initial value:

=  (
        '_merged_results',
        'infoboxes',
        'suggestions',
        'answers',
        'corrections',
        '_number_of_results',
        '_closed',
        'paging',
        'unresponsive_engines',
        'timings',
        'redirect_url',
        'engine_data',
        'on_result',
        '_lock',
    )

Definition at line 171 of file results.py.

_closed

bool searx.results.ResultContainer._closed = False

protected

Definition at line 197 of file results.py.

_lock

searx.results.ResultContainer._lock = RLock()

protected

Definition at line 203 of file results.py.

_merged_results

searx.results.ResultContainer._merged_results = []

protected

Definition at line 190 of file results.py.

_number_of_results

list searx.results.ResultContainer._number_of_results = []

protected

Definition at line 195 of file results.py.

answers

searx.results.ResultContainer.answers = AnswerSet()

Definition at line 193 of file results.py.

corrections

searx.results.ResultContainer.corrections = set()

Definition at line 194 of file results.py.

engine_data

dict[str, str | dict] searx.results.ResultContainer.engine_data = defaultdict(dict)

Definition at line 196 of file results.py.

Referenced by searx.search.models.SearchQuery.__copy__().

infoboxes

list searx.results.ResultContainer.infoboxes = []

Definition at line 191 of file results.py.

on_result

bool searx.results.ResultContainer.on_result = lambda _: True

Definition at line 202 of file results.py.

paging

bool searx.results.ResultContainer.paging = False

Definition at line 198 of file results.py.

redirect_url

searx.results.ResultContainer.redirect_url = None

Definition at line 201 of file results.py.

suggestions

set[str] searx.results.ResultContainer.suggestions = set()

Definition at line 192 of file results.py.

timings

list searx.results.ResultContainer.timings = []

Definition at line 200 of file results.py.

unresponsive_engines

Set[UnresponsiveEngine] searx.results.ResultContainer.unresponsive_engines = set()

Definition at line 199 of file results.py.

---

The documentation for this class was generated from the following file:

• /home/andrew/Documents/code/public/searxng/searx/results.py