.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
hostnames.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2# pylint: disable=too-many-branches, unused-argument
3"""During the initialization phase, the plugin checks whether a ``hostnames:``
4configuration exists. If this is not the case, the plugin is not included in the
5PluginStorage (it is not available for selection).
6
7- ``hostnames.replace``: A **mapping** of regular expressions to hostnames to be
8 replaced by other hostnames.
9
10 .. code:: yaml
11
12 hostnames:
13 replace:
14 '(.*\\.)?youtube\\.com$': 'invidious.example.com'
15 '(.*\\.)?youtu\\.be$': 'invidious.example.com'
16 ...
17
18- ``hostnames.remove``: A **list** of regular expressions of the hostnames whose
19 results should be taken from the results list.
20
21 .. code:: yaml
22
23 hostnames:
24 remove:
25 - '(.*\\.)?facebook.com$'
26 - ...
27
28- ``hostnames.high_priority``: A **list** of regular expressions for hostnames
29 whose result should be given higher priority. The results from these hosts are
30 arranged higher in the results list.
31
32 .. code:: yaml
33
34 hostnames:
35 high_priority:
36 - '(.*\\.)?wikipedia.org$'
37 - ...
38
39- ``hostnames.lower_priority``: A **list** of regular expressions for hostnames
40 whose result should be given lower priority. The results from these hosts are
41 arranged lower in the results list.
42
43 .. code:: yaml
44
45 hostnames:
46 low_priority:
47 - '(.*\\.)?google(\\..*)?$'
48 - ...
49
50If the URL matches the pattern of ``high_priority`` AND ``low_priority``, the
51higher priority wins over the lower priority.
52
53Alternatively, you can also specify a file name for the **mappings** or
54**lists** to load these from an external file:
55
56.. code:: yaml
57
58 hostnames:
59 replace: 'rewrite-hosts.yml'
60 remove:
61 - '(.*\\.)?facebook.com$'
62 - ...
63 low_priority:
64 - '(.*\\.)?google(\\..*)?$'
65 - ...
66 high_priority:
67 - '(.*\\.)?wikipedia.org$'
68 - ...
69
70The ``rewrite-hosts.yml`` from the example above must be in the folder in which
71the ``settings.yml`` file is already located (``/etc/searxng``). The file then
72only contains the lists or the mapping tables without further information on the
73namespaces. In the example above, this would be a mapping table that looks
74something like this:
75
76.. code:: yaml
77
78 '(.*\\.)?youtube\\.com$': 'invidious.example.com'
79 '(.*\\.)?youtu\\.be$': 'invidious.example.com'
80
81"""
82
83import typing as t
84
85import re
86from urllib.parse import urlunparse, urlparse
87
88from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
89
90from searx import settings
91from searx.result_types._base import MainResult, LegacyResult
92from searx.settings_loader import get_yaml_cfg
93from searx.plugins import Plugin, PluginInfo
94
95from ._core import log
96
97if t.TYPE_CHECKING:
98 import flask
99 from searx.search import SearchWithPlugins
100 from searx.extended_types import SXNG_Request
101 from searx.result_types import Result
102 from searx.plugins import PluginCfg
103
104REPLACE: dict[re.Pattern, str] = {}
105REMOVE: set = set()
106HIGH: set = set()
107LOW: set = set()
108
109
111 """Rewrite hostnames, remove results or prioritize them."""
112
113 id = "hostnames"
114
115 def __init__(self, plg_cfg: "PluginCfg") -> None:
116 super().__init__(plg_cfg)
118 id=self.id,
119 name=gettext("Hostnames plugin"),
120 description=gettext("Rewrite hostnames and remove or prioritize results based on the hostname"),
121 preference_section="general",
122 )
123
124 def on_result(self, request: "SXNG_Request", search: "SearchWithPlugins", result: "Result") -> bool:
125
126 for pattern in REMOVE:
127 if result.parsed_url and pattern.search(result.parsed_url.netloc):
128 # if the link (parsed_url) of the result match, then remove the
129 # result from the result list, in any other case, the result
130 # remains in the list / see final "return True" below.
131 # log.debug("FIXME: remove [url/parsed_url] %s %s", pattern.pattern, result.url)
132 return False
133
134 result.filter_urls(filter_url_field)
135
136 if isinstance(result, (MainResult, LegacyResult)):
137 for pattern in LOW:
138 if result.parsed_url and pattern.search(result.parsed_url.netloc):
139 result.priority = "low"
140
141 for pattern in HIGH:
142 if result.parsed_url and pattern.search(result.parsed_url.netloc):
143 result.priority = "high"
144
145 return True
146
147 def init(self, app: "flask.Flask") -> bool: # pylint: disable=unused-argument
148 global REPLACE, REMOVE, HIGH, LOW # pylint: disable=global-statement
149
150 if not settings.get(self.id):
151 # Remove plugin, if there isn't a "hostnames:" setting
152 return False
153
154 REPLACE = self._load_regular_expressions("replace") or {} # type: ignore
155 REMOVE = self._load_regular_expressions("remove") or set() # type: ignore
156 HIGH = self._load_regular_expressions("high_priority") or set() # type: ignore
157 LOW = self._load_regular_expressions("low_priority") or set() # type: ignore
158
159 return True
160
161 def _load_regular_expressions(self, settings_key) -> dict[re.Pattern, str] | set | None:
162 setting_value = settings.get(self.id, {}).get(settings_key)
163
164 if not setting_value:
165 return None
166
167 # load external file with configuration
168 if isinstance(setting_value, str):
169 setting_value = get_yaml_cfg(setting_value)
170
171 if isinstance(setting_value, list):
172 return {re.compile(r) for r in setting_value}
173
174 if isinstance(setting_value, dict):
175 return {re.compile(p): r for (p, r) in setting_value.items()}
176
177 return None
178
179
180def filter_url_field(result: "Result|LegacyResult", field_name: str, url_src: str) -> bool | str:
181 """Returns bool ``True`` to use URL unchanged (``False`` to ignore URL).
182 If URL should be modified, the returned string is the new URL to use."""
183
184 if not url_src:
185 log.debug("missing a URL in field %s", field_name)
186 return True
187
188 url_src_parsed = urlparse(url=url_src)
189
190 for pattern in REMOVE:
191 if pattern.search(url_src_parsed.netloc):
192 return False
193
194 for pattern, replacement in REPLACE.items():
195 if pattern.search(url_src_parsed.netloc):
196 new_url = url_src_parsed._replace(netloc=pattern.sub(replacement, url_src_parsed.netloc))
197 new_url = urlunparse(new_url)
198 return new_url
199
200 return True
bool init(self, "flask.Flask" app)
Definition hostnames.py:147
None __init__(self, "PluginCfg" plg_cfg)
Definition hostnames.py:115
bool on_result(self, "SXNG_Request" request, "SearchWithPlugins" search, "Result" result)
Definition hostnames.py:124
dict[re.Pattern, str]|set|None _load_regular_expressions(self, settings_key)
Definition hostnames.py:161
bool|str filter_url_field("Result|LegacyResult" result, str field_name, str url_src)
Definition hostnames.py:180