.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
traits.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Engine's traits are fetched from the origin engines and stored in a JSON file
3in the *data folder*. Most often traits are languages and region codes and
4their mapping from SearXNG's representation to the representation in the origin
5search engine. For new traits new properties can be added to the class
6:py:class:`EngineTraits`.
7
8To load traits from the persistence :py:obj:`EngineTraitsMap.from_data` can be
9used.
10"""
11
12
13import os
14import json
15import dataclasses
16import types
17import typing as t
18import pathlib
19from searx import locales
20from searx.data import data_dir, ENGINE_TRAITS
21
22if t.TYPE_CHECKING:
23 from . import Engine
24
25
26class EngineTraitsEncoder(json.JSONEncoder):
27 """Encodes :class:`EngineTraits` to a serializable object, see
28 :class:`json.JSONEncoder`."""
29
30 def default(self, o: t.Any) -> t.Any:
31 """Return dictionary of a :class:`EngineTraits` object."""
32 if isinstance(o, EngineTraits):
33 return o.__dict__
34 return super().default(o)
35
36
37@dataclasses.dataclass
39 """The class is intended to be instantiated for each engine."""
40
41 regions: dict[str, str] = dataclasses.field(default_factory=dict)
42 """Maps SearXNG's internal representation of a region to the one of the engine.
43
44 SearXNG's internal representation can be parsed by babel and the value is
45 send to the engine:
46
47 .. code:: python
48
49 regions ={
50 'fr-BE' : <engine's region name>,
51 }
52
53 for key, egnine_region regions.items():
54 searxng_region = babel.Locale.parse(key, sep='-')
55 ...
56 """
57
58 languages: dict[str, str] = dataclasses.field(default_factory=dict)
59 """Maps SearXNG's internal representation of a language to the one of the engine.
60
61 SearXNG's internal representation can be parsed by babel and the value is
62 send to the engine:
63
64 .. code:: python
65
66 languages = {
67 'ca' : <engine's language name>,
68 }
69
70 for key, egnine_lang in languages.items():
71 searxng_lang = babel.Locale.parse(key)
72 ...
73 """
74
75 all_locale: str | None = None
76 """To which locale value SearXNG's ``all`` language is mapped (shown a "Default
77 language").
78 """
79
80 data_type: t.Literal['traits_v1'] = 'traits_v1'
81 """Data type, default is 'traits_v1'.
82 """
83
84 custom: dict[str, t.Any] = dataclasses.field(default_factory=dict)
85 """A place to store engine's custom traits, not related to the SearXNG core.
86 """
87
88 def get_language(self, searxng_locale: str, default: t.Any = None):
89 """Return engine's language string that *best fits* to SearXNG's locale.
90
91 :param searxng_locale: SearXNG's internal representation of locale
92 selected by the user.
93
94 :param default: engine's default language
95
96 The *best fits* rules are implemented in
97 :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``
98 which is determined from :py:obj:`EngineTraits.all_locale`.
99 """
100 if searxng_locale == 'all' and self.all_locale is not None:
101 return self.all_locale
102 return locales.get_engine_locale(searxng_locale, self.languages, default=default)
103
104 def get_region(self, searxng_locale: str, default: t.Any = None) -> t.Any:
105 """Return engine's region string that best fits to SearXNG's locale.
106
107 :param searxng_locale: SearXNG's internal representation of locale
108 selected by the user.
109
110 :param default: engine's default region
111
112 The *best fits* rules are implemented in
113 :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``
114 which is determined from :py:obj:`EngineTraits.all_locale`.
115 """
116 if searxng_locale == 'all' and self.all_locale is not None:
117 return self.all_locale
118 return locales.get_engine_locale(searxng_locale, self.regions, default=default)
119
120 def is_locale_supported(self, searxng_locale: str) -> bool:
121 """A *locale* (SearXNG's internal representation) is considered to be
122 supported by the engine if the *region* or the *language* is supported
123 by the engine.
124
125 For verification the functions :py:func:`EngineTraits.get_region` and
126 :py:func:`EngineTraits.get_language` are used.
127 """
128 if self.data_type == 'traits_v1':
129 return bool(self.get_region(searxng_locale) or self.get_language(searxng_locale))
130
131 raise TypeError('engine traits of type %s is unknown' % self.data_type)
132
133 def copy(self):
134 """Create a copy of the dataclass object."""
135 return EngineTraits(**dataclasses.asdict(self)) # type: ignore
136
137 @classmethod
138 def fetch_traits(cls, engine: "Engine | types.ModuleType") -> "EngineTraits | None":
139 """Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
140 and set properties from the origin engine in the object ``engine_traits``. If
141 function does not exists, ``None`` is returned.
142 """
143
144 fetch_traits = getattr(engine, 'fetch_traits', None)
145 engine_traits = None
146
147 if fetch_traits:
148 engine_traits = cls()
149 fetch_traits(engine_traits)
150 return engine_traits
151
152 def set_traits(self, engine: "Engine | types.ModuleType"):
153 """Set traits from self object in a :py:obj:`.Engine` namespace.
154
155 :param engine: engine instance build by :py:func:`searx.engines.load_engine`
156 """
157
158 if self.data_type == 'traits_v1':
159 self._set_traits_v1(engine)
160 else:
161 raise TypeError('engine traits of type %s is unknown' % self.data_type)
162
163 def _set_traits_v1(self, engine: "Engine | types.ModuleType"):
164 # For an engine, when there is `language: ...` in the YAML settings the engine
165 # does support only this one language (region)::
166 #
167 # - name: google italian
168 # engine: google
169 # language: it
170 # region: it-IT
171
172 traits = self.copy()
173
174 _msg = "settings.yml - engine: '%s' / %s: '%s' not supported"
175
176 languages = traits.languages
177 if hasattr(engine, 'language'):
178 if engine.language not in languages:
179 raise ValueError(_msg % (engine.name, 'language', engine.language))
180 traits.languages = {engine.language: languages[engine.language]}
181
182 regions = traits.regions
183 if hasattr(engine, 'region'):
184 if engine.region not in regions:
185 raise ValueError(_msg % (engine.name, 'region', engine.region))
186 traits.regions = {engine.region: regions[engine.region]}
187
188 engine.language_support = bool(traits.languages or traits.regions) # type: ignore
189
190 # set the copied & modified traits in engine's namespace
191 engine.traits = traits # pyright: ignore[reportAttributeAccessIssue]
192
193
194class EngineTraitsMap(dict[str, EngineTraits]):
195 """A python dictionary to map :class:`EngineTraits` by engine name."""
196
197 ENGINE_TRAITS_FILE: pathlib.Path = (data_dir / 'engine_traits.json').resolve()
198 """File with persistence of the :py:obj:`EngineTraitsMap`."""
199
200 def save_data(self):
201 """Store EngineTraitsMap in in file :py:obj:`self.ENGINE_TRAITS_FILE`"""
202 with open(self.ENGINE_TRAITS_FILE, 'w', encoding='utf-8') as f:
203 json.dump(self, f, indent=2, sort_keys=True, cls=EngineTraitsEncoder)
204
205 @classmethod
206 def from_data(cls) -> 'EngineTraitsMap':
207 """Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""
208 obj = cls()
209 for k, v in ENGINE_TRAITS.items():
210 obj[k] = EngineTraits(**v)
211 return obj
212
213 @classmethod
214 def fetch_traits(cls, log: t.Callable[[str], None]) -> 'EngineTraitsMap':
215 from searx import engines # pylint: disable=cyclic-import, import-outside-toplevel
216
217 names = list(engines.engines)
218 names.sort()
219 obj = cls()
220
221 for engine_name in names:
222 engine: Engine | types.ModuleType = engines.engines[engine_name]
223 traits = None
224
225 # pylint: disable=broad-exception-caught
226 try:
227 traits = EngineTraits.fetch_traits(engine)
228 except Exception as exc:
229 log("FATAL: while fetch_traits %s: %s" % (engine_name, exc))
230 if os.environ.get('FORCE', '').lower() not in ['on', 'true', '1']:
231 raise
232 v = ENGINE_TRAITS.get(engine_name)
233 if v:
234 log("FORCE: re-use old values from fetch_traits - ENGINE_TRAITS[%s]" % engine_name)
235 traits = EngineTraits(**v)
236
237 if traits is not None:
238 log("%-20s: SearXNG languages --> %s " % (engine_name, len(traits.languages)))
239 log("%-20s: SearXNG regions --> %s" % (engine_name, len(traits.regions)))
240 obj[engine_name] = traits
241
242 return obj
243
244 def set_traits(self, engine: "Engine | types.ModuleType"):
245 """Set traits in a :py:obj:`Engine` namespace.
246
247 :param engine: engine instance build by :py:func:`searx.engines.load_engine`
248 """
249
250 engine_traits = EngineTraits(data_type='traits_v1')
251 if engine.name in self.keys():
252 engine_traits = self[engine.name]
253
254 elif engine.engine in self.keys():
255 # The key of the dictionary traits_map is the *engine name*
256 # configured in settings.xml. When multiple engines are configured
257 # in settings.yml to use the same origin engine (python module)
258 # these additional engines can use the languages from the origin
259 # engine. For this use the configured ``engine: ...`` from
260 # settings.yml
261 engine_traits = self[engine.engine]
262
263 engine_traits.set_traits(engine)
'EngineTraitsMap' fetch_traits(cls, t.Callable[[str], None] log)
Definition traits.py:214
set_traits(self, "Engine | types.ModuleType" engine)
Definition traits.py:244
'EngineTraitsMap' from_data(cls)
Definition traits.py:206
"EngineTraits | None" fetch_traits(cls, "Engine | types.ModuleType" engine)
Definition traits.py:138
t.Any get_region(self, str searxng_locale, t.Any default=None)
Definition traits.py:104
bool is_locale_supported(self, str searxng_locale)
Definition traits.py:120
_set_traits_v1(self, "Engine | types.ModuleType" engine)
Definition traits.py:163
get_language(self, str searxng_locale, t.Any default=None)
Definition traits.py:88
set_traits(self, "Engine | types.ModuleType" engine)
Definition traits.py:152