.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
__init__.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Implementations of the framework for the SearXNG engines.
3
4- :py:obj:`searx.enginelib.EngineCache`
5- :py:obj:`searx.enginelib.Engine`
6- :py:obj:`searx.enginelib.traits`
7
8There is a command line for developer purposes and for deeper analysis. Here is
9an example in which the command line is called in the development environment::
10
11 $ ./manage pyenv.cmd bash --norc --noprofile
12 (py3) python -m searx.enginelib --help
13
14.. hint::
15
16 The long term goal is to modularize all implementations of the engine
17 framework here in this Python package. ToDo:
18
19 - move implementations of the :ref:`searx.engines loader` to a new module in
20 the :py:obj:`searx.enginelib` namespace.
21
22-----
23
24"""
25from __future__ import annotations
26
27__all__ = ["EngineCache", "Engine", "ENGINES_CACHE"]
28
29from typing import List, Callable, TYPE_CHECKING, Any
30import string
31import typer
32
33from ..cache import ExpireCache, ExpireCacheCfg
34
35if TYPE_CHECKING:
36 from searx.enginelib import traits
37
38
39ENGINES_CACHE = ExpireCache.build_cache(
41 name="ENGINES_CACHE",
42 MAXHOLD_TIME=60 * 60 * 24 * 7, # 7 days
43 MAINTENANCE_PERIOD=60 * 60, # 2h
44 )
45)
46"""Global :py:obj:`searx.cache.ExpireCacheSQLite` instance where the cached
47values from all engines are stored. The `MAXHOLD_TIME` is 7 days and the
48`MAINTENANCE_PERIOD` is set to two hours."""
49
50app = typer.Typer()
51
52
53@app.command()
54def state():
55 """Show state for the caches of the engines."""
56
57 title = "cache tables and key/values"
58 print(title)
59 print("=" * len(title))
60 print(ENGINES_CACHE.state().report())
61 print()
62 title = f"properties of {ENGINES_CACHE.cfg.name}"
63 print(title)
64 print("=" * len(title))
65 print(str(ENGINES_CACHE.properties)) # type: ignore
66
67
68@app.command()
69def maintenance(force: bool = True):
70 """Carry out maintenance on cache of the engines."""
71 ENGINES_CACHE.maintenance(force=force)
72
73
75 """Persistent (SQLite) key/value cache that deletes its values again after
76 ``expire`` seconds (default/max: :py:obj:`MAXHOLD_TIME
77 <searx.cache.ExpireCacheCfg.MAXHOLD_TIME>`). This class is a wrapper around
78 :py:obj:`ENGINES_CACHE` (:py:obj:`ExpireCacheSQLite
79 <searx.cache.ExpireCacheSQLite>`).
80
81 In the :origin:`searx/engines/demo_offline.py` engine you can find an
82 exemplary implementation of such a cache other examples are implemented
83 in:
84
85 - :origin:`searx/engines/radio_browser.py`
86 - :origin:`searx/engines/soundcloud.py`
87 - :origin:`searx/engines/startpage.py`
88
89 .. code: python
90
91 from searx.enginelib import EngineCache
92 CACHE: EngineCache
93
94 def init(engine_settings):
95 global CACHE
96 CACHE = EngineCache(engine_settings["name"])
97
98 def request(query, params):
99 token = CACHE.get(key="token")
100 if token is None:
101 token = get_token()
102 # cache token of this engine for 1h
103 CACHE.set(key="token", value=token, expire=3600)
104 ...
105
106 For introspection of the DB, jump into developer environment and run command to
107 show cache state::
108
109 $ ./manage pyenv.cmd bash --norc --noprofile
110 (py3) python -m searx.enginelib cache state
111
112 cache tables and key/values
113 ===========================
114 [demo_offline ] 2025-04-22 11:32:50 count --> (int) 4
115 [startpage ] 2025-04-22 12:32:30 SC_CODE --> (str) fSOBnhEMlDfE20
116 [duckduckgo ] 2025-04-22 12:32:31 4dff493e.... --> (str) 4-128634958369380006627592672385352473325
117 [duckduckgo ] 2025-04-22 12:40:06 3e2583e2.... --> (str) 4-263126175288871260472289814259666848451
118 [radio_browser ] 2025-04-23 11:33:08 servers --> (list) ['https://de2.api.radio-browser.info', ...]
119 [soundcloud ] 2025-04-29 11:40:06 guest_client_id --> (str) EjkRJG0BLNEZquRiPZYdNtJdyGtTuHdp
120 [wolframalpha ] 2025-04-22 12:40:06 code --> (str) 5aa79f86205ad26188e0e26e28fb7ae7
121 number of tables: 6
122 number of key/value pairs: 7
123
124 In the "cache tables and key/values" section, the table name (engine name) is at
125 first position on the second there is the calculated expire date and on the
126 third and fourth position the key/value is shown.
127
128 About duckduckgo: The *vqd coode* of ddg depends on the query term and therefore
129 the key is a hash value of the query term (to not to store the raw query term).
130
131 In the "properties of ENGINES_CACHE" section all properties of the SQLiteAppl /
132 ExpireCache and their last modification date are shown::
133
134 properties of ENGINES_CACHE
135 ===========================
136 [last modified: 2025-04-22 11:32:27] DB_SCHEMA : 1
137 [last modified: 2025-04-22 11:32:27] LAST_MAINTENANCE :
138 [last modified: 2025-04-22 11:32:27] crypt_hash : ca612e3566fdfd7cf7efe...
139 [last modified: 2025-04-22 11:32:30] CACHE-TABLE--demo_offline: demo_offline
140 [last modified: 2025-04-22 11:32:30] CACHE-TABLE--startpage: startpage
141 [last modified: 2025-04-22 11:32:31] CACHE-TABLE--duckduckgo: duckduckgo
142 [last modified: 2025-04-22 11:33:08] CACHE-TABLE--radio_browser: radio_browser
143 [last modified: 2025-04-22 11:40:06] CACHE-TABLE--soundcloud: soundcloud
144 [last modified: 2025-04-22 11:40:06] CACHE-TABLE--wolframalpha: wolframalpha
145
146 These properties provide information about the state of the ExpireCache and
147 control the behavior. For example, the maintenance intervals are controlled by
148 the last modification date of the LAST_MAINTENANCE property and the hash value
149 of the password can be used to detect whether the password has been changed (in
150 this case the DB entries can no longer be decrypted and the entire cache must be
151 discarded).
152 """
153
154 def __init__(self, engine_name: str, expire: int | None = None):
155 self.expire = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
156 _valid = "-_." + string.ascii_letters + string.digits
157 self.table_name = "".join([c if c in _valid else "_" for c in engine_name])
158
159 def set(self, key: str, value: Any, expire: int | None = None) -> bool:
160 return ENGINES_CACHE.set(
161 key=key,
162 value=value,
163 expire=expire or self.expire,
164 ctx=self.table_name,
165 )
166
167 def get(self, key: str, default=None) -> Any:
168 return ENGINES_CACHE.get(key, default=default, ctx=self.table_name)
169
170 def secret_hash(self, name: str | bytes) -> str:
171 return ENGINES_CACHE.secret_hash(name=name)
172
173
174class Engine: # pylint: disable=too-few-public-methods
175 """Class of engine instances build from YAML settings.
176
177 Further documentation see :ref:`general engine configuration`.
178
179 .. hint::
180
181 This class is currently never initialized and only used for type hinting.
182 """
183
184 # Common options in the engine module
185
186 engine_type: str
187 """Type of the engine (:ref:`searx.search.processors`)"""
188
189 paging: bool
190 """Engine supports multiple pages."""
191
192 time_range_support: bool
193 """Engine supports search time range."""
194
195 safesearch: bool
196 """Engine supports SafeSearch"""
197
198 language_support: bool
199 """Engine supports languages (locales) search."""
200
201 language: str
202 """For an engine, when there is ``language: ...`` in the YAML settings the engine
203 does support only this one language:
204
205 .. code:: yaml
206
207 - name: google french
208 engine: google
209 language: fr
210 """
211
212 region: str
213 """For an engine, when there is ``region: ...`` in the YAML settings the engine
214 does support only this one region::
215
216 .. code:: yaml
217
218 - name: google belgium
219 engine: google
220 region: fr-BE
221 """
222
223 fetch_traits: Callable
224 """Function to to fetch engine's traits from origin."""
225
227 """Traits of the engine."""
228
229 # settings.yml
230
231 categories: List[str]
232 """Specifies to which :ref:`engine categories` the engine should be added."""
233
234 name: str
235 """Name that will be used across SearXNG to define this engine. In settings, on
236 the result page .."""
237
238 engine: str
239 """Name of the python file used to handle requests and responses to and from
240 this search engine (file name from :origin:`searx/engines` without
241 ``.py``)."""
242
243 enable_http: bool
244 """Enable HTTP (by default only HTTPS is enabled)."""
245
246 shortcut: str
247 """Code used to execute bang requests (``!foo``)"""
248
249 timeout: float
250 """Specific timeout for search-engine."""
251
252 display_error_messages: bool
253 """Display error messages on the web UI."""
254
255 proxies: dict
256 """Set proxies for a specific engine (YAML):
257
258 .. code:: yaml
259
260 proxies :
261 http: socks5://proxy:port
262 https: socks5://proxy:port
263 """
264
265 disabled: bool
266 """To disable by default the engine, but not deleting it. It will allow the
267 user to manually activate it in the settings."""
268
269 inactive: bool
270 """Remove the engine from the settings (*disabled & removed*)."""
271
272 about: dict
273 """Additional fields describing the engine.
274
275 .. code:: yaml
276
277 about:
278 website: https://example.com
279 wikidata_id: Q306656
280 official_api_documentation: https://example.com/api-doc
281 use_official_api: true
282 require_api_key: true
283 results: HTML
284 """
285
286 using_tor_proxy: bool
287 """Using tor proxy (``true``) or not (``false``) for this engine."""
288
289 send_accept_language_header: bool
290 """When this option is activated, the language (locale) that is selected by
291 the user is used to build and send a ``Accept-Language`` header in the
292 request to the origin search engine."""
293
294 tokens: List[str]
295 """A list of secret tokens to make this engine *private*, more details see
296 :ref:`private engines`."""
297
298 weight: int
299 """Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
str secret_hash(self, str|bytes name)
Definition __init__.py:170
__init__(self, str engine_name, int|None expire=None)
Definition __init__.py:154
bool set(self, str key, Any value, int|None expire=None)
Definition __init__.py:159
Any get(self, str key, default=None)
Definition __init__.py:167
maintenance(bool force=True)
Definition __init__.py:69