2"""Implementations of the framework for the SearXNG engines.
4- :py:obj:`searx.enginelib.EngineCache`
5- :py:obj:`searx.enginelib.Engine`
6- :py:obj:`searx.enginelib.traits`
8There is a command line for developer purposes and for deeper analysis. Here is
9an example in which the command line is called in the development environment::
11 $ ./manage pyenv.cmd bash --norc --noprofile
12 (py3) python -m searx.enginelib --help
16 The long term goal is to modularize all implementations of the engine
17 framework here in this Python package. ToDo:
19 - move implementations of the :ref:`searx.engines loader` to a new module in
20 the :py:obj:`searx.enginelib` namespace.
25from __future__
import annotations
27__all__ = [
"EngineCache",
"Engine",
"ENGINES_CACHE"]
29from typing
import List, Callable, TYPE_CHECKING, Any
33from ..cache
import ExpireCache, ExpireCacheCfg
39ENGINES_CACHE = ExpireCache.build_cache(
42 MAXHOLD_TIME=60 * 60 * 24 * 7,
43 MAINTENANCE_PERIOD=60 * 60,
46"""Global :py:obj:`searx.cache.ExpireCacheSQLite` instance where the cached
47values from all engines are stored. The `MAXHOLD_TIME` is 7 days and the
48`MAINTENANCE_PERIOD` is set to two hours."""
55 """Show state for the caches of the engines."""
57 title =
"cache tables and key/values"
59 print(
"=" * len(title))
60 print(ENGINES_CACHE.state().report())
62 title = f
"properties of {ENGINES_CACHE.cfg.name}"
64 print(
"=" * len(title))
65 print(str(ENGINES_CACHE.properties))
70 """Carry out maintenance on cache of the engines."""
71 ENGINES_CACHE.maintenance(force=force)
75 """Persistent (SQLite) key/value cache that deletes its values again after
76 ``expire`` seconds (default/max: :py:obj:`MAXHOLD_TIME
77 <searx.cache.ExpireCacheCfg.MAXHOLD_TIME>`). This class is a wrapper around
78 :py:obj:`ENGINES_CACHE` (:py:obj:`ExpireCacheSQLite
79 <searx.cache.ExpireCacheSQLite>`).
81 In the :origin:`searx/engines/demo_offline.py` engine you can find an
82 exemplary implementation of such a cache other examples are implemented
85 - :origin:`searx/engines/radio_browser.py`
86 - :origin:`searx/engines/soundcloud.py`
87 - :origin:`searx/engines/startpage.py`
91 from searx.enginelib import EngineCache
94 def init(engine_settings):
96 CACHE = EngineCache(engine_settings["name"])
98 def request(query, params):
99 token = CACHE.get(key="token")
102 # cache token of this engine for 1h
103 CACHE.set(key="token", value=token, expire=3600)
106 For introspection of the DB, jump into developer environment and run command to
109 $ ./manage pyenv.cmd bash --norc --noprofile
110 (py3) python -m searx.enginelib cache state
112 cache tables and key/values
113 ===========================
114 [demo_offline ] 2025-04-22 11:32:50 count --> (int) 4
115 [startpage ] 2025-04-22 12:32:30 SC_CODE --> (str) fSOBnhEMlDfE20
116 [duckduckgo ] 2025-04-22 12:32:31 4dff493e.... --> (str) 4-128634958369380006627592672385352473325
117 [duckduckgo ] 2025-04-22 12:40:06 3e2583e2.... --> (str) 4-263126175288871260472289814259666848451
118 [radio_browser ] 2025-04-23 11:33:08 servers --> (list) ['https://de2.api.radio-browser.info', ...]
119 [soundcloud ] 2025-04-29 11:40:06 guest_client_id --> (str) EjkRJG0BLNEZquRiPZYdNtJdyGtTuHdp
120 [wolframalpha ] 2025-04-22 12:40:06 code --> (str) 5aa79f86205ad26188e0e26e28fb7ae7
122 number of key/value pairs: 7
124 In the "cache tables and key/values" section, the table name (engine name) is at
125 first position on the second there is the calculated expire date and on the
126 third and fourth position the key/value is shown.
128 About duckduckgo: The *vqd coode* of ddg depends on the query term and therefore
129 the key is a hash value of the query term (to not to store the raw query term).
131 In the "properties of ENGINES_CACHE" section all properties of the SQLiteAppl /
132 ExpireCache and their last modification date are shown::
134 properties of ENGINES_CACHE
135 ===========================
136 [last modified: 2025-04-22 11:32:27] DB_SCHEMA : 1
137 [last modified: 2025-04-22 11:32:27] LAST_MAINTENANCE :
138 [last modified: 2025-04-22 11:32:27] crypt_hash : ca612e3566fdfd7cf7efe...
139 [last modified: 2025-04-22 11:32:30] CACHE-TABLE--demo_offline: demo_offline
140 [last modified: 2025-04-22 11:32:30] CACHE-TABLE--startpage: startpage
141 [last modified: 2025-04-22 11:32:31] CACHE-TABLE--duckduckgo: duckduckgo
142 [last modified: 2025-04-22 11:33:08] CACHE-TABLE--radio_browser: radio_browser
143 [last modified: 2025-04-22 11:40:06] CACHE-TABLE--soundcloud: soundcloud
144 [last modified: 2025-04-22 11:40:06] CACHE-TABLE--wolframalpha: wolframalpha
146 These properties provide information about the state of the ExpireCache and
147 control the behavior. For example, the maintenance intervals are controlled by
148 the last modification date of the LAST_MAINTENANCE property and the hash value
149 of the password can be used to detect whether the password has been changed (in
150 this case the DB entries can no longer be decrypted and the entire cache must be
154 def __init__(self, engine_name: str, expire: int |
None =
None):
155 self.
expire = expire
or ENGINES_CACHE.cfg.MAXHOLD_TIME
156 _valid =
"-_." + string.ascii_letters + string.digits
157 self.
table_name =
"".join([c
if c
in _valid
else "_" for c
in engine_name])
159 def set(self, key: str, value: Any, expire: int |
None =
None) -> bool:
160 return ENGINES_CACHE.set(
163 expire=expire
or self.
expire,
167 def get(self, key: str, default=
None) -> Any:
168 return ENGINES_CACHE.get(key, default=default, ctx=self.
table_name)
171 return ENGINES_CACHE.secret_hash(name=name)
175 """Class of engine instances build from YAML settings.
177 Further documentation see :ref:`general engine configuration`.
181 This class is currently never initialized and only used for type hinting.
187 """Type of the engine (:ref:`searx.search.processors`)"""
190 """Engine supports multiple pages."""
192 time_range_support: bool
193 """Engine supports search time range."""
196 """Engine supports SafeSearch"""
198 language_support: bool
199 """Engine supports languages (locales) search."""
202 """For an engine, when there is ``language: ...`` in the YAML settings the engine
203 does support only this one language:
207 - name: google french
213 """For an engine, when there is ``region: ...`` in the YAML settings the engine
214 does support only this one region::
218 - name: google belgium
223 fetch_traits: Callable
224 """Function to to fetch engine's traits from origin."""
227 """Traits of the engine."""
231 categories: List[str]
232 """Specifies to which :ref:`engine categories` the engine should be added."""
235 """Name that will be used across SearXNG to define this engine. In settings, on
236 the result page .."""
239 """Name of the python file used to handle requests and responses to and from
240 this search engine (file name from :origin:`searx/engines` without
244 """Enable HTTP (by default only HTTPS is enabled)."""
247 """Code used to execute bang requests (``!foo``)"""
250 """Specific timeout for search-engine."""
252 display_error_messages: bool
253 """Display error messages on the web UI."""
256 """Set proxies for a specific engine (YAML):
261 http: socks5://proxy:port
262 https: socks5://proxy:port
266 """To disable by default the engine, but not deleting it. It will allow the
267 user to manually activate it in the settings."""
270 """Remove the engine from the settings (*disabled & removed*)."""
273 """Additional fields describing the engine.
278 website: https://example.com
280 official_api_documentation: https://example.com/api-doc
281 use_official_api: true
282 require_api_key: true
286 using_tor_proxy: bool
287 """Using tor proxy (``true``) or not (``false``) for this engine."""
289 send_accept_language_header: bool
290 """When this option is activated, the language (locale) that is selected by
291 the user is used to build and send a ``Accept-Language`` header in the
292 request to the origin search engine."""
295 """A list of secret tokens to make this engine *private*, more details see
296 :ref:`private engines`."""
299 """Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
str secret_hash(self, str|bytes name)
__init__(self, str engine_name, int|None expire=None)
bool set(self, str key, Any value, int|None expire=None)
Any get(self, str key, default=None)
maintenance(bool force=True)