.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
zlibrary.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""`Z-Library`_ (abbreviated as z-lib, formerly BookFinder) is a shadow library
3project for file-sharing access to scholarly journal articles, academic texts
4and general-interest books. It began as a mirror of Library Genesis, from which
5most of its books originate.
6
7.. _Z-Library: https://zlibrary-global.se/
8
9Configuration
10=============
11
12The engine has the following additional settings:
13
14- :py:obj:`zlib_year_from`
15- :py:obj:`zlib_year_to`
16- :py:obj:`zlib_ext`
17
18With this options a SearXNG maintainer is able to configure **additional**
19engines for specific searches in Z-Library. For example a engine to search
20only for EPUB from 2010 to 2020.
21
22.. code:: yaml
23
24 - name: z-library 2010s epub
25 engine: zlibrary
26 shortcut: zlib2010s
27 zlib_year_from: '2010'
28 zlib_year_to: '2020'
29 zlib_ext: 'EPUB'
30
31Implementations
32===============
33
34"""
35
36import typing as t
37from datetime import datetime
38from urllib.parse import quote
39from lxml import html
40from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
41
42from searx.utils import extract_text, eval_xpath, eval_xpath_list
43from searx.enginelib.traits import EngineTraits
44from searx.data import ENGINE_TRAITS
45from searx.exceptions import SearxException
46
47if t.TYPE_CHECKING:
48 from searx.extended_types import SXNG_Response
49
50# about
51about: dict[str, t.Any] = {
52 "website": "https://zlibrary-global.se",
53 "wikidata_id": "Q104863992",
54 "official_api_documentation": None,
55 "use_official_api": False,
56 "require_api_key": False,
57 "results": "HTML",
58}
59
60categories: list[str] = ["files"]
61paging: bool = True
62base_url: str = "https://zlibrary-global.se"
63
64zlib_year_from: str = ""
65"""Filter z-library's results by year from. E.g '2010'.
66"""
67
68zlib_year_to: str = ""
69"""Filter z-library's results by year to. E.g. '2010'.
70"""
71
72zlib_ext: str = ""
73"""Filter z-library's results by a file ending. Common filters for example are
74``PDF`` and ``EPUB``.
75"""
76
77
78def init(engine_settings: dict[str, t.Any] | None = None) -> None: # pylint: disable=unused-argument
79 """Check of engine's settings."""
80 traits: EngineTraits = EngineTraits(**ENGINE_TRAITS["z-library"])
81
82 if zlib_ext and zlib_ext not in traits.custom["ext"]:
83 raise ValueError(f"invalid setting ext: {zlib_ext}")
84 if zlib_year_from and zlib_year_from not in traits.custom["year_from"]:
85 raise ValueError(f"invalid setting year_from: {zlib_year_from}")
86 if zlib_year_to and zlib_year_to not in traits.custom["year_to"]:
87 raise ValueError(f"invalid setting year_to: {zlib_year_to}")
88
89
90def request(query: str, params: dict[str, t.Any]) -> dict[str, t.Any]:
91 lang: str = traits.get_language(params["language"], traits.all_locale) # type: ignore
92 search_url: str = (
93 base_url
94 + "/s/{search_query}/?page={pageno}"
95 + "&yearFrom={zlib_year_from}"
96 + "&yearTo={zlib_year_to}"
97 + "&languages[]={lang}"
98 + "&extensions[]={zlib_ext}"
99 )
100 params["url"] = search_url.format(
101 search_query=quote(query),
102 pageno=params["pageno"],
103 lang=lang,
104 zlib_year_from=zlib_year_from,
105 zlib_year_to=zlib_year_to,
106 zlib_ext=zlib_ext,
107 )
108 params["verify"] = False
109 return params
110
111
113 return bool(dom.xpath('//title') and "seized" in dom.xpath('//title')[0].text.lower())
114
115
116def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]:
117 results: list[dict[str, t.Any]] = []
118 dom = html.fromstring(resp.text)
119
120 if domain_is_seized(dom):
121 raise SearxException(f"zlibrary domain is seized: {base_url}")
122
123 for item in dom.xpath('//div[@id="searchResultBox"]//div[contains(@class, "resItemBox")]'):
124 results.append(_parse_result(item))
125
126 return results
127
128
129def _text(item, selector: str) -> str | None:
130 return extract_text(eval_xpath(item, selector))
131
132
133i18n_language = gettext("Language")
134i18n_book_rating = gettext("Book rating")
135i18n_file_quality = gettext("File quality")
136
137
138def _parse_result(item) -> dict[str, t.Any]:
139
140 author_elements = eval_xpath_list(item, './/div[@class="authors"]//a[@itemprop="author"]')
141
142 result = {
143 "template": "paper.html",
144 "url": base_url + item.xpath('(.//a[starts-with(@href, "/book/")])[1]/@href')[0],
145 "title": _text(item, './/*[@itemprop="name"]'),
146 "authors": [extract_text(author) for author in author_elements],
147 "publisher": _text(item, './/a[@title="Publisher"]'),
148 "type": _text(item, './/div[contains(@class, "property__file")]//div[contains(@class, "property_value")]'),
149 }
150
151 thumbnail: str = _text(item, './/img[contains(@class, "cover")]/@data-src')
152 if not thumbnail.startswith('/'):
153 result["thumbnail"] = thumbnail
154
155 year = _text(item, './/div[contains(@class, "property_year")]//div[contains(@class, "property_value")]')
156 if year:
157 result["publishedDate"] = datetime.strptime(year, '%Y')
158
159 content = []
160 language = _text(item, './/div[contains(@class, "property_language")]//div[contains(@class, "property_value")]')
161 if language:
162 content.append(f"{i18n_language}: {language.capitalize()}")
163 book_rating = _text(item, './/span[contains(@class, "book-rating-interest-score")]')
164 if book_rating and float(book_rating):
165 content.append(f"{i18n_book_rating}: {book_rating}")
166 file_quality = _text(item, './/span[contains(@class, "book-rating-quality-score")]')
167 if file_quality and float(file_quality):
168 content.append(f"{i18n_file_quality}: {file_quality}")
169 result["content"] = " | ".join(content)
170
171 return result
172
173
174def fetch_traits(engine_traits: EngineTraits) -> None:
175 """Fetch languages and other search arguments from zlibrary's search form."""
176 # pylint: disable=import-outside-toplevel, too-many-branches
177
178 import babel
179 import httpx
180
181 from searx.network import get # see https://github.com/searxng/searxng/issues/762
182 from searx.locales import language_tag
183
184 def _use_old_values():
185 # don't change anything, re-use the existing values
186 engine_traits.all_locale = ENGINE_TRAITS["z-library"]["all_locale"]
187 engine_traits.custom = ENGINE_TRAITS["z-library"]["custom"]
188 engine_traits.languages = ENGINE_TRAITS["z-library"]["languages"]
189
190 try:
191 resp = get(base_url, verify=False)
192 except (SearxException, httpx.HTTPError) as exc:
193 print(f"ERROR: zlibrary domain '{base_url}' is seized?")
194 print(f" --> {exc}")
195 _use_old_values()
196 return
197
198 if not resp.ok:
199 raise RuntimeError("Response from zlibrary's search page is not OK.")
200 dom = html.fromstring(resp.text) # type: ignore
201
202 if domain_is_seized(dom):
203 print(f"ERROR: zlibrary domain is seized: {base_url}")
204 _use_old_values()
205 return
206
207 engine_traits.all_locale = ""
208 engine_traits.custom["ext"] = []
209 engine_traits.custom["year_from"] = []
210 engine_traits.custom["year_to"] = []
211
212 for year in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_yearFrom']/option"):
213 engine_traits.custom["year_from"].append(year.get("value"))
214
215 for year in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_yearTo']/option"):
216 engine_traits.custom["year_to"].append(year.get("value"))
217
218 for ext in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_extensions']/option"):
219 value: str | None = ext.get("value")
220 if value is None:
221 value = ""
222 engine_traits.custom["ext"].append(value)
223
224 # Handle languages
225 # Z-library uses English names for languages, so we need to map them to their respective locales
226 language_name_locale_map: dict[str, babel.Locale] = {}
227 for locale in babel.core.localedata.locale_identifiers(): # type: ignore
228 # Create a Locale object for the current locale
229 loc = babel.Locale.parse(locale)
230 if loc.english_name is None:
231 continue
232 language_name_locale_map[loc.english_name.lower()] = loc
233
234 for x in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_languages']/option"):
235 eng_lang = x.get("value")
236 if eng_lang is None:
237 continue
238 try:
239 locale = language_name_locale_map[eng_lang.lower()]
240 except KeyError:
241 # silently ignore unknown languages
242 # print("ERROR: %s is unknown by babel" % (eng_lang))
243 continue
244 sxng_lang = language_tag(locale)
245 conflict = engine_traits.languages.get(sxng_lang)
246 if conflict:
247 if conflict != eng_lang:
248 print("CONFLICT: babel %s --> %s, %s" % (sxng_lang, conflict, eng_lang))
249 continue
250 engine_traits.languages[sxng_lang] = eng_lang
None init(dict[str, t.Any]|None engine_settings=None)
Definition zlibrary.py:78
dict[str, t.Any] request(str query, dict[str, t.Any] params)
Definition zlibrary.py:90
None fetch_traits(EngineTraits engine_traits)
Definition zlibrary.py:174
list[dict[str, t.Any]] response("SXNG_Response" resp)
Definition zlibrary.py:116
dict[str, t.Any] _parse_result(item)
Definition zlibrary.py:138
str|None _text(item, str selector)
Definition zlibrary.py:129