.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
archlinux.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""
3Arch Linux Wiki
4~~~~~~~~~~~~~~~
5
6This implementation does not use a official API: Mediawiki provides API, but
7Arch Wiki blocks access to it.
8
9"""
10
11from urllib.parse import urlencode, urljoin, urlparse
12import lxml
13import babel
14
15from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
16from searx.enginelib.traits import EngineTraits
17from searx.locales import language_tag
18
19
20about = {
21 "website": 'https://wiki.archlinux.org/',
22 "wikidata_id": 'Q101445877',
23 "official_api_documentation": None,
24 "use_official_api": False,
25 "require_api_key": False,
26 "results": 'HTML',
27}
28
29# engine dependent config
30categories = ['it', 'software wikis']
31paging = True
32main_wiki = 'wiki.archlinux.org'
33
34
35def request(query, params):
36
37 sxng_lang = params['searxng_locale'].split('-')[0]
38 netloc: str = traits.custom['wiki_netloc'].get(sxng_lang, main_wiki) # type: ignore
39 title: str = traits.custom['title'].get(sxng_lang, 'Special:Search') # type: ignore
40 base_url = 'https://' + netloc + '/index.php?'
41 offset = (params['pageno'] - 1) * 20
42
43 if netloc == main_wiki:
44 eng_lang: str = traits.get_language(sxng_lang, 'English') # type: ignore
45 query += ' (' + eng_lang + ')'
46 # wiki.archlinux.org is protected by anubis
47 # - https://github.com/searxng/searxng/issues/4646#issuecomment-2817848019
48 params['headers']['User-Agent'] = "SearXNG"
49 elif netloc == 'wiki.archlinuxcn.org':
50 base_url = 'https://' + netloc + '/wzh/index.php?'
51
52 args = {
53 'search': query,
54 'title': title,
55 'limit': 20,
56 'offset': offset,
57 'profile': 'default',
58 }
59
60 params['url'] = base_url + urlencode(args)
61 return params
62
63
64def response(resp):
65
66 results = []
67 dom = lxml.html.fromstring(resp.text) # type: ignore
68
69 # get the base URL for the language in which request was made
70 sxng_lang = resp.search_params['searxng_locale'].split('-')[0]
71 netloc: str = traits.custom['wiki_netloc'].get(sxng_lang, main_wiki) # type: ignore
72 base_url = 'https://' + netloc + '/index.php?'
73
74 for result in eval_xpath_list(dom, '//ul[@class="mw-search-results"]/li'):
75 link = eval_xpath_getindex(result, './/div[@class="mw-search-result-heading"]/a', 0)
76 content = extract_text(result.xpath('.//div[@class="searchresult"]'))
77 results.append(
78 {
79 'url': urljoin(base_url, link.get('href')), # type: ignore
80 'title': extract_text(link),
81 'content': content,
82 }
83 )
84
85 return results
86
87
88def fetch_traits(engine_traits: EngineTraits):
89 """Fetch languages from Archlinux-Wiki. The location of the Wiki address of a
90 language is mapped in a :py:obj:`custom field
91 <searx.enginelib.traits.EngineTraits.custom>` (``wiki_netloc``). Depending
92 on the location, the ``title`` argument in the request is translated.
93
94 .. code:: python
95
96 "custom": {
97 "wiki_netloc": {
98 "de": "wiki.archlinux.de",
99 # ...
100 "zh": "wiki.archlinuxcn.org"
101 }
102 "title": {
103 "de": "Spezial:Suche",
104 # ...
105 "zh": "Special:\u641c\u7d22"
106 },
107 },
108
109 """
110 # pylint: disable=import-outside-toplevel
111 from searx.network import get # see https://github.com/searxng/searxng/issues/762
112
113 engine_traits.custom['wiki_netloc'] = {}
114 engine_traits.custom['title'] = {}
115
116 title_map = {
117 'de': 'Spezial:Suche',
118 'fa': 'ویژه:جستجو',
119 'ja': '特別:検索',
120 'zh': 'Special:搜索',
121 }
122
123 resp = get('https://wiki.archlinux.org/')
124 if not resp.ok: # type: ignore
125 print("ERROR: response from wiki.archlinux.org is not OK.")
126
127 dom = lxml.html.fromstring(resp.text) # type: ignore
128 for a in eval_xpath_list(dom, "//a[@class='interlanguage-link-target']"):
129
130 sxng_tag = language_tag(babel.Locale.parse(a.get('lang'), sep='-'))
131 # zh_Hans --> zh
132 sxng_tag = sxng_tag.split('_')[0]
133
134 netloc = urlparse(a.get('href')).netloc
135 if netloc != 'wiki.archlinux.org':
136 title = title_map.get(sxng_tag)
137 if not title:
138 print("ERROR: title tag from %s (%s) is unknown" % (netloc, sxng_tag))
139 continue
140 engine_traits.custom['wiki_netloc'][sxng_tag] = netloc
141 engine_traits.custom['title'][sxng_tag] = title # type: ignore
142
143 eng_tag = extract_text(eval_xpath_list(a, ".//span"))
144 engine_traits.languages[sxng_tag] = eng_tag # type: ignore
145
146 engine_traits.languages['en'] = 'English'
request(query, params)
Definition archlinux.py:35
fetch_traits(EngineTraits engine_traits)
Definition archlinux.py:88