.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
mojeek.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Mojeek (general, images, news)"""
3
4from datetime import datetime
5from urllib.parse import urlencode
6from lxml import html
7
8from dateutil.relativedelta import relativedelta
9from searx.utils import eval_xpath, eval_xpath_list, extract_text
10from searx.enginelib.traits import EngineTraits
11
12about = {
13 'website': 'https://mojeek.com',
14 'wikidata_id': 'Q60747299',
15 'official_api_documentation': 'https://www.mojeek.com/support/api/search/request_parameters.html',
16 'use_official_api': False,
17 'require_api_key': False,
18 'results': 'HTML',
19}
20paging = True # paging is only supported for general search
21safesearch = True
22time_range_support = True # time range search is supported for general and news
23max_page = 10
24
25base_url = "https://www.mojeek.com"
26
27categories = ["general", "web"]
28search_type = "" # leave blank for general, other possible values: images, news
29
30results_xpath = '//ul[@class="results-standard"]/li/a[@class="ob"]'
31url_xpath = './@href'
32title_xpath = '../h2/a'
33content_xpath = '..//p[@class="s"]'
34suggestion_xpath = '//div[@class="top-info"]/p[@class="top-info spell"]/em/a'
35
36image_results_xpath = '//div[@id="results"]/div[contains(@class, "image")]'
37image_url_xpath = './a/@href'
38image_title_xpath = './a/@data-title'
39image_img_src_xpath = './a/img/@src'
40
41news_results_xpath = '//section[contains(@class, "news-search-result")]//article'
42news_url_xpath = './/h2/a/@href'
43news_title_xpath = './/h2/a'
44news_content_xpath = './/p[@class="s"]'
45
46language_param = 'lb'
47region_param = 'arc'
48
49_delta_kwargs = {'day': 'days', 'week': 'weeks', 'month': 'months', 'year': 'years'}
50
51
52def init(_):
53 if search_type not in ('', 'images', 'news'):
54 raise ValueError(f"Invalid search type {search_type}")
55
56
57def request(query, params):
58 args = {
59 'q': query,
60 'safe': min(params['safesearch'], 1),
61 language_param: traits.get_language(params['searxng_locale'], traits.custom['language_all']),
62 region_param: traits.get_region(params['searxng_locale'], traits.custom['region_all']),
63 }
64
65 if search_type:
66 args['fmt'] = search_type
67
68 if search_type == '':
69 args['s'] = 10 * (params['pageno'] - 1)
70
71 if params['time_range'] and search_type != 'images':
72 kwargs = {_delta_kwargs[params['time_range']]: 1}
73 args["since"] = (datetime.now() - relativedelta(**kwargs)).strftime("%Y%m%d") # type: ignore
74 logger.debug(args["since"])
75
76 params['url'] = f"{base_url}/search?{urlencode(args)}"
77
78 return params
79
80
82 results = []
83
84 for result in eval_xpath_list(dom, results_xpath):
85 results.append(
86 {
87 'url': extract_text(eval_xpath(result, url_xpath)),
88 'title': extract_text(eval_xpath(result, title_xpath)),
89 'content': extract_text(eval_xpath(result, content_xpath)),
90 }
91 )
92
93 for suggestion in eval_xpath(dom, suggestion_xpath):
94 results.append({'suggestion': extract_text(suggestion)})
95
96 return results
97
98
100 results = []
101
102 for result in eval_xpath_list(dom, image_results_xpath):
103 results.append(
104 {
105 'template': 'images.html',
106 'url': extract_text(eval_xpath(result, image_url_xpath)),
107 'title': extract_text(eval_xpath(result, image_title_xpath)),
108 'img_src': base_url + extract_text(eval_xpath(result, image_img_src_xpath)), # type: ignore
109 'content': '',
110 }
111 )
112
113 return results
114
115
117 results = []
118
119 for result in eval_xpath_list(dom, news_results_xpath):
120 results.append(
121 {
122 'url': extract_text(eval_xpath(result, news_url_xpath)),
123 'title': extract_text(eval_xpath(result, news_title_xpath)),
124 'content': extract_text(eval_xpath(result, news_content_xpath)),
125 }
126 )
127
128 return results
129
130
131def response(resp):
132 dom = html.fromstring(resp.text)
133
134 if search_type == '':
135 return _general_results(dom)
136
137 if search_type == 'images':
138 return _image_results(dom)
139
140 if search_type == 'news':
141 return _news_results(dom)
142
143 raise ValueError(f"Invalid search type {search_type}")
144
145
146def fetch_traits(engine_traits: EngineTraits):
147 # pylint: disable=import-outside-toplevel
148 from searx import network
149 from searx.locales import get_official_locales, region_tag
150 from babel import Locale, UnknownLocaleError
151 import contextlib
152
153 resp = network.get(base_url + "/preferences", headers={'Accept-Language': 'en-US,en;q=0.5'})
154 dom = html.fromstring(resp.text) # type: ignore
155
156 languages = eval_xpath_list(dom, f'//select[@name="{language_param}"]/option/@value')
157
158 engine_traits.custom['language_all'] = languages[0]
159
160 for code in languages[1:]:
161 with contextlib.suppress(UnknownLocaleError):
162 locale = Locale(code)
163 engine_traits.languages[locale.language] = code
164
165 regions = eval_xpath_list(dom, f'//select[@name="{region_param}"]/option/@value')
166
167 engine_traits.custom['region_all'] = regions[1]
168
169 for code in regions[2:]:
170 for locale in get_official_locales(code, engine_traits.languages):
171 engine_traits.regions[region_tag(locale)] = code
request(query, params)
Definition mojeek.py:57
fetch_traits(EngineTraits engine_traits)
Definition mojeek.py:146