.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
yahoo.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Yahoo Search (Web)
3
4Languages are supported by mapping the language to a domain. If domain is not
5found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.
6
7"""
8
9from typing import TYPE_CHECKING
10from urllib.parse import (
11 unquote,
12 urlencode,
13)
14from lxml import html
15
16from searx.utils import (
17 eval_xpath_getindex,
18 eval_xpath_list,
19 extract_text,
20 html_to_text,
21)
22from searx.enginelib.traits import EngineTraits
23
24traits: EngineTraits
25
26if TYPE_CHECKING:
27 import logging
28
29 logger: logging.Logger
30
31# about
32about = {
33 "website": 'https://search.yahoo.com/',
34 "wikidata_id": None,
35 "official_api_documentation": 'https://developer.yahoo.com/api/',
36 "use_official_api": False,
37 "require_api_key": False,
38 "results": 'HTML',
39}
40
41# engine dependent config
42categories = ['general', 'web']
43paging = True
44time_range_support = True
45# send_accept_language_header = True
46
47time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm'}
48safesearch_dict = {0: 'p', 1: 'i', 2: 'r'}
49
50region2domain = {
51 "CO": "co.search.yahoo.com", # Colombia
52 "TH": "th.search.yahoo.com", # Thailand
53 "VE": "ve.search.yahoo.com", # Venezuela
54 "CL": "cl.search.yahoo.com", # Chile
55 "HK": "hk.search.yahoo.com", # Hong Kong
56 "PE": "pe.search.yahoo.com", # Peru
57 "CA": "ca.search.yahoo.com", # Canada
58 "DE": "de.search.yahoo.com", # Germany
59 "FR": "fr.search.yahoo.com", # France
60 "TW": "tw.search.yahoo.com", # Taiwan
61 "GB": "uk.search.yahoo.com", # United Kingdom
62 "UK": "uk.search.yahoo.com",
63 "BR": "br.search.yahoo.com", # Brazil
64 "IN": "in.search.yahoo.com", # India
65 "ES": "espanol.search.yahoo.com", # Espanol
66 "PH": "ph.search.yahoo.com", # Philippines
67 "AR": "ar.search.yahoo.com", # Argentina
68 "MX": "mx.search.yahoo.com", # Mexico
69 "SG": "sg.search.yahoo.com", # Singapore
70}
71"""Map regions to domain"""
72
73lang2domain = {
74 'zh_chs': 'hk.search.yahoo.com',
75 'zh_cht': 'tw.search.yahoo.com',
76 'any': 'search.yahoo.com',
77 'en': 'search.yahoo.com',
78 'bg': 'search.yahoo.com',
79 'cs': 'search.yahoo.com',
80 'da': 'search.yahoo.com',
81 'el': 'search.yahoo.com',
82 'et': 'search.yahoo.com',
83 'he': 'search.yahoo.com',
84 'hr': 'search.yahoo.com',
85 'ja': 'search.yahoo.com',
86 'ko': 'search.yahoo.com',
87 'sk': 'search.yahoo.com',
88 'sl': 'search.yahoo.com',
89}
90"""Map language to domain"""
91
92yahoo_languages = {
93 "all": "any",
94 "ar": "ar", # Arabic
95 "bg": "bg", # Bulgarian
96 "cs": "cs", # Czech
97 "da": "da", # Danish
98 "de": "de", # German
99 "el": "el", # Greek
100 "en": "en", # English
101 "es": "es", # Spanish
102 "et": "et", # Estonian
103 "fi": "fi", # Finnish
104 "fr": "fr", # French
105 "he": "he", # Hebrew
106 "hr": "hr", # Croatian
107 "hu": "hu", # Hungarian
108 "it": "it", # Italian
109 "ja": "ja", # Japanese
110 "ko": "ko", # Korean
111 "lt": "lt", # Lithuanian
112 "lv": "lv", # Latvian
113 "nl": "nl", # Dutch
114 "no": "no", # Norwegian
115 "pl": "pl", # Polish
116 "pt": "pt", # Portuguese
117 "ro": "ro", # Romanian
118 "ru": "ru", # Russian
119 "sk": "sk", # Slovak
120 "sl": "sl", # Slovenian
121 "sv": "sv", # Swedish
122 "th": "th", # Thai
123 "tr": "tr", # Turkish
124 "zh": "zh_chs", # Chinese (Simplified)
125 "zh_Hans": "zh_chs",
126 'zh-CN': "zh_chs",
127 "zh_Hant": "zh_cht", # Chinese (Traditional)
128 "zh-HK": "zh_cht",
129 'zh-TW': "zh_cht",
130}
131
132
133def build_sb_cookie(cookie_params):
134 """Build sB cookie parameter from provided parameters.
135
136 :param cookie_params: Dictionary of cookie parameters
137 :type cookie_params: dict
138 :returns: Formatted cookie string
139 :rtype: str
140
141 Example:
142 >>> cookie_params = {'v': '1', 'vm': 'p', 'fl': '1', 'vl': 'lang_fr'}
143 >>> build_sb_cookie(cookie_params)
144 'v=1&vm=p&fl=1&vl=lang_fr'
145 """
146
147 cookie_parts = []
148 for key, value in cookie_params.items():
149 cookie_parts.append(f"{key}={value}")
150
151 return "&".join(cookie_parts)
152
153
154def request(query, params):
155 """Build Yahoo search request."""
156
157 lang, region = (params["language"].split("-") + [None])[:2]
158 lang = yahoo_languages.get(lang, "any")
159
160 # Build URL parameters
161 # - p (str): Search query string
162 # - btf (str): Time filter, maps to values like 'd' (day), 'w' (week), 'm' (month)
163 # - iscqry (str): Empty string, necessary for results to appear properly on first page
164 # - b (int): Search offset for pagination
165 # - pz (str): Amount of results expected for the page
166 url_params = {'p': query}
167
168 btf = time_range_dict.get(params['time_range'])
169 if btf:
170 url_params['btf'] = btf
171
172 if params['pageno'] == 1:
173 url_params['iscqry'] = ''
174 elif params['pageno'] >= 2:
175 url_params['b'] = params['pageno'] * 7 + 1 # 8, 15, 21, etc.
176 url_params['pz'] = 7
177 url_params['bct'] = 0
178 url_params['xargs'] = 0
179
180 # Build sB cookie (for filters)
181 # - vm (str): SafeSearch filter, maps to values like 'p' (None), 'i' (Moderate), 'r' (Strict)
182 # - fl (bool): Indicates if a search language is used or not
183 # - vl (str): The search language to use (e.g. lang_fr)
184 sbcookie_params = {
185 'v': 1,
186 'vm': safesearch_dict[params['safesearch']],
187 'fl': 1,
188 'vl': f'lang_{lang}',
189 'pn': 10,
190 'rw': 'new',
191 'userset': 1,
192 }
193 params['cookies']['sB'] = build_sb_cookie(sbcookie_params)
194
195 # Search region/language
196 domain = region2domain.get(region)
197 if not domain:
198 domain = lang2domain.get(lang, f'{lang}.search.yahoo.com')
199 logger.debug(f'domain selected: {domain}')
200 logger.debug(f'cookies: {params["cookies"]}')
201
202 params['url'] = f'https://{domain}/search?{urlencode(url_params)}'
203 params['domain'] = domain
204
205
206def parse_url(url_string):
207 """remove yahoo-specific tracking-url"""
208
209 endings = ['/RS', '/RK']
210 endpositions = []
211 start = url_string.find('http', url_string.find('/RU=') + 1)
212
213 for ending in endings:
214 endpos = url_string.rfind(ending)
215 if endpos > -1:
216 endpositions.append(endpos)
217
218 if start == 0 or len(endpositions) == 0:
219 return url_string
220
221 end = min(endpositions)
222 return unquote(url_string[start:end])
223
224
225def response(resp):
226 """parse response"""
227
228 results = []
229 dom = html.fromstring(resp.text)
230
231 url_xpath = './/div[contains(@class,"compTitle")]/h3/a/@href'
232 title_xpath = './/h3//a/@aria-label'
233
234 domain = resp.search_params['domain']
235 if domain == "search.yahoo.com":
236 url_xpath = './/div[contains(@class,"compTitle")]/a/@href'
237 title_xpath = './/div[contains(@class,"compTitle")]/a/h3/span'
238
239 # parse results
240 for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
241 url = eval_xpath_getindex(result, url_xpath, 0, default=None)
242 if url is None:
243 continue
244 url = parse_url(url)
245
246 title = eval_xpath_getindex(result, title_xpath, 0, default='')
247 title: str = extract_text(title)
248 content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='')
249 content: str = extract_text(content, allow_none=True)
250
251 # append result
252 results.append(
253 {
254 'url': url,
255 # title sometimes contains HTML tags / see
256 # https://github.com/searxng/searxng/issues/3790
257 'title': " ".join(html_to_text(title).strip().split()),
258 'content': " ".join(html_to_text(content).strip().split()),
259 }
260 )
261
262 for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]//table//a'):
263 # append suggestion
264 results.append({'suggestion': extract_text(suggestion)})
265
266 return results
parse_url(url_string)
Definition yahoo.py:206
request(query, params)
Definition yahoo.py:154
build_sb_cookie(cookie_params)
Definition yahoo.py:133