199def request(query: str, params: dict[str, t.Any]) ->
None:
202 params[
'headers'][
'Accept-Encoding'] =
'gzip, deflate'
204 args: dict[str, t.Any] = {
209 args[
'spellcheck'] =
'1'
211 if brave_category
in (
'search',
'goggles'):
212 if params.get(
'pageno', 1) - 1:
213 args[
'offset'] = params.get(
'pageno', 1) - 1
214 if time_range_map.get(params[
'time_range']):
215 args[
'tf'] = time_range_map.get(params[
'time_range'])
217 if brave_category ==
'goggles':
218 args[
'goggles_id'] = Goggles
220 params[
"url"] = f
"{base_url}{brave_category}?{urlencode(args)}"
224 params[
'cookies'][
'safesearch'] = safesearch_map.get(params[
'safesearch'],
'off')
226 params[
'cookies'][
'useLocation'] =
'0'
227 params[
'cookies'][
'summarizer'] =
'0'
229 engine_region = traits.get_region(params[
'searxng_locale'],
'all')
230 params[
'cookies'][
'country'] = engine_region.split(
'-')[-1].lower()
232 ui_lang = locales.get_engine_locale(params[
'searxng_locale'], traits.custom[
"ui_lang"],
'en-us')
233 params[
'cookies'][
'ui_lang'] = ui_lang
235 logger.debug(
"cookies %s", params[
'cookies'])
237 params[
'headers'][
'Sec-Fetch-Dest'] =
"document"
238 params[
'headers'][
'Sec-Fetch-Mode'] =
"navigate"
239 params[
'headers'][
'Sec-Fetch-Site'] =
"same-origin"
240 params[
'headers'][
'Sec-Fetch-User'] =
"?1"
284 dom = html.fromstring(resp.text)
288 answer_tag = eval_xpath_getindex(dom,
'//div[@class="answer"]', 0, default=
None)
290 url = eval_xpath_getindex(dom,
'//div[@id="featured_snippet"]/a[@class="result-header"]/@href', 0, default=
None)
291 answer = extract_text(answer_tag)
292 if answer
is not None:
293 result_list.add(result_list.types.Answer(answer=answer, url=url))
296 xpath_results =
'//div[contains(@class, "snippet ")]'
298 for result
in eval_xpath_list(dom, xpath_results):
300 url = eval_xpath_getindex(result,
'.//a[contains(@class, "h")]/@href', 0, default=
None)
301 title_tag = eval_xpath_getindex(
302 result,
'.//a[contains(@class, "h")]//div[contains(@class, "title")]', 0, default=
None
304 if url
is None or title_tag
is None or not urlparse(url).netloc:
307 content: str = extract_text(
308 eval_xpath_getindex(result,
'.//div[contains(@class, "snippet-description")]', 0, default=
'')
310 pub_date_raw = eval_xpath(result,
'substring-before(.//div[contains(@class, "snippet-description")], "-")')
312 if pub_date
and content.startswith(pub_date_raw):
313 content = content.lstrip(pub_date_raw).strip(
"- \n\t")
315 thumbnail = eval_xpath_getindex(result,
'.//img[contains(@class, "thumb")]/@src', 0, default=
'')
319 'title': extract_text(title_tag),
321 'publishedDate': pub_date,
322 'thumbnail': thumbnail,
325 video_tag = eval_xpath_getindex(
326 result,
'.//div[contains(@class, "video-snippet") and @data-macro="video"]', 0, default=
None
328 if video_tag
is not None:
333 iframe_src = get_embeded_stream_url(url)
335 item[
'iframe_src'] = iframe_src
336 item[
'template'] =
'videos.html'
337 item[
'thumbnail'] = eval_xpath_getindex(video_tag,
'.//img/@src', 0, default=
'')
338 pub_date_raw = extract_text(
339 eval_xpath(video_tag,
'.//div[contains(@class, "snippet-attributes")]/div/text()')
343 item[
'thumbnail'] = eval_xpath_getindex(video_tag,
'.//img/@src', 0, default=
'')
345 result_list.append(item)
353 dom = html.fromstring(resp.text)
355 for result
in eval_xpath_list(dom,
'//div[contains(@class, "results")]//div[@data-type="news"]'):
360 url = eval_xpath_getindex(result,
'.//a[contains(@class, "result-header")]/@href', 0, default=
None)
364 title = extract_text(eval_xpath_list(result,
'.//span[contains(@class, "snippet-title")]'))
365 content = extract_text(eval_xpath_list(result,
'.//p[contains(@class, "desc")]'))
366 thumbnail = eval_xpath_getindex(result,
'.//div[contains(@class, "image-wrapper")]//img/@src', 0, default=
'')
372 "thumbnail": thumbnail,
375 result_list.append(item)
428 """Fetch :ref:`languages <brave languages>` and :ref:`regions <brave
429 regions>` from Brave."""
433 import babel.languages
437 engine_traits.custom[
"ui_lang"] = {}
440 'Accept-Encoding':
'gzip, deflate',
442 lang_map = {
'no':
'nb'}
446 resp = get(
'https://search.brave.com/settings', headers=headers)
449 print(
"ERROR: response from Brave is not OK.")
450 dom = html.fromstring(resp.text)
452 for option
in dom.xpath(
'//section//option[@value="en-us"]/../option'):
454 ui_lang = option.get(
'value')
456 l = babel.Locale.parse(ui_lang, sep=
'-')
458 sxng_tag = region_tag(babel.Locale.parse(ui_lang, sep=
'-'))
460 sxng_tag = language_tag(babel.Locale.parse(ui_lang, sep=
'-'))
462 except babel.UnknownLocaleError:
463 print(
"ERROR: can't determine babel locale of Brave's (UI) language %s" % ui_lang)
466 conflict = engine_traits.custom[
"ui_lang"].get(sxng_tag)
468 if conflict != ui_lang:
469 print(
"CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, ui_lang))
471 engine_traits.custom[
"ui_lang"][sxng_tag] = ui_lang
475 resp = get(
'https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js', headers=headers)
478 print(
"ERROR: response from Brave is not OK.")
480 country_js = resp.text[resp.text.index(
"options:{all") + len(
'options:') :]
481 country_js = country_js[: country_js.index(
"},k={default")]
482 country_tags = js_variable_to_python(country_js)
484 for k, v
in country_tags.items():
486 engine_traits.all_locale =
'all'
488 country_tag = v[
'value']
491 for lang_tag
in babel.languages.get_official_languages(country_tag, de_facto=
True):
492 lang_tag = lang_map.get(lang_tag, lang_tag)
493 sxng_tag = region_tag(babel.Locale.parse(
'%s_%s' % (lang_tag, country_tag.upper())))
496 conflict = engine_traits.regions.get(sxng_tag)
498 if conflict != country_tag:
499 print(
"CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, country_tag))
501 engine_traits.regions[sxng_tag] = country_tag