.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
duckduckgo_extra.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""
3DuckDuckGo Extra (images, videos, news)
4~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5"""
6
7from datetime import datetime
8from urllib.parse import urlencode
9from searx.utils import get_embeded_stream_url, html_to_text
10
11from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
12from searx.engines.duckduckgo import get_ddg_lang, get_vqd
13
14# about
15about = {
16 "website": 'https://duckduckgo.com/',
17 "wikidata_id": 'Q12805',
18 "use_official_api": False,
19 "require_api_key": False,
20 "results": 'JSON (site requires js to get images)',
21}
22
23# engine dependent config
24categories = ['images', 'web']
25ddg_category = 'images'
26"""The category must be any of ``images``, ``videos`` and ``news``
27"""
28paging = True
29safesearch = True
30send_accept_language_header = True
31
32safesearch_cookies = {0: '-2', 1: None, 2: '1'}
33safesearch_args = {0: '1', 1: None, 2: '1'}
34
35search_path_map = {'images': 'i', 'videos': 'v', 'news': 'news'}
36
37
38def request(query, params):
39 eng_region: str = traits.get_region(params['searxng_locale'], traits.all_locale) # type: ignore
40
41 # request needs a vqd argument
42 vqd = get_vqd(query, eng_region, force_request=True)
43
44 if not vqd:
45 # some search terms do not have results and therefore no vqd value
46 params['url'] = None
47 return params
48
49 eng_lang = get_ddg_lang(traits, params['searxng_locale'])
50
51 args = {
52 'q': query,
53 'o': 'json',
54 # 'u': 'bing',
55 'l': eng_region,
56 'f': ',,,,,',
57 'vqd': vqd,
58 }
59
60 if params['pageno'] > 1:
61 args['s'] = (params['pageno'] - 1) * 100
62
63 params['cookies']['ad'] = eng_lang # zh_CN
64 params['cookies']['ah'] = eng_region # "us-en,de-de"
65 params['cookies']['l'] = eng_region # "hk-tzh"
66
67 safe_search = safesearch_cookies.get(params['safesearch'])
68 if safe_search is not None:
69 params['cookies']['p'] = safe_search # "-2", "1"
70 safe_search = safesearch_args.get(params['safesearch'])
71 if safe_search is not None:
72 args['p'] = safe_search # "-1", "1"
73
74 logger.debug("cookies: %s", params['cookies'])
75
76 params['url'] = f'https://duckduckgo.com/{search_path_map[ddg_category]}.js?{urlencode(args)}'
77
78 # sending these two headers prevents rate limiting for the query
79 params['headers'] = {
80 'Referer': 'https://duckduckgo.com/',
81 'X-Requested-With': 'XMLHttpRequest',
82 }
83
84 return params
85
86
87def _image_result(result):
88 return {
89 'template': 'images.html',
90 'url': result['url'],
91 'title': result['title'],
92 'content': '',
93 'thumbnail_src': result['thumbnail'],
94 'img_src': result['image'],
95 'resolution': '%s x %s' % (result['width'], result['height']),
96 'source': result['source'],
97 }
98
99
100def _video_result(result):
101 return {
102 'template': 'videos.html',
103 'url': result['content'],
104 'title': result['title'],
105 'content': result['description'],
106 'thumbnail': result['images'].get('small') or result['images'].get('medium'),
107 'iframe_src': get_embeded_stream_url(result['content']),
108 'source': result['provider'],
109 'length': result['duration'],
110 'metadata': result.get('uploader'),
111 }
112
113
114def _news_result(result):
115 return {
116 'url': result['url'],
117 'title': result['title'],
118 'content': html_to_text(result['excerpt']),
119 'source': result['source'],
120 'publishedDate': datetime.fromtimestamp(result['date']),
121 }
122
123
124def response(resp):
125 results = []
126 res_json = resp.json()
127
128 for result in res_json['results']:
129 if ddg_category == 'images':
130 results.append(_image_result(result))
131 elif ddg_category == 'videos':
132 results.append(_video_result(result))
133 elif ddg_category == 'news':
134 results.append(_news_result(result))
135 else:
136 raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
137
138 return results