.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
google_scholar.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""This is the implementation of the Google Scholar engine.
3
4Compared to other Google services the Scholar engine has a simple GET REST-API
5and there does not exists `async` API. Even though the API slightly vintage we
6can make use of the :ref:`google API` to assemble the arguments of the GET
7request.
8"""
9
10from urllib.parse import urlencode
11from datetime import datetime
12from lxml import html
13
14from searx.utils import (
15 eval_xpath,
16 eval_xpath_getindex,
17 eval_xpath_list,
18 extract_text,
19)
20
21from searx.exceptions import SearxEngineCaptchaException
22
23from searx.engines.google import fetch_traits # pylint: disable=unused-import
24from searx.engines.google import (
25 get_google_info,
26 time_range_dict,
27)
28
29# about
30about = {
31 "website": 'https://scholar.google.com',
32 "wikidata_id": 'Q494817',
33 "official_api_documentation": 'https://developers.google.com/custom-search',
34 "use_official_api": False,
35 "require_api_key": False,
36 "results": 'HTML',
37}
38
39# engine dependent config
40categories = ['science', 'scientific publications']
41paging = True
42max_page = 50
43"""`Google max 50 pages`_
44
45.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
46"""
47language_support = True
48time_range_support = True
49safesearch = False
50send_accept_language_header = True
51
52
53def time_range_args(params):
54 """Returns a dictionary with a time range arguments based on
55 ``params['time_range']``.
56
57 Google Scholar supports a detailed search by year. Searching by *last
58 month* or *last week* (as offered by SearXNG) is uncommon for scientific
59 publications and is not supported by Google Scholar.
60
61 To limit the result list when the users selects a range, all the SearXNG
62 ranges (*day*, *week*, *month*, *year*) are mapped to *year*. If no range
63 is set an empty dictionary of arguments is returned. Example; when
64 user selects a time range (current year minus one in 2022):
65
66 .. code:: python
67
68 { 'as_ylo' : 2021 }
69
70 """
71 ret_val = {}
72 if params['time_range'] in time_range_dict:
73 ret_val['as_ylo'] = datetime.now().year - 1
74 return ret_val
75
76
78 """In case of CAPTCHA Google Scholar open its own *not a Robot* dialog and is
79 not redirected to ``sorry.google.com``.
80 """
81 if eval_xpath(dom, "//form[@id='gs_captcha_f']"):
83
84
85def request(query, params):
86 """Google-Scholar search request"""
87
88 google_info = get_google_info(params, traits)
89 # subdomain is: scholar.google.xy
90 google_info['subdomain'] = google_info['subdomain'].replace("www.", "scholar.")
91
92 args = {
93 'q': query,
94 **google_info['params'],
95 'start': (params['pageno'] - 1) * 10,
96 'as_sdt': '2007', # include patents / to disable set '0,5'
97 'as_vis': '0', # include citations / to disable set '1'
98 }
99 args.update(time_range_args(params))
100
101 params['url'] = 'https://' + google_info['subdomain'] + '/scholar?' + urlencode(args)
102 params['cookies'] = google_info['cookies']
103 params['headers'].update(google_info['headers'])
104 return params
105
106
107def parse_gs_a(text: str | None):
108 """Parse the text written in green.
109
110 Possible formats:
111 * "{authors} - {journal}, {year} - {publisher}"
112 * "{authors} - {year} - {publisher}"
113 * "{authors} - {publisher}"
114 """
115 if text is None or text == "":
116 return None, None, None, None
117
118 s_text = text.split(' - ')
119 authors = s_text[0].split(', ')
120 publisher = s_text[-1]
121 if len(s_text) != 3:
122 return authors, None, publisher, None
123
124 # the format is "{authors} - {journal}, {year} - {publisher}" or "{authors} - {year} - {publisher}"
125 # get journal and year
126 journal_year = s_text[1].split(', ')
127 # journal is optional and may contains some coma
128 if len(journal_year) > 1:
129 journal = ', '.join(journal_year[0:-1])
130 if journal == '…':
131 journal = None
132 else:
133 journal = None
134 # year
135 year = journal_year[-1]
136 try:
137 publishedDate = datetime.strptime(year.strip(), '%Y')
138 except ValueError:
139 publishedDate = None
140 return authors, journal, publisher, publishedDate
141
142
143def response(resp): # pylint: disable=too-many-locals
144 """Parse response from Google Scholar"""
145 results = []
146
147 # convert the text to dom
148 dom = html.fromstring(resp.text)
149 detect_google_captcha(dom)
150
151 # parse results
152 for result in eval_xpath_list(dom, '//div[@data-rp]'):
153
154 title = extract_text(eval_xpath(result, './/h3[1]//a'))
155
156 if not title:
157 # this is a [ZITATION] block
158 continue
159
160 pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ctg2"]'))
161 if pub_type:
162 pub_type = pub_type[1:-1].lower()
163
164 url = eval_xpath_getindex(result, './/h3[1]//a/@href', 0)
165 content = extract_text(eval_xpath(result, './/div[@class="gs_rs"]'))
166 authors, journal, publisher, publishedDate = parse_gs_a(
167 extract_text(eval_xpath(result, './/div[@class="gs_a"]'))
168 )
169 if publisher in url:
170 publisher = None
171
172 # cited by
173 comments = extract_text(eval_xpath(result, './/div[@class="gs_fl"]/a[starts-with(@href,"/scholar?cites=")]'))
174
175 # link to the html or pdf document
176 html_url = None
177 pdf_url = None
178 doc_url = eval_xpath_getindex(result, './/div[@class="gs_or_ggsm"]/a/@href', 0, default=None)
179 doc_type = extract_text(eval_xpath(result, './/span[@class="gs_ctg2"]'))
180 if doc_type == "[PDF]":
181 pdf_url = doc_url
182 else:
183 html_url = doc_url
184
185 results.append(
186 {
187 'template': 'paper.html',
188 'type': pub_type,
189 'url': url,
190 'title': title,
191 'authors': authors,
192 'publisher': publisher,
193 'journal': journal,
194 'publishedDate': publishedDate,
195 'content': content,
196 'comments': comments,
197 'html_url': html_url,
198 'pdf_url': pdf_url,
199 }
200 )
201
202 # parse suggestion
203 for suggestion in eval_xpath(dom, '//div[contains(@class, "gs_qsuggest_wrap")]//li//a'):
204 # append suggestion
205 results.append({'suggestion': extract_text(suggestion)})
206
207 for correction in eval_xpath(dom, '//div[@class="gs_r gs_pda"]/a'):
208 results.append({'correction': extract_text(correction)})
209
210 return results