40def response(resp):
41
42 start_tag = 'window.MESON.initialState = {'
43 end_tag = '}};'
44
45 dom = html.fromstring(resp.text)
46 script = utils.eval_xpath_getindex(dom, '//script', 0, default=None).text
47
48 pos = script.index(start_tag) + len(start_tag) - 1
49 script = script[pos:]
50 pos = script.index(end_tag) + len(end_tag) - 1
51 script = script[:pos]
52
53 json_resp = utils.js_variable_to_python(script)
54
55 results = []
56
57 for item in json_resp['search']['webResults']['results']:
58
59 pubdate_original = item.get('pubdate_original')
60 if pubdate_original:
61 pubdate_original = dateutil.parser.parse(pubdate_original)
62 metadata = [item.get(field) for field in ['category_l1', 'catsy'] if item.get(field)]
63
64 results.append(
65 {
66 "url": item['url'].split('&ueid')[0],
67 "title": item['title'],
68 "content": item['abstract'],
69 "publishedDate": pubdate_original,
70
71 "metadata": ' | '.join(metadata),
72 }
73 )
74
75 return results
1.13.2