39def response(resp):
40 dom = html.fromstring(resp.text)
41 results = []
42
43 for item in dom.xpath('//li[contains(@id, "sogou_vr_")]'):
44 title = extract_text(item.xpath('.//h3/a'))
45 url = extract_text(item.xpath('.//h3/a/@href'))
46
47 if url.startswith("/link?url="):
48 url = f"{base_url}{url}"
49
50 content = extract_text(item.xpath('.//p[@class="txt-info"]'))
51 if not content:
52 content = extract_text(item.xpath('.//p[contains(@class, "txt-info")]'))
53
54 thumbnail = extract_text(item.xpath('.//div[@class="img-box"]/a/img/@src'))
55 if thumbnail and thumbnail.startswith("//"):
56 thumbnail = f"https:{thumbnail}"
57
58 published_date = None
59 timestamp = extract_text(item.xpath('.//script[contains(text(), "timeConvert")]'))
60 if timestamp:
61 match = re.search(r"timeConvert\('(\d+)'\)", timestamp)
62 if match:
63 published_date = datetime.fromtimestamp(int(match.group(1)))
64
65 if title and url:
66 results.append(
67 {
68 "title": title,
69 "url": url,
70 "content": content,
71 'thumbnail': thumbnail,
72 "publishedDate": published_date,
73 }
74 )
75
76 return results
1.13.2