.oO SearXNG Developer Documentation Oo.
Loading...
Searching...
No Matches
peertube.py
Go to the documentation of this file.
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Peertube and :py:obj:`SepiaSearch <searx.engines.sepiasearch>` do share
3(more or less) the same REST API and the schema of the JSON result is identical.
4
5"""
6
7import re
8from urllib.parse import urlencode
9from datetime import datetime, timedelta
10from dateutil.parser import parse
11from dateutil.relativedelta import relativedelta
12
13import babel
14
15from searx.network import get # see https://github.com/searxng/searxng/issues/762
16from searx.locales import language_tag
17from searx.utils import html_to_text, humanize_number
18from searx.enginelib.traits import EngineTraits
19
20traits: EngineTraits
21
22about = {
23 # pylint: disable=line-too-long
24 "website": 'https://joinpeertube.org',
25 "wikidata_id": 'Q50938515',
26 "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos',
27 "use_official_api": True,
28 "require_api_key": False,
29 "results": 'JSON',
30}
31
32# engine dependent config
33categories = ["videos"]
34paging = True
35base_url = "https://peer.tube"
36"""Base URL of the Peertube instance. A list of instances is available at:
37
38- https://instances.joinpeertube.org/instances
39"""
40
41time_range_support = True
42time_range_table = {
43 'day': relativedelta(),
44 'week': relativedelta(weeks=-1),
45 'month': relativedelta(months=-1),
46 'year': relativedelta(years=-1),
47}
48
49safesearch = True
50safesearch_table = {0: 'both', 1: 'false', 2: 'false'}
51
52
53def request(query, params):
54 """Assemble request for the Peertube API"""
55
56 if not query:
57 return False
58
59 # eng_region = traits.get_region(params['searxng_locale'], 'en_US')
60 eng_lang = traits.get_language(params['searxng_locale'], None)
61
62 params['url'] = (
63 base_url.rstrip("/")
64 + "/api/v1/search/videos?"
65 + urlencode(
66 {
67 'search': query,
68 'searchTarget': 'search-index', # Vidiversum
69 'resultType': 'videos',
70 'start': (params['pageno'] - 1) * 10,
71 'count': 10,
72 # -createdAt: sort by date ascending / createdAt: date descending
73 'sort': '-match', # sort by *match descending*
74 'nsfw': safesearch_table[params['safesearch']],
75 }
76 )
77 )
78
79 if eng_lang is not None:
80 params['url'] += '&languageOneOf[]=' + eng_lang
81 params['url'] += '&boostLanguages[]=' + eng_lang
82
83 if params['time_range'] in time_range_table:
84 time = datetime.now().date() + time_range_table[params['time_range']]
85 params['url'] += '&startDate=' + time.isoformat()
86
87 return params
88
89
90def response(resp):
91 return video_response(resp)
92
93
95 """Parse video response from SepiaSearch and Peertube instances."""
96 results = []
97
98 json_data = resp.json()
99
100 if 'data' not in json_data:
101 return []
102
103 for result in json_data['data']:
104 metadata = [
105 x
106 for x in [
107 result.get('channel', {}).get('displayName'),
108 result.get('channel', {}).get('name') + '@' + result.get('channel', {}).get('host'),
109 ', '.join(result.get('tags', [])),
110 ]
111 if x
112 ]
113
114 duration = result.get('duration')
115 if duration:
116 duration = timedelta(seconds=duration)
117
118 results.append(
119 {
120 'url': result['url'],
121 'title': result['name'],
122 'content': html_to_text(result.get('description') or ''),
123 'author': result.get('account', {}).get('displayName'),
124 'length': duration,
125 'views': humanize_number(result['views']),
126 'template': 'videos.html',
127 'publishedDate': parse(result['publishedAt']),
128 'iframe_src': result.get('embedUrl'),
129 'thumbnail': result.get('thumbnailUrl') or result.get('previewUrl'),
130 'metadata': ' | '.join(metadata),
131 }
132 )
133
134 return results
135
136
137def fetch_traits(engine_traits: EngineTraits):
138 """Fetch languages from peertube's search-index source code.
139
140 See videoLanguages_ in commit `8ed5c729 - Refactor and redesign client`_
141
142 .. _8ed5c729 - Refactor and redesign client:
143 https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729
144 .. _videoLanguages:
145 https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729#3d8747f9a60695c367c70bb64efba8f403721fad_0_291
146 """
147
148 resp = get(
149 'https://framagit.org/framasoft/peertube/search-index/-/raw/master/client/src/components/Filters.vue',
150 # the response from search-index repository is very slow
151 timeout=60,
152 )
153
154 if not resp.ok: # type: ignore
155 print("ERROR: response from peertube is not OK.")
156 return
157
158 js_lang = re.search(r"videoLanguages \‍(\‍)[^\n]+(.*?)\]", resp.text, re.DOTALL) # type: ignore
159 if not js_lang:
160 print("ERROR: can't determine languages from peertube")
161 return
162
163 for lang in re.finditer(r"\{ id: '([a-z]+)', label:", js_lang.group(1)):
164 eng_tag = lang.group(1)
165 if eng_tag == 'oc':
166 # Occitanis not known by babel, its closest relative is Catalan
167 # but 'ca' is already in the list of engine_traits.languages -->
168 # 'oc' will be ignored.
169 continue
170 try:
171 sxng_tag = language_tag(babel.Locale.parse(eng_tag))
172 except babel.UnknownLocaleError:
173 print("ERROR: %s is unknown by babel" % eng_tag)
174 continue
175
176 conflict = engine_traits.languages.get(sxng_tag)
177 if conflict:
178 if conflict != eng_tag:
179 print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
180 continue
181 engine_traits.languages[sxng_tag] = eng_tag
182
183 engine_traits.languages['zh_Hans'] = 'zh'
184 engine_traits.languages['zh_Hant'] = 'zh'
request(query, params)
Definition peertube.py:53
fetch_traits(EngineTraits engine_traits)
Definition peertube.py:137