198 lexer = get_lexer_by_name(language, stripall=
True)
200 except Exception
as e:
201 logger.warning(
"pygments lexer: %s " % e)
203 lexer = get_lexer_by_name(
'text', stripall=
True)
208 line_code_start =
None
211 for line, code
in codelines:
213 line_code_start = line
216 if last_line
is not None and last_line + 1 != line:
219 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
220 html_code = html_code + highlight(tmp_code, lexer, formatter)
224 line_code_start = line
227 tmp_code += code +
'\n'
233 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
234 html_code = html_code + highlight(tmp_code, lexer, formatter)
372 kwargs[
'client_settings'] = str(
375 json.dumps(client_settings),
381 kwargs[
'preferences'] = sxng_request.preferences
382 kwargs.update(client_settings)
385 kwargs[
'endpoint'] =
'results' if 'q' in kwargs
else sxng_request.endpoint
386 kwargs[
'cookies'] = sxng_request.cookies
387 kwargs[
'errors'] = sxng_request.errors
388 kwargs[
'link_token'] = link_token.get_token()
390 kwargs[
'categories_as_tabs'] = list(settings[
'categories_as_tabs'].keys())
392 kwargs[
'DEFAULT_CATEGORY'] = DEFAULT_CATEGORY
395 kwargs[
'sxng_locales'] = [l
for l
in sxng_locales
if l[0]
in settings[
'search'][
'languages']]
397 locale = sxng_request.preferences.get_value(
'locale')
400 if locale
in RTL_LOCALES
and 'rtl' not in kwargs:
403 if 'current_language' not in kwargs:
404 kwargs[
'current_language'] = parse_lang(sxng_request.preferences, {},
RawTextQuery(
'', []))
407 kwargs[
'search_formats'] = [x
for x
in settings[
'search'][
'formats']
if x !=
'html']
408 kwargs[
'instance_name'] =
get_setting(
'general.instance_name')
409 kwargs[
'searx_version'] = VERSION_STRING
410 kwargs[
'searx_git_url'] = GIT_URL
411 kwargs[
'enable_metrics'] =
get_setting(
'general.enable_metrics')
412 kwargs[
'get_setting'] = get_setting
413 kwargs[
'get_pretty_url'] = get_pretty_url
417 if donation_url
is True:
419 kwargs[
'donation_url'] = donation_url
422 kwargs[
'url_for'] = custom_url_for
423 kwargs[
'image_proxify'] = image_proxify
424 kwargs[
'favicon_url'] = favicons.favicon_url
425 kwargs[
'proxify'] = morty_proxify
if settings[
'result_proxy'][
'url']
is not None else None
426 kwargs[
'proxify_results'] = settings[
'result_proxy'][
'proxify_results']
427 kwargs[
'cache_url'] = settings[
'ui'][
'cache_url']
428 kwargs[
'get_result_template'] = get_result_template
429 kwargs[
'opensearch_url'] = (
430 url_for(
'opensearch')
434 'method': sxng_request.preferences.get_value(
'method'),
435 'autocomplete': sxng_request.preferences.get_value(
'autocomplete'),
439 kwargs[
'urlparse'] = urlparse
441 start_time = default_timer()
442 result = render_template(
'{}/{}'.format(kwargs[
'theme'], template_name), **kwargs)
443 sxng_request.render_time += default_timer() - start_time
450 sxng_request.start_time = default_timer()
451 sxng_request.render_time = 0
452 sxng_request.timings = []
453 sxng_request.errors = []
455 client_pref = ClientPref.from_http_request(sxng_request)
457 preferences =
Preferences(themes, list(categories.keys()), engines, searx.plugins.STORAGE, client_pref)
459 user_agent = sxng_request.headers.get(
'User-Agent',
'').lower()
460 if 'webkit' in user_agent
and 'android' in user_agent:
461 preferences.key_value_settings[
'method'].value =
'GET'
462 sxng_request.preferences = preferences
465 preferences.parse_dict(sxng_request.cookies)
467 except Exception
as e:
468 logger.exception(e, exc_info=
True)
469 sxng_request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
473 sxng_request.form = dict(sxng_request.form.items())
474 for k, v
in sxng_request.args.items():
475 if k
not in sxng_request.form:
476 sxng_request.form[k] = v
478 if sxng_request.form.get(
'preferences'):
479 preferences.parse_encoded_data(sxng_request.form[
'preferences'])
482 preferences.parse_dict(sxng_request.form)
483 except Exception
as e:
484 logger.exception(e, exc_info=
True)
485 sxng_request.errors.append(gettext(
'Invalid settings'))
489 if not preferences.get_value(
"language"):
491 preferences.parse_dict({
"language": language})
492 logger.debug(
'set language %s (from browser)', preferences.get_value(
"language"))
496 if not preferences.get_value(
"locale"):
498 preferences.parse_dict({
"locale": locale})
499 logger.debug(
'set locale %s (from browser)', preferences.get_value(
"locale"))
502 sxng_request.user_plugins = []
503 allowed_plugins = preferences.plugins.get_enabled()
504 disabled_plugins = preferences.plugins.get_disabled()
505 for plugin
in searx.plugins.STORAGE:
506 if (plugin.id
not in disabled_plugins)
or plugin.id
in allowed_plugins:
507 sxng_request.user_plugins.append(plugin.id)
522 total_time = default_timer() - sxng_request.start_time
524 'total;dur=' + str(round(total_time * 1000, 3)),
525 'render;dur=' + str(round(sxng_request.render_time * 1000, 3)),
527 if len(sxng_request.timings) > 0:
528 timings = sorted(sxng_request.timings, key=
lambda t: t.total)
530 'total_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.total * 1000, 3))
for i, t
in enumerate(timings)
533 'load_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.load * 1000, 3))
534 for i, t
in enumerate(timings)
537 timings_all = timings_all + timings_total + timings_load
538 response.headers.add(
'Server-Timing',
', '.join(timings_all))
610 """Search query in q and return results.
612 Supported outputs: html, json, csv, rss.
618 output_format = sxng_request.form.get(
'format',
'html')
619 if output_format
not in OUTPUT_FORMATS:
620 output_format =
'html'
622 if output_format
not in settings[
'search'][
'formats']:
626 if not sxng_request.form.get(
'q'):
627 if output_format ==
'html':
631 selected_categories=get_selected_categories(sxng_request.preferences, sxng_request.form),
638 raw_text_query =
None
639 result_container =
None
641 search_query, raw_text_query, _, _, selected_locale = get_search_query_from_webapp(
642 sxng_request.preferences, sxng_request.form
645 result_container = search_obj.search()
647 except SearxParameterException
as e:
648 logger.exception(
'search error: SearxParameterException')
650 except Exception
as e:
651 logger.exception(e, exc_info=
True)
652 return index_error(output_format, gettext(
'search error')), 500
655 if result_container.redirect_url:
656 return redirect(result_container.redirect_url)
660 sxng_request.timings = result_container.get_timings()
664 if output_format ==
'json':
666 response = webutils.get_json_response(search_query, result_container)
667 return Response(response, mimetype=
'application/json')
669 if output_format ==
'csv':
672 webutils.write_csv_response(csv, result_container)
675 response = Response(csv.stream.read(), mimetype=
'application/csv')
676 cont_disp =
'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
677 response.headers.add(
'Content-Disposition', cont_disp)
682 current_template =
None
683 previous_result =
None
685 results = result_container.get_ordered_results()
687 if search_query.redirect_to_first_result
and results:
688 return redirect(results[0][
'url'], 302)
690 for result
in results:
691 if output_format ==
'html':
692 if 'content' in result
and result[
'content']:
693 result[
'content'] = highlight_content(escape(result[
'content'][:1024]), search_query.query)
694 if 'title' in result
and result[
'title']:
695 result[
'title'] = highlight_content(escape(result[
'title']
or ''), search_query.query)
699 if current_template != result.template:
700 result.open_group =
True
702 previous_result.close_group =
True
703 current_template = result.template
704 previous_result = result
707 previous_result.close_group =
True
711 if output_format ==
'rss':
713 'opensearch_response_rss.xml',
715 q=sxng_request.form[
'q'],
716 number_of_results=result_container.number_of_results,
718 return Response(response_rss, mimetype=
'text/xml')
723 suggestion_urls = list(
725 lambda suggestion: {
'url': raw_text_query.changeQuery(suggestion).getFullQuery(),
'title': suggestion},
726 result_container.suggestions,
730 correction_urls = list(
732 lambda correction: {
'url': raw_text_query.changeQuery(correction).getFullQuery(),
'title': correction},
733 result_container.corrections,
738 engine_timings = sorted(result_container.get_timings(), reverse=
True, key=
lambda e: e.total)
739 max_response_time = engine_timings[0].total
if engine_timings
else None
740 engine_timings_pairs = [(timing.engine, timing.total)
for timing
in engine_timings]
749 q=sxng_request.form[
'q'],
750 selected_categories = search_query.categories,
751 pageno = search_query.pageno,
752 time_range = search_query.time_range
or '',
753 number_of_results = format_decimal(result_container.number_of_results),
754 suggestions = suggestion_urls,
755 answers = result_container.answers,
756 corrections = correction_urls,
757 infoboxes = result_container.infoboxes,
758 engine_data = result_container.engine_data,
759 paging = result_container.paging,
760 unresponsive_engines = webutils.get_translated_errors(
761 result_container.unresponsive_engines
763 current_locale = sxng_request.preferences.get_value(
"locale"),
764 current_language = selected_locale,
765 search_language = match_locale(
766 search_obj.search_query.lang,
767 settings[
'search'][
'languages'],
768 fallback=sxng_request.preferences.get_value(
"language")
770 timeout_limit = sxng_request.form.get(
'timeout_limit',
None),
771 timings = engine_timings_pairs,
772 max_response_time = max_response_time
777@app.route('/about', methods=['GET'])
802 """Return autocompleter results"""
808 disabled_engines = sxng_request.preferences.engines.get_disabled()
811 raw_text_query =
RawTextQuery(sxng_request.form.get(
'q',
''), disabled_engines)
812 sug_prefix = raw_text_query.getQuery()
814 for obj
in searx.answerers.STORAGE.ask(sug_prefix):
815 if isinstance(obj, Answer):
816 results.append(obj.answer)
820 if len(raw_text_query.autocomplete_list) == 0
and len(sug_prefix) > 0:
823 sxng_locale = sxng_request.preferences.get_value(
'language')
824 backend_name = sxng_request.preferences.get_value(
'autocomplete')
826 for result
in search_autocomplete(backend_name, sug_prefix, sxng_locale):
829 if result != sug_prefix:
830 results.append(raw_text_query.changeQuery(result).getFullQuery())
832 if len(raw_text_query.autocomplete_list) > 0:
833 for autocomplete_text
in raw_text_query.autocomplete_list:
834 results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
836 if sxng_request.headers.get(
'X-Requested-With') ==
'XMLHttpRequest':
838 suggestions = json.dumps(results)
839 mimetype =
'application/json'
842 suggestions = json.dumps([sug_prefix, results])
843 mimetype =
'application/x-suggestions+json'
845 suggestions = escape(suggestions,
False)
846 return Response(suggestions, mimetype=mimetype)
849@app.route('/preferences', methods=['GET', 'POST'])
851 """Render preferences page && save user preferences"""
857 if sxng_request.args.get(
'preferences')
or sxng_request.form.get(
'preferences'):
858 resp = make_response(redirect(url_for(
'index', _external=
True)))
859 return sxng_request.preferences.save(resp)
862 if sxng_request.method ==
'POST':
863 resp = make_response(redirect(url_for(
'index', _external=
True)))
865 sxng_request.preferences.parse_form(sxng_request.form)
866 except ValidationException:
867 sxng_request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
869 return sxng_request.preferences.save(resp)
872 image_proxy = sxng_request.preferences.get_value(
'image_proxy')
873 disabled_engines = sxng_request.preferences.engines.get_disabled()
874 allowed_plugins = sxng_request.preferences.plugins.get_enabled()
877 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
879 engines_by_category = {}
882 engines_by_category[c] = [e
for e
in categories[c]
if e.name
in filtered_engines]
884 list.sort(engines_by_category[c], key=
lambda e: e.name)
890 for _, e
in filtered_engines.items():
891 h = histogram(
'engine', e.name,
'time',
'total')
892 median = round(h.percentage(50), 1)
if h.count > 0
else None
893 rate80 = round(h.percentage(80), 1)
if h.count > 0
else None
894 rate95 = round(h.percentage(95), 1)
if h.count > 0
else None
896 max_rate95 = max(max_rate95, rate95
or 0)
898 result_count_sum = histogram(
'engine', e.name,
'result',
'count').sum
899 successful_count = counter(
'engine', e.name,
'search',
'count',
'successful')
900 result_count = int(result_count_sum / float(successful_count))
if successful_count
else 0
906 'warn_timeout': e.timeout > settings[
'outgoing'][
'request_timeout'],
907 'supports_selected_language': e.traits.is_locale_supported(
908 str(sxng_request.preferences.get_value(
'language')
or 'all')
910 'result_count': result_count,
916 engine_errors = get_engine_errors(filtered_engines)
917 checker_results = checker_get_result()
919 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
921 for _, e
in filtered_engines.items():
922 checker_result = checker_results.get(e.name, {})
923 checker_success = checker_result.get(
'success',
True)
924 errors = engine_errors.get(e.name)
or []
925 if counter(
'engine', e.name,
'search',
'count',
'sent') == 0:
928 elif checker_success
and not errors:
930 elif 'simple' in checker_result.get(
'errors', {}):
936 reliability = 100 - sum([error[
'percentage']
for error
in errors
if not error.get(
'secondary')])
938 reliabilities[e.name] = {
939 'reliability': reliability,
941 'checker': checker_results.get(e.name, {}).get(
'errors', {}).keys(),
945 reliabilities_errors = []
947 error_user_text =
None
948 if error.get(
'secondary')
or 'exception_classname' not in error:
950 error_user_text = exception_classname_to_text.get(error.get(
'exception_classname'))
952 error_user_text = exception_classname_to_text[
None]
953 if error_user_text
not in reliabilities_errors:
954 reliabilities_errors.append(error_user_text)
955 reliabilities[e.name][
'errors'] = reliabilities_errors
959 for _, e
in filtered_engines.items():
960 supports_selected_language = e.traits.is_locale_supported(
961 str(sxng_request.preferences.get_value(
'language')
or 'all')
963 safesearch = e.safesearch
964 time_range_support = e.time_range_support
965 for checker_test_name
in checker_results.get(e.name, {}).get(
'errors', {}):
966 if supports_selected_language
and checker_test_name.startswith(
'lang_'):
967 supports_selected_language =
'?'
968 elif safesearch
and checker_test_name ==
'safesearch':
970 elif time_range_support
and checker_test_name ==
'time_range':
971 time_range_support =
'?'
973 'supports_selected_language': supports_selected_language,
974 'safesearch': safesearch,
975 'time_range_support': time_range_support,
982 selected_categories = get_selected_categories(sxng_request.preferences, sxng_request.form),
983 locales = LOCALE_NAMES,
984 current_locale = sxng_request.preferences.get_value(
"locale"),
985 image_proxy = image_proxy,
986 engines_by_category = engines_by_category,
988 max_rate95 = max_rate95,
989 reliabilities = reliabilities,
991 answer_storage = searx.answerers.STORAGE.info,
992 disabled_engines = disabled_engines,
993 autocomplete_backends = autocomplete_backends,
994 favicon_resolver_names = favicons.proxy.CFG.resolver_map.keys(),
995 shortcuts = {y: x
for x, y
in engine_shortcuts.items()},
997 plugins_storage = searx.plugins.STORAGE.info,
998 current_doi_resolver = get_doi_resolver(),
999 allowed_plugins = allowed_plugins,
1000 preferences_url_params = sxng_request.preferences.get_as_url_params(),
1001 locked_preferences =
get_setting(
"preferences.lock", []),
1014 url = sxng_request.args.get(
'url')
1018 if not is_hmac_of(settings[
'server'][
'secret_key'], url.encode(), sxng_request.args.get(
'h',
'')):
1021 maximum_size = 5 * 1024 * 1024
1022 forward_resp =
False
1026 'User-Agent': gen_useragent(),
1027 'Accept':
'image/webp,*/*',
1028 'Accept-Encoding':
'gzip, deflate',
1032 set_context_network_name(
'image_proxy')
1033 resp, stream = http_stream(method=
'GET', url=url, headers=request_headers, allow_redirects=
True)
1034 content_length = resp.headers.get(
'Content-Length')
1035 if content_length
and content_length.isdigit()
and int(content_length) > maximum_size:
1036 return 'Max size', 400
1038 if resp.status_code != 200:
1039 logger.debug(
'image-proxy: wrong response code: %i', resp.status_code)
1040 if resp.status_code >= 400:
1041 return '', resp.status_code
1044 if not resp.headers.get(
'Content-Type',
'').startswith(
'image/')
and not resp.headers.get(
1046 ).startswith(
'binary/octet-stream'):
1047 logger.debug(
'image-proxy: wrong content-type: %s', resp.headers.get(
'Content-Type',
''))
1051 except httpx.HTTPError:
1052 logger.exception(
'HTTP error')
1055 if resp
and not forward_resp:
1060 except httpx.HTTPError:
1061 logger.exception(
'HTTP error on closing')
1064 nonlocal resp, stream
1070 except httpx.HTTPError
as e:
1071 logger.debug(
'Exception while closing response', e)
1074 headers = dict_subset(resp.headers, {
'Content-Type',
'Content-Encoding',
'Content-Length',
'Length'})
1075 response = Response(stream, mimetype=resp.headers[
'Content-Type'], headers=headers, direct_passthrough=
True)
1076 response.call_on_close(close_stream)
1078 except httpx.HTTPError:
1083@app.route('/engine_descriptions.json', methods=['GET'])
1086 result = ENGINE_DESCRIPTIONS[
'en'].copy()
1088 for engine, description
in ENGINE_DESCRIPTIONS.get(locale, {}).items():
1089 result[engine] = description
1090 for engine, description
in result.items():
1091 if len(description) == 2
and description[1] ==
'ref':
1092 ref_engine, ref_lang = description[0].split(
':')
1093 description = ENGINE_DESCRIPTIONS[ref_lang][ref_engine]
1094 if isinstance(description, str):
1095 description = [description,
'wikipedia']
1096 result[engine] = description
1099 for engine_name, engine_mod
in engines.items():
1100 descr = getattr(engine_mod,
'about', {}).get(
'description',
None)
1101 if descr
is not None:
1102 result[engine_name] = [descr,
"SearXNG config"]
1104 return jsonify(result)
1107@app.route('/stats', methods=['GET'])
1109 """Render engine statistics page."""
1110 sort_order = sxng_request.args.get(
'sort', default=
'name', type=str)
1111 selected_engine_name = sxng_request.args.get(
'engine', default=
None, type=str)
1113 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
1114 if selected_engine_name:
1115 if selected_engine_name
not in filtered_engines:
1116 selected_engine_name =
None
1118 filtered_engines = [selected_engine_name]
1120 checker_results = checker_get_result()
1122 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
1125 engine_stats = get_engines_stats(filtered_engines)
1126 engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
1128 if sort_order
not in STATS_SORT_PARAMETERS:
1131 reverse, key_name, default_value = STATS_SORT_PARAMETERS[sort_order]
1133 def get_key(engine_stat):
1134 reliability = engine_reliabilities.get(engine_stat[
'name'], {}).get(
'reliability', 0)
1135 reliability_order = 0
if reliability
else 1
1136 if key_name ==
'reliability':
1138 reliability_order = 0
1140 key = engine_stat.get(key_name)
or default_value
1142 reliability_order = 1 - reliability_order
1143 return (reliability_order, key, engine_stat[
'name'])
1145 technical_report = []
1146 for error
in engine_reliabilities.get(selected_engine_name, {}).get(
'errors', []):
1147 technical_report.append(
1149 Error: {error['exception_classname'] or error['log_message']} \
1150 Parameters: {error['log_parameters']} \
1151 File name: {error['filename'] }:{ error['line_no'] } \
1152 Error Function: {error['function']} \
1153 Code: {error['code']} \
1158 technical_report =
' '.join(technical_report)
1160 engine_stats[
'time'] = sorted(engine_stats[
'time'], reverse=reverse, key=get_key)
1164 sort_order = sort_order,
1165 engine_stats = engine_stats,
1166 engine_reliabilities = engine_reliabilities,
1167 selected_engine_name = selected_engine_name,
1168 searx_git_branch = GIT_BRANCH,
1169 technical_report = technical_report,
1174@app.route('/stats/errors', methods=['GET'])
1189 password = settings[
'general'].get(
"open_metrics")
1191 if not (settings[
'general'].get(
"enable_metrics")
and password):
1192 return Response(
'open metrics is disabled', status=404, mimetype=
'text/plain')
1194 if not sxng_request.authorization
or sxng_request.authorization.password != password:
1195 return Response(
'access forbidden', status=401, mimetype=
'text/plain')
1197 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
1199 checker_results = checker_get_result()
1201 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
1204 engine_stats = get_engines_stats(filtered_engines)
1205 engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
1206 metrics_text = openmetrics(engine_stats, engine_reliabilities)
1208 return Response(metrics_text, mimetype=
'text/plain')
1211@app.route('/robots.txt', methods=['GET'])