195 lexer = get_lexer_by_name(language, stripall=
True)
197 except Exception
as e:
198 logger.warning(
"pygments lexer: %s " % e)
200 lexer = get_lexer_by_name(
'text', stripall=
True)
205 line_code_start =
None
208 for line, code
in codelines:
210 line_code_start = line
213 if last_line
is not None and last_line + 1 != line:
216 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
217 html_code = html_code + highlight(tmp_code, lexer, formatter)
221 line_code_start = line
224 tmp_code += code +
'\n'
230 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
231 html_code = html_code + highlight(tmp_code, lexer, formatter)
374 kwargs[
'client_settings'] = str(
377 json.dumps(client_settings),
383 kwargs[
'preferences'] = sxng_request.preferences
384 kwargs.update(client_settings)
387 kwargs[
'endpoint'] =
'results' if 'q' in kwargs
else sxng_request.endpoint
388 kwargs[
'cookies'] = sxng_request.cookies
389 kwargs[
'errors'] = sxng_request.errors
390 kwargs[
'link_token'] = link_token.get_token()
392 kwargs[
'categories_as_tabs'] = list(settings[
'categories_as_tabs'].keys())
394 kwargs[
'DEFAULT_CATEGORY'] = DEFAULT_CATEGORY
397 kwargs[
'sxng_locales'] = [l
for l
in sxng_locales
if l[0]
in settings[
'search'][
'languages']]
399 locale = sxng_request.preferences.get_value(
'locale')
402 if locale
in RTL_LOCALES
and 'rtl' not in kwargs:
405 if 'current_language' not in kwargs:
406 kwargs[
'current_language'] = parse_lang(sxng_request.preferences, {},
RawTextQuery(
'', []))
409 kwargs[
'search_formats'] = [x
for x
in settings[
'search'][
'formats']
if x !=
'html']
410 kwargs[
'instance_name'] =
get_setting(
'general.instance_name')
411 kwargs[
'searx_version'] = VERSION_STRING
412 kwargs[
'searx_git_url'] = GIT_URL
413 kwargs[
'enable_metrics'] =
get_setting(
'general.enable_metrics')
414 kwargs[
'get_setting'] = get_setting
415 kwargs[
'get_pretty_url'] = get_pretty_url
419 if donation_url
is True:
421 kwargs[
'donation_url'] = donation_url
424 kwargs[
'url_for'] = custom_url_for
425 kwargs[
'image_proxify'] = image_proxify
426 kwargs[
'favicon_url'] = favicons.favicon_url
427 kwargs[
'proxify'] = morty_proxify
if settings[
'result_proxy'][
'url']
is not None else None
428 kwargs[
'proxify_results'] = settings[
'result_proxy'][
'proxify_results']
429 kwargs[
'cache_url'] = settings[
'ui'][
'cache_url']
430 kwargs[
'get_result_template'] = get_result_template
431 kwargs[
'opensearch_url'] = (
432 url_for(
'opensearch')
436 'method': sxng_request.preferences.get_value(
'method'),
437 'autocomplete': sxng_request.preferences.get_value(
'autocomplete'),
441 kwargs[
'urlparse'] = urlparse
443 start_time = default_timer()
444 result = render_template(
'{}/{}'.format(kwargs[
'theme'], template_name), **kwargs)
445 sxng_request.render_time += default_timer() - start_time
452 sxng_request.start_time = default_timer()
453 sxng_request.render_time = 0
454 sxng_request.timings = []
455 sxng_request.errors = []
457 client_pref = ClientPref.from_http_request(sxng_request)
459 preferences =
Preferences(themes, list(categories.keys()), engines, searx.plugins.STORAGE, client_pref)
461 user_agent = sxng_request.headers.get(
'User-Agent',
'').lower()
462 if 'webkit' in user_agent
and 'android' in user_agent:
463 preferences.key_value_settings[
'method'].value =
'GET'
464 sxng_request.preferences = preferences
467 preferences.parse_dict(sxng_request.cookies)
469 except Exception
as e:
470 logger.exception(e, exc_info=
True)
471 sxng_request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
475 sxng_request.form = dict(sxng_request.form.items())
476 for k, v
in sxng_request.args.items():
477 if k
not in sxng_request.form:
478 sxng_request.form[k] = v
480 if sxng_request.form.get(
'preferences'):
481 preferences.parse_encoded_data(sxng_request.form[
'preferences'])
484 preferences.parse_dict(sxng_request.form)
485 except Exception
as e:
486 logger.exception(e, exc_info=
True)
487 sxng_request.errors.append(gettext(
'Invalid settings'))
491 if not preferences.get_value(
"language"):
493 preferences.parse_dict({
"language": language})
494 logger.debug(
'set language %s (from browser)', preferences.get_value(
"language"))
498 if not preferences.get_value(
"locale"):
500 preferences.parse_dict({
"locale": locale})
501 logger.debug(
'set locale %s (from browser)', preferences.get_value(
"locale"))
504 sxng_request.user_plugins = []
505 allowed_plugins = preferences.plugins.get_enabled()
506 disabled_plugins = preferences.plugins.get_disabled()
507 for plugin
in searx.plugins.STORAGE:
508 if (plugin.id
not in disabled_plugins)
or plugin.id
in allowed_plugins:
509 sxng_request.user_plugins.append(plugin.id)
524 total_time = default_timer() - sxng_request.start_time
526 'total;dur=' + str(round(total_time * 1000, 3)),
527 'render;dur=' + str(round(sxng_request.render_time * 1000, 3)),
529 if len(sxng_request.timings) > 0:
530 timings = sorted(sxng_request.timings, key=
lambda t: t.total)
532 'total_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.total * 1000, 3))
for i, t
in enumerate(timings)
535 'load_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.load * 1000, 3))
536 for i, t
in enumerate(timings)
539 timings_all = timings_all + timings_total + timings_load
540 response.headers.add(
'Server-Timing',
', '.join(timings_all))
612 """Search query in q and return results.
614 Supported outputs: html, json, csv, rss.
620 output_format = sxng_request.form.get(
'format',
'html')
621 if output_format
not in OUTPUT_FORMATS:
622 output_format =
'html'
624 if output_format
not in settings[
'search'][
'formats']:
628 if not sxng_request.form.get(
'q'):
629 if output_format ==
'html':
633 selected_categories=get_selected_categories(sxng_request.preferences, sxng_request.form),
640 raw_text_query =
None
641 result_container =
None
643 search_query, raw_text_query, _, _, selected_locale = get_search_query_from_webapp(
644 sxng_request.preferences, sxng_request.form
647 result_container = search_obj.search()
649 except SearxParameterException
as e:
650 logger.exception(
'search error: SearxParameterException')
652 except Exception
as e:
653 logger.exception(e, exc_info=
True)
654 return index_error(output_format, gettext(
'search error')), 500
657 if result_container.redirect_url:
658 return redirect(result_container.redirect_url)
662 sxng_request.timings = result_container.get_timings()
666 if output_format ==
'json':
668 response = webutils.get_json_response(search_query, result_container)
669 return Response(response, mimetype=
'application/json')
671 if output_format ==
'csv':
674 webutils.write_csv_response(csv, result_container)
677 response = Response(csv.stream.read(), mimetype=
'application/csv')
678 cont_disp =
'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
679 response.headers.add(
'Content-Disposition', cont_disp)
684 current_template =
None
685 previous_result =
None
687 results = result_container.get_ordered_results()
689 if search_query.redirect_to_first_result
and results:
690 return redirect(results[0][
'url'], 302)
692 for result
in results:
693 if output_format ==
'html':
694 if 'content' in result
and result[
'content']:
695 result[
'content'] = highlight_content(escape(result[
'content'][:1024]), search_query.query)
696 if 'title' in result
and result[
'title']:
697 result[
'title'] = highlight_content(escape(result[
'title']
or ''), search_query.query)
701 if current_template != result.template:
702 result.open_group =
True
704 previous_result.close_group =
True
705 current_template = result.template
706 previous_result = result
709 previous_result.close_group =
True
713 if output_format ==
'rss':
715 'opensearch_response_rss.xml',
717 q=sxng_request.form[
'q'],
718 number_of_results=result_container.number_of_results,
720 return Response(response_rss, mimetype=
'text/xml')
725 suggestion_urls = list(
727 lambda suggestion: {
'url': raw_text_query.changeQuery(suggestion).getFullQuery(),
'title': suggestion},
728 result_container.suggestions,
732 correction_urls = list(
734 lambda correction: {
'url': raw_text_query.changeQuery(correction).getFullQuery(),
'title': correction},
735 result_container.corrections,
740 engine_timings = sorted(result_container.get_timings(), reverse=
True, key=
lambda e: e.total)
741 max_response_time = engine_timings[0].total
if engine_timings
else None
742 engine_timings_pairs = [(timing.engine, timing.total)
for timing
in engine_timings]
751 q=sxng_request.form[
'q'],
752 selected_categories = search_query.categories,
753 pageno = search_query.pageno,
754 time_range = search_query.time_range
or '',
755 number_of_results = format_decimal(result_container.number_of_results),
756 suggestions = suggestion_urls,
757 answers = result_container.answers,
758 corrections = correction_urls,
759 infoboxes = result_container.infoboxes,
760 engine_data = result_container.engine_data,
761 paging = result_container.paging,
762 unresponsive_engines = webutils.get_translated_errors(
763 result_container.unresponsive_engines
765 current_locale = sxng_request.preferences.get_value(
"locale"),
766 current_language = selected_locale,
767 search_language = match_locale(
768 search_obj.search_query.lang,
769 settings[
'search'][
'languages'],
770 fallback=sxng_request.preferences.get_value(
"language")
772 timeout_limit = sxng_request.form.get(
'timeout_limit',
None),
773 timings = engine_timings_pairs,
774 max_response_time = max_response_time
779@app.route('/about', methods=['GET'])
804 """Return autocompleter results"""
810 disabled_engines = sxng_request.preferences.engines.get_disabled()
813 raw_text_query =
RawTextQuery(sxng_request.form.get(
'q',
''), disabled_engines)
814 sug_prefix = raw_text_query.getQuery()
816 for obj
in searx.answerers.STORAGE.ask(sug_prefix):
817 if isinstance(obj, Answer):
818 results.append(obj.answer)
822 if len(raw_text_query.autocomplete_list) == 0
and len(sug_prefix) > 0:
825 sxng_locale = sxng_request.preferences.get_value(
'language')
826 backend_name = sxng_request.preferences.get_value(
'autocomplete')
828 for result
in search_autocomplete(backend_name, sug_prefix, sxng_locale):
831 if result != sug_prefix:
832 results.append(raw_text_query.changeQuery(result).getFullQuery())
834 if len(raw_text_query.autocomplete_list) > 0:
835 for autocomplete_text
in raw_text_query.autocomplete_list:
836 results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
838 if sxng_request.headers.get(
'X-Requested-With') ==
'XMLHttpRequest':
840 suggestions = json.dumps(results)
841 mimetype =
'application/json'
844 suggestions = json.dumps([sug_prefix, results])
845 mimetype =
'application/x-suggestions+json'
847 suggestions = escape(suggestions,
False)
848 return Response(suggestions, mimetype=mimetype)
851@app.route('/preferences', methods=['GET', 'POST'])
853 """Render preferences page && save user preferences"""
859 if sxng_request.args.get(
'preferences')
or sxng_request.form.get(
'preferences'):
860 resp = make_response(redirect(url_for(
'index', _external=
True)))
861 return sxng_request.preferences.save(resp)
864 if sxng_request.method ==
'POST':
865 resp = make_response(redirect(url_for(
'index', _external=
True)))
867 sxng_request.preferences.parse_form(sxng_request.form)
868 except ValidationException:
869 sxng_request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
871 return sxng_request.preferences.save(resp)
874 image_proxy = sxng_request.preferences.get_value(
'image_proxy')
875 disabled_engines = sxng_request.preferences.engines.get_disabled()
876 allowed_plugins = sxng_request.preferences.plugins.get_enabled()
879 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
881 engines_by_category = {}
884 engines_by_category[c] = [e
for e
in categories[c]
if e.name
in filtered_engines]
886 list.sort(engines_by_category[c], key=
lambda e: e.name)
892 for _, e
in filtered_engines.items():
893 h = histogram(
'engine', e.name,
'time',
'total')
894 median = round(h.percentage(50), 1)
if h.count > 0
else None
895 rate80 = round(h.percentage(80), 1)
if h.count > 0
else None
896 rate95 = round(h.percentage(95), 1)
if h.count > 0
else None
898 max_rate95 = max(max_rate95, rate95
or 0)
900 result_count_sum = histogram(
'engine', e.name,
'result',
'count').sum
901 successful_count = counter(
'engine', e.name,
'search',
'count',
'successful')
902 result_count = int(result_count_sum / float(successful_count))
if successful_count
else 0
908 'warn_timeout': e.timeout > settings[
'outgoing'][
'request_timeout'],
909 'supports_selected_language': e.traits.is_locale_supported(
910 str(sxng_request.preferences.get_value(
'language')
or 'all')
912 'result_count': result_count,
918 engine_errors = get_engine_errors(filtered_engines)
919 checker_results = checker_get_result()
921 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
923 for _, e
in filtered_engines.items():
924 checker_result = checker_results.get(e.name, {})
925 checker_success = checker_result.get(
'success',
True)
926 errors = engine_errors.get(e.name)
or []
927 if counter(
'engine', e.name,
'search',
'count',
'sent') == 0:
930 elif checker_success
and not errors:
932 elif 'simple' in checker_result.get(
'errors', {}):
938 reliability = 100 - sum([error[
'percentage']
for error
in errors
if not error.get(
'secondary')])
940 reliabilities[e.name] = {
941 'reliability': reliability,
943 'checker': checker_results.get(e.name, {}).get(
'errors', {}).keys(),
947 reliabilities_errors = []
949 error_user_text =
None
950 if error.get(
'secondary')
or 'exception_classname' not in error:
952 error_user_text = exception_classname_to_text.get(error.get(
'exception_classname'))
954 error_user_text = exception_classname_to_text[
None]
955 if error_user_text
not in reliabilities_errors:
956 reliabilities_errors.append(error_user_text)
957 reliabilities[e.name][
'errors'] = reliabilities_errors
961 for _, e
in filtered_engines.items():
962 supports_selected_language = e.traits.is_locale_supported(
963 str(sxng_request.preferences.get_value(
'language')
or 'all')
965 safesearch = e.safesearch
966 time_range_support = e.time_range_support
967 for checker_test_name
in checker_results.get(e.name, {}).get(
'errors', {}):
968 if supports_selected_language
and checker_test_name.startswith(
'lang_'):
969 supports_selected_language =
'?'
970 elif safesearch
and checker_test_name ==
'safesearch':
972 elif time_range_support
and checker_test_name ==
'time_range':
973 time_range_support =
'?'
975 'supports_selected_language': supports_selected_language,
976 'safesearch': safesearch,
977 'time_range_support': time_range_support,
984 selected_categories = get_selected_categories(sxng_request.preferences, sxng_request.form),
985 locales = LOCALE_NAMES,
986 current_locale = sxng_request.preferences.get_value(
"locale"),
987 image_proxy = image_proxy,
988 engines_by_category = engines_by_category,
990 max_rate95 = max_rate95,
991 reliabilities = reliabilities,
993 answer_storage = searx.answerers.STORAGE.info,
994 disabled_engines = disabled_engines,
995 autocomplete_backends = autocomplete_backends,
996 favicon_resolver_names = favicons.proxy.CFG.resolver_map.keys(),
997 shortcuts = {y: x
for x, y
in engine_shortcuts.items()},
999 plugins_storage = searx.plugins.STORAGE.info,
1000 current_doi_resolver = get_doi_resolver(),
1001 allowed_plugins = allowed_plugins,
1002 preferences_url_params = sxng_request.preferences.get_as_url_params(),
1003 locked_preferences =
get_setting(
"preferences.lock", []),
1016 url = sxng_request.args.get(
'url')
1020 if not is_hmac_of(settings[
'server'][
'secret_key'], url.encode(), sxng_request.args.get(
'h',
'')):
1023 maximum_size = 5 * 1024 * 1024
1024 forward_resp =
False
1028 'User-Agent': gen_useragent(),
1029 'Accept':
'image/webp,*/*',
1030 'Accept-Encoding':
'gzip, deflate',
1034 set_context_network_name(
'image_proxy')
1035 resp, stream = http_stream(method=
'GET', url=url, headers=request_headers, allow_redirects=
True)
1036 content_length = resp.headers.get(
'Content-Length')
1037 if content_length
and content_length.isdigit()
and int(content_length) > maximum_size:
1038 return 'Max size', 400
1040 if resp.status_code != 200:
1041 logger.debug(
'image-proxy: wrong response code: %i', resp.status_code)
1042 if resp.status_code >= 400:
1043 return '', resp.status_code
1046 if not resp.headers.get(
'Content-Type',
'').startswith(
'image/')
and not resp.headers.get(
1048 ).startswith(
'binary/octet-stream'):
1049 logger.debug(
'image-proxy: wrong content-type: %s', resp.headers.get(
'Content-Type',
''))
1053 except httpx.HTTPError:
1054 logger.exception(
'HTTP error')
1057 if resp
and not forward_resp:
1062 except httpx.HTTPError:
1063 logger.exception(
'HTTP error on closing')
1066 nonlocal resp, stream
1072 except httpx.HTTPError
as e:
1073 logger.debug(
'Exception while closing response', e)
1076 headers = dict_subset(resp.headers, {
'Content-Type',
'Content-Encoding',
'Content-Length',
'Length'})
1077 response = Response(stream, mimetype=resp.headers[
'Content-Type'], headers=headers, direct_passthrough=
True)
1078 response.call_on_close(close_stream)
1080 except httpx.HTTPError:
1085@app.route('/engine_descriptions.json', methods=['GET'])
1088 result = ENGINE_DESCRIPTIONS[
'en'].copy()
1090 for engine, description
in ENGINE_DESCRIPTIONS.get(locale, {}).items():
1091 result[engine] = description
1092 for engine, description
in result.items():
1093 if len(description) == 2
and description[1] ==
'ref':
1094 ref_engine, ref_lang = description[0].split(
':')
1095 description = ENGINE_DESCRIPTIONS[ref_lang][ref_engine]
1096 if isinstance(description, str):
1097 description = [description,
'wikipedia']
1098 result[engine] = description
1101 for engine_name, engine_mod
in engines.items():
1102 descr = getattr(engine_mod,
'about', {}).get(
'description',
None)
1103 if descr
is not None:
1104 result[engine_name] = [descr,
"SearXNG config"]
1106 return jsonify(result)
1109@app.route('/stats', methods=['GET'])
1111 """Render engine statistics page."""
1112 sort_order = sxng_request.args.get(
'sort', default=
'name', type=str)
1113 selected_engine_name = sxng_request.args.get(
'engine', default=
None, type=str)
1115 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
1116 if selected_engine_name:
1117 if selected_engine_name
not in filtered_engines:
1118 selected_engine_name =
None
1120 filtered_engines = [selected_engine_name]
1122 checker_results = checker_get_result()
1124 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
1127 engine_stats = get_engines_stats(filtered_engines)
1128 engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
1130 if sort_order
not in STATS_SORT_PARAMETERS:
1133 reverse, key_name, default_value = STATS_SORT_PARAMETERS[sort_order]
1135 def get_key(engine_stat):
1136 reliability = engine_reliabilities.get(engine_stat[
'name'], {}).get(
'reliability', 0)
1137 reliability_order = 0
if reliability
else 1
1138 if key_name ==
'reliability':
1140 reliability_order = 0
1142 key = engine_stat.get(key_name)
or default_value
1144 reliability_order = 1 - reliability_order
1145 return (reliability_order, key, engine_stat[
'name'])
1147 technical_report = []
1148 for error
in engine_reliabilities.get(selected_engine_name, {}).get(
'errors', []):
1149 technical_report.append(
1151 Error: {error['exception_classname'] or error['log_message']} \
1152 Parameters: {error['log_parameters']} \
1153 File name: {error['filename'] }:{ error['line_no'] } \
1154 Error Function: {error['function']} \
1155 Code: {error['code']} \
1160 technical_report =
' '.join(technical_report)
1162 engine_stats[
'time'] = sorted(engine_stats[
'time'], reverse=reverse, key=get_key)
1166 sort_order = sort_order,
1167 engine_stats = engine_stats,
1168 engine_reliabilities = engine_reliabilities,
1169 selected_engine_name = selected_engine_name,
1170 searx_git_branch = GIT_BRANCH,
1171 technical_report = technical_report,
1176@app.route('/stats/errors', methods=['GET'])
1191 password = settings[
'general'].get(
"open_metrics")
1193 if not (settings[
'general'].get(
"enable_metrics")
and password):
1194 return Response(
'open metrics is disabled', status=404, mimetype=
'text/plain')
1196 if not sxng_request.authorization
or sxng_request.authorization.password != password:
1197 return Response(
'access forbidden', status=401, mimetype=
'text/plain')
1199 filtered_engines = dict(filter(
lambda kv: sxng_request.preferences.validate_token(kv[1]), engines.items()))
1201 checker_results = checker_get_result()
1203 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
1206 engine_stats = get_engines_stats(filtered_engines)
1207 engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
1208 metrics_text = openmetrics(engine_stats, engine_reliabilities)
1210 return Response(metrics_text, mimetype=
'text/plain')
1213@app.route('/robots.txt', methods=['GET'])