216 lexer = get_lexer_by_name(language, stripall=
True)
218 except Exception
as e:
219 logger.warning(
"pygments lexer: %s " % e)
221 lexer = get_lexer_by_name(
'text', stripall=
True)
226 line_code_start =
None
229 for line, code
in codelines:
231 line_code_start = line
234 if last_line
is not None and last_line + 1 != line:
237 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
238 html_code = html_code + highlight(tmp_code, lexer, formatter)
242 line_code_start = line
245 tmp_code += code +
'\n'
251 formatter = HtmlFormatter(linenos=
'inline', linenostart=line_code_start, cssclass=
"code-highlight")
252 html_code = html_code + highlight(tmp_code, lexer, formatter)
372 kwargs[
'client_settings'] = str(
383 kwargs[
'endpoint'] =
'results' if 'q' in kwargs
else request.endpoint
384 kwargs[
'cookies'] = request.cookies
385 kwargs[
'errors'] = request.errors
386 kwargs[
'link_token'] = link_token.get_token()
389 kwargs[
'preferences'] = request.preferences
390 kwargs[
'autocomplete'] = request.preferences.get_value(
'autocomplete')
391 kwargs[
'infinite_scroll'] = request.preferences.get_value(
'infinite_scroll')
392 kwargs[
'search_on_category_select'] = request.preferences.get_value(
'search_on_category_select')
393 kwargs[
'hotkeys'] = request.preferences.get_value(
'hotkeys')
394 kwargs[
'results_on_new_tab'] = request.preferences.get_value(
'results_on_new_tab')
395 kwargs[
'advanced_search'] = request.preferences.get_value(
'advanced_search')
396 kwargs[
'query_in_title'] = request.preferences.get_value(
'query_in_title')
397 kwargs[
'safesearch'] = str(request.preferences.get_value(
'safesearch'))
398 kwargs[
'theme'] = request.preferences.get_value(
'theme')
399 kwargs[
'method'] = request.preferences.get_value(
'method')
400 kwargs[
'categories_as_tabs'] = list(settings[
'categories_as_tabs'].keys())
402 kwargs[
'DEFAULT_CATEGORY'] = DEFAULT_CATEGORY
405 kwargs[
'sxng_locales'] = [l
for l
in sxng_locales
if l[0]
in settings[
'search'][
'languages']]
407 locale = request.preferences.get_value(
'locale')
410 if locale
in RTL_LOCALES
and 'rtl' not in kwargs:
413 if 'current_language' not in kwargs:
414 kwargs[
'current_language'] = parse_lang(request.preferences, {},
RawTextQuery(
'', []))
417 kwargs[
'search_formats'] = [x
for x
in settings[
'search'][
'formats']
if x !=
'html']
418 kwargs[
'instance_name'] = get_setting(
'general.instance_name')
419 kwargs[
'searx_version'] = VERSION_STRING
420 kwargs[
'searx_git_url'] = GIT_URL
421 kwargs[
'enable_metrics'] = get_setting(
'general.enable_metrics')
422 kwargs[
'get_setting'] = get_setting
423 kwargs[
'get_pretty_url'] = get_pretty_url
426 donation_url = get_setting(
'general.donation_url')
427 if donation_url
is True:
429 kwargs[
'donation_url'] = donation_url
432 kwargs[
'url_for'] = custom_url_for
433 kwargs[
'image_proxify'] = image_proxify
434 kwargs[
'proxify'] = morty_proxify
if settings[
'result_proxy'][
'url']
is not None else None
435 kwargs[
'proxify_results'] = settings[
'result_proxy'][
'proxify_results']
436 kwargs[
'cache_url'] = settings[
'ui'][
'cache_url']
437 kwargs[
'get_result_template'] = get_result_template
438 kwargs[
'doi_resolver'] = get_doi_resolver(request.preferences)
439 kwargs[
'opensearch_url'] = (
440 url_for(
'opensearch')
444 'method': request.preferences.get_value(
'method'),
445 'autocomplete': request.preferences.get_value(
'autocomplete'),
449 kwargs[
'urlparse'] = urlparse
452 kwargs[
'scripts'] = set()
453 for plugin
in request.user_plugins:
454 for script
in plugin.js_dependencies:
455 kwargs[
'scripts'].add(script)
458 kwargs[
'styles'] = set()
459 for plugin
in request.user_plugins:
460 for css
in plugin.css_dependencies:
461 kwargs[
'styles'].add(css)
463 start_time = default_timer()
464 result = render_template(
'{}/{}'.
format(kwargs[
'theme'], template_name), **kwargs)
465 request.render_time += default_timer() - start_time
472 request.start_time = default_timer()
473 request.render_time = 0
477 client_pref = ClientPref.from_http_request(request)
479 preferences =
Preferences(themes, list(categories.keys()), engines, plugins, client_pref)
481 user_agent = request.headers.get(
'User-Agent',
'').lower()
482 if 'webkit' in user_agent
and 'android' in user_agent:
483 preferences.key_value_settings[
'method'].value =
'GET'
484 request.preferences = preferences
487 preferences.parse_dict(request.cookies)
489 except Exception
as e:
490 logger.exception(e, exc_info=
True)
491 request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
495 request.form = dict(request.form.items())
496 for k, v
in request.args.items():
497 if k
not in request.form:
500 if request.form.get(
'preferences'):
501 preferences.parse_encoded_data(request.form[
'preferences'])
504 preferences.parse_dict(request.form)
505 except Exception
as e:
506 logger.exception(e, exc_info=
True)
507 request.errors.append(gettext(
'Invalid settings'))
511 if not preferences.get_value(
"language"):
513 preferences.parse_dict({
"language": language})
514 logger.debug(
'set language %s (from browser)', preferences.get_value(
"language"))
518 if not preferences.get_value(
"locale"):
520 preferences.parse_dict({
"locale": locale})
521 logger.debug(
'set locale %s (from browser)', preferences.get_value(
"locale"))
524 request.user_plugins = []
525 allowed_plugins = preferences.plugins.get_enabled()
526 disabled_plugins = preferences.plugins.get_disabled()
527 for plugin
in plugins:
528 if (plugin.default_on
and plugin.id
not in disabled_plugins)
or plugin.id
in allowed_plugins:
529 request.user_plugins.append(plugin)
544 total_time = default_timer() - request.start_time
546 'total;dur=' + str(round(total_time * 1000, 3)),
547 'render;dur=' + str(round(request.render_time * 1000, 3)),
549 if len(request.timings) > 0:
550 timings = sorted(request.timings, key=
lambda t: t.total)
552 'total_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.total * 1000, 3))
for i, t
in enumerate(timings)
555 'load_' + str(i) +
'_' + t.engine +
';dur=' + str(round(t.load * 1000, 3))
556 for i, t
in enumerate(timings)
559 timings_all = timings_all + timings_total + timings_load
560 response.headers.add(
'Server-Timing',
', '.join(timings_all))
624 """Search query in q and return results.
626 Supported outputs: html, json, csv, rss.
632 output_format = request.form.get(
'format',
'html')
633 if output_format
not in OUTPUT_FORMATS:
634 output_format =
'html'
636 if output_format
not in settings[
'search'][
'formats']:
640 if not request.form.get(
'q'):
641 if output_format ==
'html':
645 selected_categories=get_selected_categories(request.preferences, request.form),
652 raw_text_query =
None
653 result_container =
None
655 search_query, raw_text_query, _, _, selected_locale = get_search_query_from_webapp(
656 request.preferences, request.form
659 result_container = search.search()
661 except SearxParameterException
as e:
662 logger.exception(
'search error: SearxParameterException')
664 except Exception
as e:
665 logger.exception(e, exc_info=
True)
666 return index_error(output_format, gettext(
'search error')), 500
669 if result_container.redirect_url:
670 return redirect(result_container.redirect_url)
674 request.timings = result_container.get_timings()
678 if output_format ==
'json':
680 response = webutils.get_json_response(search_query, result_container)
681 return Response(response, mimetype=
'application/json')
683 if output_format ==
'csv':
686 webutils.write_csv_response(csv, result_container)
689 response = Response(csv.stream.read(), mimetype=
'application/csv')
690 cont_disp =
'attachment;Filename=searx_-_{0}.csv'.
format(search_query.query)
691 response.headers.add(
'Content-Disposition', cont_disp)
696 current_template =
None
697 previous_result =
None
699 results = result_container.get_ordered_results()
701 if search_query.redirect_to_first_result
and results:
702 return redirect(results[0][
'url'], 302)
704 for result
in results:
705 if output_format ==
'html':
706 if 'content' in result
and result[
'content']:
707 result[
'content'] = highlight_content(escape(result[
'content'][:1024]), search_query.query)
708 if 'title' in result
and result[
'title']:
709 result[
'title'] = highlight_content(escape(result[
'title']
or ''), search_query.query)
712 result[
'pretty_url'] = webutils.prettify_url(result[
'url'])
713 if result.get(
'publishedDate'):
715 result[
'pubdate'] = result[
'publishedDate'].strftime(
'%Y-%m-%d %H:%M:%S%z')
717 result[
'publishedDate'] =
None
719 result[
'publishedDate'] = webutils.searxng_l10n_timespan(result[
'publishedDate'])
723 if current_template != result.get(
'template'):
724 result[
'open_group'] =
True
726 previous_result[
'close_group'] =
True
727 current_template = result.get(
'template')
728 previous_result = result
731 previous_result[
'close_group'] =
True
735 if output_format ==
'rss':
737 'opensearch_response_rss.xml',
739 answers=result_container.answers,
740 corrections=result_container.corrections,
741 suggestions=result_container.suggestions,
743 number_of_results=result_container.number_of_results,
745 return Response(response_rss, mimetype=
'text/xml')
750 suggestion_urls = list(
752 lambda suggestion: {
'url': raw_text_query.changeQuery(suggestion).getFullQuery(),
'title': suggestion},
753 result_container.suggestions,
757 correction_urls = list(
759 lambda correction: {
'url': raw_text_query.changeQuery(correction).getFullQuery(),
'title': correction},
760 result_container.corrections,
772 selected_categories = search_query.categories,
773 pageno = search_query.pageno,
774 time_range = search_query.time_range
or '',
775 number_of_results = format_decimal(result_container.number_of_results),
776 suggestions = suggestion_urls,
777 answers = result_container.answers,
778 corrections = correction_urls,
779 infoboxes = result_container.infoboxes,
780 engine_data = result_container.engine_data,
781 paging = result_container.paging,
782 unresponsive_engines = webutils.get_translated_errors(
783 result_container.unresponsive_engines
785 current_locale = request.preferences.get_value(
"locale"),
786 current_language = selected_locale,
787 search_language = match_locale(
788 search.search_query.lang,
789 settings[
'search'][
'languages'],
790 fallback=request.preferences.get_value(
"language")
792 timeout_limit = request.form.get(
'timeout_limit',
None)
797@app.route('/about', methods=['GET'])
822 """Return autocompleter results"""
828 disabled_engines = request.preferences.engines.get_disabled()
831 raw_text_query =
RawTextQuery(request.form.get(
'q',
''), disabled_engines)
832 sug_prefix = raw_text_query.getQuery()
836 if len(raw_text_query.autocomplete_list) == 0
and len(sug_prefix) > 0:
839 sxng_locale = request.preferences.get_value(
'language')
840 backend_name = request.preferences.get_value(
'autocomplete')
842 for result
in search_autocomplete(backend_name, sug_prefix, sxng_locale):
845 if result != sug_prefix:
846 results.append(raw_text_query.changeQuery(result).getFullQuery())
848 if len(raw_text_query.autocomplete_list) > 0:
849 for autocomplete_text
in raw_text_query.autocomplete_list:
850 results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
852 for answers
in ask(raw_text_query):
853 for answer
in answers:
854 results.append(str(answer[
'answer']))
856 if request.headers.get(
'X-Requested-With') ==
'XMLHttpRequest':
858 suggestions = json.dumps(results)
859 mimetype =
'application/json'
862 suggestions = json.dumps([sug_prefix, results])
863 mimetype =
'application/x-suggestions+json'
865 suggestions = escape(suggestions,
False)
866 return Response(suggestions, mimetype=mimetype)
869@app.route('/preferences', methods=['GET', 'POST'])
871 """Render preferences page && save user preferences"""
877 if request.args.get(
'preferences')
or request.form.get(
'preferences'):
878 resp = make_response(redirect(url_for(
'index', _external=
True)))
879 return request.preferences.save(resp)
882 if request.method ==
'POST':
883 resp = make_response(redirect(url_for(
'index', _external=
True)))
885 request.preferences.parse_form(request.form)
886 except ValidationException:
887 request.errors.append(gettext(
'Invalid settings, please edit your preferences'))
889 return request.preferences.save(resp)
892 image_proxy = request.preferences.get_value(
'image_proxy')
893 disabled_engines = request.preferences.engines.get_disabled()
894 allowed_plugins = request.preferences.plugins.get_enabled()
897 filtered_engines = dict(filter(
lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
899 engines_by_category = {}
902 engines_by_category[c] = [e
for e
in categories[c]
if e.name
in filtered_engines]
904 list.sort(engines_by_category[c], key=
lambda e: e.name)
910 for _, e
in filtered_engines.items():
911 h = histogram(
'engine', e.name,
'time',
'total')
912 median = round(h.percentage(50), 1)
if h.count > 0
else None
913 rate80 = round(h.percentage(80), 1)
if h.count > 0
else None
914 rate95 = round(h.percentage(95), 1)
if h.count > 0
else None
916 max_rate95 = max(max_rate95, rate95
or 0)
918 result_count_sum = histogram(
'engine', e.name,
'result',
'count').sum
919 successful_count = counter(
'engine', e.name,
'search',
'count',
'successful')
920 result_count = int(result_count_sum / float(successful_count))
if successful_count
else 0
926 'warn_timeout': e.timeout > settings[
'outgoing'][
'request_timeout'],
927 'supports_selected_language': e.traits.is_locale_supported(
928 str(request.preferences.get_value(
'language')
or 'all')
930 'result_count': result_count,
936 engine_errors = get_engine_errors(filtered_engines)
937 checker_results = checker_get_result()
939 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
941 for _, e
in filtered_engines.items():
942 checker_result = checker_results.get(e.name, {})
943 checker_success = checker_result.get(
'success',
True)
944 errors = engine_errors.get(e.name)
or []
945 if counter(
'engine', e.name,
'search',
'count',
'sent') == 0:
948 elif checker_success
and not errors:
950 elif 'simple' in checker_result.get(
'errors', {}):
956 reliability = 100 - sum([error[
'percentage']
for error
in errors
if not error.get(
'secondary')])
958 reliabilities[e.name] = {
959 'reliability': reliability,
961 'checker': checker_results.get(e.name, {}).get(
'errors', {}).keys(),
965 reliabilities_errors = []
967 error_user_text =
None
968 if error.get(
'secondary')
or 'exception_classname' not in error:
970 error_user_text = exception_classname_to_text.get(error.get(
'exception_classname'))
972 error_user_text = exception_classname_to_text[
None]
973 if error_user_text
not in reliabilities_errors:
974 reliabilities_errors.append(error_user_text)
975 reliabilities[e.name][
'errors'] = reliabilities_errors
979 for _, e
in filtered_engines.items():
980 supports_selected_language = e.traits.is_locale_supported(
981 str(request.preferences.get_value(
'language')
or 'all')
983 safesearch = e.safesearch
984 time_range_support = e.time_range_support
985 for checker_test_name
in checker_results.get(e.name, {}).get(
'errors', {}):
986 if supports_selected_language
and checker_test_name.startswith(
'lang_'):
987 supports_selected_language =
'?'
988 elif safesearch
and checker_test_name ==
'safesearch':
990 elif time_range_support
and checker_test_name ==
'time_range':
991 time_range_support =
'?'
993 'supports_selected_language': supports_selected_language,
994 'safesearch': safesearch,
995 'time_range_support': time_range_support,
1001 selected_categories = get_selected_categories(request.preferences, request.form),
1002 locales = LOCALE_NAMES,
1003 current_locale = request.preferences.get_value(
"locale"),
1004 image_proxy = image_proxy,
1005 engines_by_category = engines_by_category,
1007 max_rate95 = max_rate95,
1008 reliabilities = reliabilities,
1009 supports = supports,
1011 {
'info': a.self_info(),
'keywords': a.keywords}
1014 disabled_engines = disabled_engines,
1015 autocomplete_backends = autocomplete_backends,
1016 shortcuts = {y: x
for x, y
in engine_shortcuts.items()},
1019 doi_resolvers = settings[
'doi_resolvers'],
1020 current_doi_resolver = get_doi_resolver(request.preferences),
1021 allowed_plugins = allowed_plugins,
1022 preferences_url_params = request.preferences.get_as_url_params(),
1023 locked_preferences = settings[
'preferences'][
'lock'],
1029@app.route('/image_proxy', methods=['GET'])
1033 url = request.args.get(
'url')
1037 if not is_hmac_of(settings[
'server'][
'secret_key'], url.encode(), request.args.get(
'h',
'')):
1040 maximum_size = 5 * 1024 * 1024
1041 forward_resp =
False
1045 'User-Agent': gen_useragent(),
1046 'Accept':
'image/webp,*/*',
1047 'Accept-Encoding':
'gzip, deflate',
1051 set_context_network_name(
'image_proxy')
1052 resp, stream = http_stream(method=
'GET', url=url, headers=request_headers, allow_redirects=
True)
1053 content_length = resp.headers.get(
'Content-Length')
1054 if content_length
and content_length.isdigit()
and int(content_length) > maximum_size:
1055 return 'Max size', 400
1057 if resp.status_code != 200:
1058 logger.debug(
'image-proxy: wrong response code: %i', resp.status_code)
1059 if resp.status_code >= 400:
1060 return '', resp.status_code
1063 if not resp.headers.get(
'Content-Type',
'').startswith(
'image/')
and not resp.headers.get(
1065 ).startswith(
'binary/octet-stream'):
1066 logger.debug(
'image-proxy: wrong content-type: %s', resp.headers.get(
'Content-Type',
''))
1070 except httpx.HTTPError:
1071 logger.exception(
'HTTP error')
1074 if resp
and not forward_resp:
1079 except httpx.HTTPError:
1080 logger.exception(
'HTTP error on closing')
1083 nonlocal resp, stream
1089 except httpx.HTTPError
as e:
1090 logger.debug(
'Exception while closing response', e)
1093 headers = dict_subset(resp.headers, {
'Content-Type',
'Content-Encoding',
'Content-Length',
'Length'})
1094 response = Response(stream, mimetype=resp.headers[
'Content-Type'], headers=headers, direct_passthrough=
True)
1095 response.call_on_close(close_stream)
1097 except httpx.HTTPError:
1102@app.route('/engine_descriptions.json', methods=['GET'])
1104 locale = get_locale().split(
'_')[0]
1105 result = ENGINE_DESCRIPTIONS[
'en'].copy()
1107 for engine, description
in ENGINE_DESCRIPTIONS.get(locale, {}).items():
1108 result[engine] = description
1109 for engine, description
in result.items():
1110 if len(description) == 2
and description[1] ==
'ref':
1111 ref_engine, ref_lang = description[0].split(
':')
1112 description = ENGINE_DESCRIPTIONS[ref_lang][ref_engine]
1113 if isinstance(description, str):
1114 description = [description,
'wikipedia']
1115 result[engine] = description
1118 for engine_name, engine_mod
in engines.items():
1119 descr = getattr(engine_mod,
'about', {}).get(
'description',
None)
1120 if descr
is not None:
1121 result[engine_name] = [descr,
"SearXNG config"]
1123 return jsonify(result)
1126@app.route('/stats', methods=['GET'])
1128 """Render engine statistics page."""
1129 sort_order = request.args.get(
'sort', default=
'name', type=str)
1130 selected_engine_name = request.args.get(
'engine', default=
None, type=str)
1132 filtered_engines = dict(filter(
lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
1133 if selected_engine_name:
1134 if selected_engine_name
not in filtered_engines:
1135 selected_engine_name =
None
1137 filtered_engines = [selected_engine_name]
1139 checker_results = checker_get_result()
1141 checker_results[
'engines']
if checker_results[
'status'] ==
'ok' and 'engines' in checker_results
else {}
1144 engine_stats = get_engines_stats(filtered_engines)
1145 engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
1147 if sort_order
not in STATS_SORT_PARAMETERS:
1150 reverse, key_name, default_value = STATS_SORT_PARAMETERS[sort_order]
1152 def get_key(engine_stat):
1153 reliability = engine_reliabilities.get(engine_stat[
'name'], {}).get(
'reliability', 0)
1154 reliability_order = 0
if reliability
else 1
1155 if key_name ==
'reliability':
1157 reliability_order = 0
1159 key = engine_stat.get(key_name)
or default_value
1161 reliability_order = 1 - reliability_order
1162 return (reliability_order, key, engine_stat[
'name'])
1164 technical_report = []
1165 for error
in engine_reliabilities.get(selected_engine_name, {}).get(
'errors', []):
1166 technical_report.append(
1168 Error: {error['exception_classname'] or error['log_message']} \
1169 Parameters: {error['log_parameters']} \
1170 File name: {error['filename'] }:{ error['line_no'] } \
1171 Error Function: {error['function']} \
1172 Code: {error['code']} \
1177 technical_report =
' '.join(technical_report)
1179 engine_stats[
'time'] = sorted(engine_stats[
'time'], reverse=reverse, key=get_key)
1183 sort_order = sort_order,
1184 engine_stats = engine_stats,
1185 engine_reliabilities = engine_reliabilities,
1186 selected_engine_name = selected_engine_name,
1187 searx_git_branch = GIT_BRANCH,
1188 technical_report = technical_report,
1193@app.route('/stats/errors', methods=['GET'])