[feat] marginalia: add support for pagination

- SVGRepo uses Cloudflare for every session, no matter
if you're opening it in a browser or not

client/simple/src/js/toolkit.ts

@@ -5,7 +5,6 @@ import type { KeyBindingLayout } from "./main/keyboard.ts";
 // synced with searx/webapp.py get_client_settings
 type Settings = {
   plugins?: string[];
-  advanced_search?: boolean;
   autocomplete?: string;
   autocomplete_min?: number;
   doi_resolver?: string;

docs/admin/settings/index.rst

@@ -19,6 +19,7 @@ Settings
    settings_search
    settings_server
    settings_ui
+   settings_preferences
    settings_redis
    settings_valkey
    settings_outgoing

docs/admin/settings/settings_preferences.rst

@@ -0,0 +1,8 @@
+.. _settings preferences:
+
+================
+``preferences:``
+================
+
+.. autoclass:: searx._settings.SettingsPref
+   :members:

docs/admin/settings/settings_server.rst

@@ -47,6 +47,7 @@
   activated:
 
   - :py:obj:`searx.botdetection.link_token` in the :ref:`limiter`
+  - :ref:`image_proxy`
 
 .. _image_proxy:
 

searx/__init__.py

@@ -10,6 +10,7 @@ from os.path import dirname, abspath
 import logging
 
 import msgspec
+from ._settings import SettingsPref
 
 # Debug
 LOG_FORMAT_DEBUG: str = '%(levelname)-7s %(name)-30.30s: %(message)s'
@@ -47,6 +48,12 @@ def init_settings():
     settings.clear()
     settings.update(cfg)
 
+    if get_setting("server.public_instance"):
+        # enable image proxy for public instances #6125
+        settings["server"]["image_proxy"] = True
+        pref: SettingsPref = get_setting("preferences")
+        pref.lock.add("image_proxy")
+
     sxng_debug = get_setting("general.debug")
     if sxng_debug:
         _logging_config_debug()
@@ -66,7 +73,7 @@ def init_settings():
     if settings['server']['public_instance']:
         logger.warning(
             "Be aware you have activated features intended only for public instances. "
-            "This force the usage of the limiter and link_token / "
+            "This force the usage of the limiter, link_token and image proxy / "
             "see https://docs.searxng.org/admin/searx.limiter.html"
         )
 

searx/_settings.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Implementation of the :py:obj:`preference <searx.preference>` settings."""
+# pylint: disable = too-few-public-methods
+
+import typing as t
+import msgspec
+
+
+class SettingsPref(msgspec.Struct, kw_only=True, forbid_unknown_fields=True):
+    """Options for configuring the preferences
+
+    .. code:: yaml
+
+       preferences:
+         lock:
+           - favicon_resolver
+           - image_proxy
+           - method
+       # ...
+
+    """
+
+    lock: set[
+        t.Literal[
+            "categories",
+            "language",
+            "locale",
+            "autocomplete",
+            "favicon_resolver",
+            "image_proxy",
+            "method",
+            "safesearch",
+            "theme",
+            "results_on_new_tab",
+            "doi_resolver",
+            "simple_style",
+            "center_alignment",
+            "query_in_title",
+            "search_on_category_select",
+        ]
+    ] = set()
+    """Lock arbitrary settings on the preferences page."""

searx/engines/marginalia.py

@@ -45,7 +45,7 @@ about = {
 base_url = "https://api2.marginalia-search.com"
 safesearch = True
 categories = ["general"]
-paging = False
+paging = True
 results_per_page = 20
 api_key = None
 """To get an API key, please follow the instructions from `Key and license`_
@@ -85,7 +85,12 @@ class ApiSearchResults(t.TypedDict):
 
 def request(query: str, params: dict[str, t.Any]):
 
-    query_params = {"count": results_per_page, "nsfw": min(params["safesearch"], 1), "query": query}
+    query_params = {
+        "page": params["pageno"],
+        "count": results_per_page,
+        "nsfw": min(params["safesearch"], 1),
+        "query": query,
+    }
 
     params["url"] = f"{base_url}/search?{urlencode(query_params)}"
     params["headers"]["User-Agent"] = searxng_useragent()

searx/engines/svgrepo.py

@@ -1,44 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Svgrepo (icons)"""
-
-from lxml import html
-from searx.utils import extract_text, eval_xpath, eval_xpath_list
-
-about = {
-    "website": 'https://www.svgrepo.com',
-    "official_api_documentation": 'https://svgapi.com',
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": 'HTML',
-}
-
-paging = True
-categories = ['images', 'icons']
-base_url = "https://www.svgrepo.com"
-
-results_xpath = "//div[@class='style_nodeListing__7Nmro']/div"
-url_xpath = ".//a/@href"
-title_xpath = ".//a/@title"
-img_src_xpath = ".//img/@src"
-
-
-def request(query, params):
-    params['url'] = f"{base_url}/vectors/{query}/{params['pageno']}/"
-    return params
-
-
-def response(resp):
-    results = []
-
-    dom = html.fromstring(resp.text)
-    for result in eval_xpath_list(dom, results_xpath):
-        results.append(
-            {
-                'template': 'images.html',
-                'url': base_url + extract_text(eval_xpath(result, url_xpath)),
-                'title': extract_text(eval_xpath(result, title_xpath)).replace(" SVG File", "").replace("Show ", ""),
-                'img_src': extract_text(eval_xpath(result, img_src_xpath)),
-            }
-        )
-
-    return results

searx/preferences.py

@@ -17,13 +17,14 @@ import babel.core
 
 import searx.plugins
 
-from searx import settings, autocomplete, favicons
+from searx import get_setting, settings, autocomplete, favicons
 from searx.enginelib import Engine
 from searx.engines import DEFAULT_CATEGORY
 from searx.extended_types import SXNG_Request
 from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE
 
+from ._settings import SettingsPref
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
 DOI_RESOLVERS = list(settings['doi_resolvers'])
@@ -386,6 +387,7 @@ class ClientPref:
         return cls(locale=locale)
 
 
+@t.final
 class Preferences:
     """Validates and saves preferences to cookies"""
 
@@ -400,95 +402,91 @@ class Preferences:
 
         super().__init__()
 
+        self.cfg: SettingsPref = get_setting("preferences")
+
         self.key_value_settings: dict[str, Setting] = {
-            # fmt: off
             'categories': MultipleChoiceSetting(
-                ['general'],
-                locked=is_locked('categories'),
-                choices=categories + ['none']
+                ["general"],
+                locked="categories" in self.cfg.lock,
+                choices=categories + ["none"],
             ),
             'language': SearchLanguageSetting(
-                settings['search']['default_lang'],
-                locked=is_locked('language'),
-                choices=settings['search']['languages'] + ['']
+                get_setting("search.default_lang"),
+                locked="language" in self.cfg.lock,
+                choices=get_setting("search.languages") + [""],
             ),
             'locale': EnumStringSetting(
-                settings['ui']['default_locale'],
-                locked=is_locked('locale'),
-                choices=list(LOCALE_NAMES.keys()) + ['']
+                get_setting("ui.default_locale"),
+                locked="locale" in self.cfg.lock,
+                choices=list(LOCALE_NAMES.keys()) + [""],
             ),
             'autocomplete': EnumStringSetting(
-                settings['search']['autocomplete'],
-                locked=is_locked('autocomplete'),
-                choices=list(autocomplete.backends.keys()) + ['']
+                get_setting("search.autocomplete"),
+                locked="autocomplete" in self.cfg.lock,
+                choices=list(autocomplete.backends.keys()) + [""],
             ),
             'favicon_resolver': EnumStringSetting(
-                settings['search']['favicon_resolver'],
-                locked=is_locked('favicon_resolver'),
-                choices=list(favicons.proxy.CFG.resolver_map.keys()) + ['']
+                get_setting("search.favicon_resolver"),
+                locked="favicon_resolver" in self.cfg.lock,
+                choices=list(favicons.proxy.CFG.resolver_map.keys()) + [''],
             ),
             'image_proxy': BooleanSetting(
-                settings['server']['image_proxy'],
-                locked=is_locked('image_proxy')
+                get_setting("server.image_proxy"),
+                locked="image_proxy" in self.cfg.lock,
             ),
             'method': EnumStringSetting(
-                settings['server']['method'],
-                locked=is_locked('method'),
-                choices=('GET', 'POST')
+                get_setting("server.method"),
+                locked="method" in self.cfg.lock,
+                choices=("GET", "POST"),
             ),
             'safesearch': MapSetting(
-                settings['search']['safe_search'],
-                locked=is_locked('safesearch'),
+                get_setting("search.safe_search"),
+                locked="safesearch" in self.cfg.lock,
                 map={
-                    '0': 0,
-                    '1': 1,
-                    '2': 2
-                }
+                    "0": 0,
+                    "1": 1,
+                    "2": 2,
+                },
             ),
             'theme': EnumStringSetting(
-                settings['ui']['default_theme'],
-                locked=is_locked('theme'),
-                choices=themes
+                get_setting("ui.default_theme"),
+                locked="theme" in self.cfg.lock,
+                choices=themes,
             ),
             'results_on_new_tab': BooleanSetting(
-                settings['ui']['results_on_new_tab'],
-                locked=is_locked('results_on_new_tab')
+                get_setting("ui.results_on_new_tab"),
+                locked="results_on_new_tab" in self.cfg.lock,
             ),
             'doi_resolver': MultipleChoiceSetting(
-                [settings['default_doi_resolver'], ],
-                locked=is_locked('doi_resolver'),
-                choices=DOI_RESOLVERS
+                [get_setting("default_doi_resolver")],
+                locked="doi_resolver" in self.cfg.lock,
+                choices=DOI_RESOLVERS,
             ),
             'simple_style': EnumStringSetting(
-                settings['ui']['theme_args']['simple_style'],
-                locked=is_locked('simple_style'),
-                choices=['', 'auto', 'light', 'dark', 'black']
+                get_setting("ui.theme_args.simple_style"),
+                locked="simple_style" in self.cfg.lock,
+                choices=["", "auto", "light", "dark", "black"],
             ),
             'center_alignment': BooleanSetting(
-                settings['ui']['center_alignment'],
-                locked=is_locked('center_alignment')
-            ),
-            'advanced_search': BooleanSetting(
-                settings['ui']['advanced_search'],
-                locked=is_locked('advanced_search')
+                get_setting("ui.center_alignment"),
+                locked="center_alignment" in self.cfg.lock,
             ),
             'query_in_title': BooleanSetting(
-                settings['ui']['query_in_title'],
-                locked=is_locked('query_in_title')
+                get_setting("ui.query_in_title"),
+                locked="query_in_title" in self.cfg.lock,
             ),
             'search_on_category_select': BooleanSetting(
-                settings['ui']['search_on_category_select'],
-                locked=is_locked('search_on_category_select')
+                get_setting("ui.search_on_category_select"),
+                locked="search_on_category_select" in self.cfg.lock,
             ),
             'hotkeys': EnumStringSetting(
-                settings['ui']['hotkeys'],
-                choices=['default', 'vim']
+                get_setting("ui.hotkeys"),
+                choices=["default", "vim"],
             ),
             'url_formatting': EnumStringSetting(
-                settings['ui']['url_formatting'],
-                choices=['pretty', 'full', 'host']
+                get_setting("ui.url_formatting"),
+                choices=["pretty", "full", "host"],
             ),
-            # fmt: on
         }
 
         self.engines = EnginesSetting('engines', engines=engines.values())
@@ -597,12 +595,3 @@ class Preferences:
                     break
 
         return valid
-
-
-def is_locked(setting_name: str):
-    """Checks if a given setting name is locked by settings.yml"""
-    if 'preferences' not in settings:
-        return False
-    if 'lock' not in settings['preferences']:
-        return False
-    return setting_name in settings['preferences']['lock']

searx/settings.yml

@@ -154,14 +154,13 @@ ui:
   # URL formatting: pretty, full or host
   url_formatting: pretty
 
+preferences:
   # Lock arbitrary settings on the preferences page.
-#
-# preferences:
-#   lock:
+  lock: []
 #     - categories
 #     - language
 #     - autocomplete
-#     - favicon
+#     - favicon_resolver
 #     - safesearch
 #     - method
 #     - doi_resolver
@@ -2549,12 +2548,6 @@ engines:
       results: HTML
       language: de
 
-  - name: svgrepo
-    engine: svgrepo
-    shortcut: svg
-    timeout: 10.0
-    disabled: true
-
   - name: tootfinder
     engine: tootfinder
     shortcut: toot

searx/settings_defaults.py

@@ -15,6 +15,7 @@ import msgspec
 from typing_extensions import override
 from .brand import SettingsBrand
 from .sxng_locales import sxng_locales
+from ._settings import SettingsPref
 
 searx_dir = abspath(dirname(__file__))
 
@@ -146,6 +147,8 @@ def apply_schema(settings: dict[str, t.Any], schema: dict[str, t.Any], path_list
                 # Type Validation at runtime:
                 # https://jcristharif.com/msgspec/structs.html#type-validation
                 cfg_dict = settings.get(key)
+                if cfg_dict is None:
+                    cfg_dict = {}
                 cfg_json = msgspec.json.encode(cfg_dict)
                 settings[key] = msgspec.json.decode(cfg_json, type=value)
             except msgspec.ValidationError as e:
@@ -236,16 +239,13 @@ SCHEMA: dict[str, t.Any] = {
         },
         'center_alignment': SettingsValue(bool, False),
         'results_on_new_tab': SettingsValue(bool, False),
-        'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
         'cache_url': SettingsValue(str, 'https://web.archive.org/web/'),
         'search_on_category_select': SettingsValue(bool, True),
         'hotkeys': SettingsValue(('default', 'vim'), 'default'),
         'url_formatting': SettingsValue(('pretty', 'full', 'host'), 'pretty'),
     },
-    'preferences': {
-        'lock': SettingsValue(list, []),
-    },
+    "preferences": SettingsPref,
     'outgoing': {
         'useragent_suffix': SettingsValue(str, ''),
         'request_timeout': SettingsValue(numbers.Real, 3.0),

searx/templates/simple/preferences.html

@@ -180,7 +180,7 @@
     {%- if 'autocomplete' not in locked_preferences -%}
       {%- include 'simple/preferences/autocomplete.html' -%}
     {%- endif -%}
-    {%- if 'favicon' not in locked_preferences -%}
+    {%- if 'favicon_resolver' not in locked_preferences -%}
       {%- include 'simple/preferences/favicon.html' -%}
     {%- endif -%}
     {% if 'safesearch' not in locked_preferences %}

searx/webadapter.py

@@ -8,7 +8,7 @@ from searx.webutils import VALID_LANGUAGE_CODE
 from searx.query import RawTextQuery
 from searx.engines import categories, engines
 from searx.search.models import SearchQuery, EngineRef
-from searx.preferences import Preferences, is_locked
+from searx.preferences import Preferences
 
 
 # remove duplicate queries.
@@ -53,7 +53,7 @@ def parse_pageno(form: Dict[str, str]) -> int:
 
 
 def parse_lang(preferences: Preferences, form: Dict[str, str], raw_text_query: RawTextQuery) -> str:
-    if is_locked('language'):
+    if "language" in preferences.cfg.lock:
         return preferences.get_value('language')
     # get language
     # set specific language if set on request, query or preferences
@@ -73,7 +73,7 @@ def parse_lang(preferences: Preferences, form: Dict[str, str], raw_text_query: R
 
 
 def parse_safesearch(preferences: Preferences, form: Dict[str, str]) -> int:
-    if is_locked('safesearch'):
+    if "safesearch" in preferences.cfg.lock:
         return preferences.get_value('safesearch')
 
     if 'safesearch' in form:
@@ -135,7 +135,7 @@ def parse_category_form(query_categories: List[str], name: str, value: str) -> N
 def get_selected_categories(preferences: Preferences, form: Optional[Dict[str, str]]) -> List[str]:
     selected_categories = []
 
-    if not is_locked('categories') and form is not None:
+    if not "categories" in preferences.cfg.lock and form is not None:
         for name, value in form.items():
             parse_category_form(selected_categories, name, value)
 
@@ -175,7 +175,7 @@ def parse_generic(preferences: Preferences, form: Dict[str, str], disabled_engin
 
     # set categories/engines
     explicit_engine_list = False
-    if not is_locked('categories'):
+    if not "categories" in preferences.cfg.lock:
         # parse the form only if the categories are not locked
         for pd_name, pd in form.items():  # pylint: disable=invalid-name
             if pd_name == 'engines':
@@ -266,7 +266,7 @@ def get_search_query_from_webapp(
     if query_lang == 'auto':
         query_lang = preferences.client.locale_tag or 'all'
 
-    if not is_locked('categories') and raw_text_query.specific:
+    if not "categories" in preferences.cfg.lock and raw_text_query.specific:
         # if engines are calculated from query,
         # set categories by using that information
         query_engineref_list = raw_text_query.enginerefs

searx/webapp.py

@@ -377,7 +377,6 @@ def get_client_settings():
         'theme_static_path': custom_url_for('static', filename='themes/simple'),
         'results_on_new_tab': req_pref.get_value('results_on_new_tab'),
         'favicon_resolver': req_pref.get_value('favicon_resolver'),
-        'advanced_search': req_pref.get_value('advanced_search'),
         'query_in_title': req_pref.get_value('query_in_title'),
         'safesearch': req_pref.get_value('safesearch'),
         'theme': req_pref.get_value('theme'),
@@ -977,7 +976,7 @@ def preferences():
         current_doi_resolver = get_doi_resolver(),
         allowed_plugins = allowed_plugins,
         preferences_url_params = sxng_request.preferences.get_as_url_params(),
-        locked_preferences = get_setting("preferences.lock", []),
+        locked_preferences = get_setting("preferences").lock,
         doi_resolvers = get_setting("doi_resolvers", {}),
         # fmt: on
     )