[chore] complete and normalize the attributes of engine objects (#6258)

Drop outdated engine attributes: supported_languages, language_aliases

Complete, normalize and document the type definitions for the engine-module and
engine-class.

For the ``engine.about`` section of the configuration, a type check is performed
based on structure ``searx.enginelib.EngineAbout``.

The property ``engine.about.language`` no longer exists; existing values have
been migrated to ``engine.language``.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/360search.py

@@ -22,8 +22,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "zh",
 }
+language = "zh"
 
 # Engine Configuration
 categories = ["general"]

searx/engines/__builtins__.pyi

@@ -5,19 +5,19 @@ intended monkey patching of the engine modules.
 .. attention::
 
    Monkey-patching modules is a practice from the past that shouldn't be
-   expanded upon.  In the long run, there should be an engine class that can be
-   inherited.  However, as long as this class doesn't exist, and as long as all
-   engine modules aren't converted to an engine class, these builtin types will
-   still be needed.
+   expanded upon.  In the long run, engines should be instances of
+   :py:obj:`searx.enginelib.Engine`. However, as long as long as all engine
+   modules aren't converted to this class, these builtin types will still be
+   needed.
 """
 
 import logging
 from searx.enginelib import traits as _traits
 
 logger: logging.Logger
-supported_languages: str
-language_aliases: str
 language_support: bool
+language: str
+region: str
 traits: _traits.EngineTraits
 
 # from searx.engines.ENGINE_DEFAULT_ARGS

searx/engines/__init__.py

@@ -17,37 +17,44 @@ from os.path import realpath, dirname
 
 import types
 import inspect
+import msgspec
 
 from searx import logger, settings
 from searx.utils import load_module
 
-if t.TYPE_CHECKING:
-    from searx.enginelib import Engine
+from searx.enginelib import Engine, EngineAbout
 
 logger = logger.getChild('engines')
 ENGINE_DIR = dirname(realpath(__file__))
 
 # Defaults for the namespace of an engine module, see load_engine()
-ENGINE_DEFAULT_ARGS: dict[str, int | str | list[t.Any] | dict[str, t.Any] | bool] = {
+ENGINE_DEFAULT_ARGS: dict[str, t.Any] = {
     # Common options in the engine module
     "engine_type": "online",
     "paging": False,
+    "max_page": 0,
     "time_range_support": False,
     "safesearch": False,
+    "language_support": False,
     # settings.yml
     "categories": ["general"],
+    "language": "",
+    "region": "",
     "enable_http": False,
     "shortcut": "-",
     "timeout": settings["outgoing"]["request_timeout"],
     "display_error_messages": True,
     "disabled": False,
     "inactive": False,
-    "about": {},
+    "about": EngineAbout(),
     "using_tor_proxy": False,
     "send_accept_language_header": True,
     "tokens": [],
-    "max_page": 0,
+    "weight": 1.0,
 }
+"""Default values that are set in an engine of type *module*, please compare
+with the class :py:obj:`searx.enginelib.Engine`."""
+
 # set automatically when an engine does not have any tab category
 DEFAULT_CATEGORY = 'other'
 
@@ -178,13 +185,27 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
 
 def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
     # set engine attributes from engine_data
+
+    kvargs: dict[str, t.Any]
+    if isinstance(engine.about, EngineAbout):
+        kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
+    else:
+        kvargs = {**engine.about, **engine_data.get("about", {})}
+
+    try:
+        engine.about = EngineAbout(**kvargs)
+    except TypeError as exc:
+        raise TypeError(
+            f"engine {engine_data['name']} ({engine_data['engine']}) - in the about section --> {exc}"
+        ) from exc
+
     for param_name, param_value in engine_data.items():
+        if param_name == "about":
+            continue
         if param_name == 'categories':
             if isinstance(param_value, str):
                 param_value = list(map(str.strip, param_value.split(',')))
             engine.categories = param_value  # type: ignore
-        elif hasattr(engine, 'about') and param_name == 'about':
-            engine.about = {**engine.about, **engine_data['about']}  # type: ignore
         else:
             setattr(engine, param_name, param_value)
 

searx/engines/acfun.py

@@ -16,12 +16,12 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "zh",
 }
 
 # Engine Configuration
 categories = ["videos"]
 paging = True
+language = "zh"
 
 # Base URL
 base_url = "https://www.acfun.cn"

searx/engines/ansa.py

@@ -42,8 +42,8 @@ about = {
     'use_official_api': False,
     'require_api_key': False,
     'results': 'HTML',
-    'language': 'it',
 }
+language = "it"
 
 
 def request(query, params):

searx/engines/azure.py

@@ -54,8 +54,8 @@ about = {
     "use_official_api": True,
     "require_api_key": True,
     "results": "JSON",
-    "language": "en",
 }
+language = "en"
 
 CACHE: EngineCache
 """Persistent (SQLite) key/value cache that deletes its values after ``expire``

searx/engines/baidu.py

@@ -23,8 +23,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "JSON",
-    "language": "zh",
 }
+language = "zh"
 
 paging = True
 categories = []

searx/engines/bpb.py

@@ -13,8 +13,8 @@ about = {
     'use_official_api': False,
     'require_api_key': False,
     'results': 'JSON',
-    'language': 'de',
 }
+language = "de"
 
 paging = True
 categories = ['general']

searx/engines/chefkoch.py

@@ -10,8 +10,8 @@ about = {
     'use_official_api': False,
     'require_api_key': False,
     'results': 'JSON',
-    'language': 'de',
 }
+language = "de"
 
 paging = True
 categories = []

searx/engines/chinaso.py

@@ -70,13 +70,13 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "JSON",
-    "language": "zh",
 }
 
 paging = True
 time_range_support = True
 results_per_page = 10
 categories = []
+language = "zh"
 
 ChinasoCategoryType = t.Literal['news', 'videos', 'images']
 """ChinaSo supports news, videos, images search.

searx/engines/demo_offline.py

@@ -24,7 +24,7 @@ import typing as t
 import json
 
 from searx.result_types import EngineResults
-from searx.enginelib import EngineCache
+from searx.enginelib import EngineCache, EngineAbout
 
 if t.TYPE_CHECKING:
     from searx.search.processors import RequestParams
@@ -35,13 +35,11 @@ categories = ["general"]
 disabled = True
 timeout = 2.0
 
-about = {
-    "wikidata_id": None,
-    "official_api_documentation": None,
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": "JSON",
-}
+language = "en"
+about = EngineAbout(
+    results="JSON",
+    description="Demo offline engine Engine with results in the English language.",
+)
 
 # if there is a need for globals, use a leading underline
 _my_offline_engine: str = ""

searx/engines/demo_online.py

@@ -25,6 +25,7 @@ import typing as t
 
 from urllib.parse import urlencode
 from searx.result_types import EngineResults
+from searx.enginelib import EngineAbout
 
 if t.TYPE_CHECKING:
     from searx.extended_types import SXNG_Response
@@ -43,14 +44,14 @@ page_size = 20
 search_api = "https://api.artic.edu/api/v1/artworks/search"
 image_api = "https://www.artic.edu/iiif/2/"
 
-about = {
-    "website": "https://www.artic.edu",
-    "wikidata_id": "Q239303",
-    "official_api_documentation": "http://api.artic.edu/docs/",
-    "use_official_api": True,
-    "require_api_key": False,
-    "results": "JSON",
-}
+about = EngineAbout(
+    website="https://www.artic.edu",
+    wikidata_id="Q239303",
+    official_api_documentation="http://api.artic.edu/docs/",
+    use_official_api=True,
+    require_api_key=False,
+    results="JSON",
+)
 
 
 # if there is a need for globals, use a leading underline

searx/engines/destatis.py

@@ -11,8 +11,8 @@ about = {
     'use_official_api': False,
     'require_api_key': False,
     'results': 'HTML',
-    'language': 'de',
 }
+language = "de"
 
 categories = []
 paging = True

searx/engines/duden.py

@@ -14,8 +14,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": 'HTML',
-    "language": 'de',
 }
+language = "de"
 
 categories = ['dictionaries']
 paging = True

searx/engines/elasticsearch.py

@@ -55,7 +55,7 @@ about = {
     'official_api_documentation': 'https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html',
     'use_official_api': True,
     'require_api_key': False,
-    'format': 'JSON',
+    "results": "JSON",
 }
 
 base_url = 'http://localhost:9200'

searx/engines/geizhals.py

@@ -27,8 +27,8 @@ about = {
     'official_api_documentation': None,
     'require_api_key': False,
     'results': 'HTML',
-    'language': 'de',
 }
+language = "de"
 paging = True
 categories = ['shopping']
 

searx/engines/il_post.py

@@ -34,8 +34,8 @@ about = {
     "use_official_api": True,
     "require_api_key": False,
     "results": "JSON",
-    "language": "it",
 }
+language = "it"
 
 
 def request(query, params):

searx/engines/ina.py

@@ -16,8 +16,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": 'HTML',
-    "language": 'fr',
 }
+language = "fr"
 
 # engine dependent config
 categories = ['videos']

searx/engines/iqiyi.py

@@ -14,9 +14,9 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "JSON",
-    "language": "zh",
 }
 
+language = "zh"
 paging = True
 time_range_support = True
 categories = ["videos"]

searx/engines/jisho.py

@@ -13,8 +13,8 @@ about = {
     "use_official_api": True,
     "require_api_key": False,
     "results": 'JSON',
-    "language": 'ja',
 }
+language = "ja"
 
 categories = ['dictionaries']
 paging = False
@@ -110,8 +110,8 @@ def get_infobox(alt_forms, result_url, definitions):
     # definitions
     infobox_content.append(
         '''
-        <small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a> 
-        and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a> 
+        <small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
+        and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
         by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
         <ul>
     '''

searx/engines/json_engine.py

@@ -79,6 +79,9 @@ from json import loads
 from urllib.parse import urlencode
 from searx.utils import to_string, html_to_text
 from searx.network import raise_for_httperror
+from searx.enginelib import EngineAbout
+
+about = EngineAbout()
 
 search_url = None
 """

searx/engines/mediathekviewweb.py

@@ -11,9 +11,9 @@ about = {
     "use_official_api": True,
     "require_api_key": False,
     "results": 'JSON',
-    "language": "de",
 }
 
+language = "de"
 categories = ['videos']
 paging = True
 time_range_support = False

searx/engines/moviepilot.py

@@ -35,8 +35,9 @@ about = {
     'use_official_api': False,
     'require_api_key': False,
     'results': 'JSON',
-    'language': 'de',
 }
+language = "de"
+
 paging = True
 categories = ["movies"]
 

searx/engines/naver.py

@@ -26,8 +26,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "ko",
 }
+language = "ko"
 
 categories = []
 paging = True

searx/engines/niconico.py

@@ -13,8 +13,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "ja",
 }
+language = "ja"
 
 categories = ["videos"]
 paging = True

searx/engines/photon.py

@@ -28,7 +28,7 @@ search_string = 'api/?{query}&limit={limit}'
 result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
 
 # list of supported languages
-supported_languages = ['de', 'en', 'fr', 'it']
+photon_supported_languages = ["de", "en", "fr", "it"]
 
 
 # do search-request
@@ -37,7 +37,7 @@ def request(query, params):
 
     if params['language'] != 'all':
         language = params['language'].split('_')[0]
-        if language in supported_languages:
+        if language in photon_supported_languages:
             params['url'] = params['url'] + "&lang=" + language
 
     # using SearXNG User-Agent

searx/engines/presearch.py

@@ -77,7 +77,7 @@ from searx.utils import gen_useragent, html_to_text, parse_duration_string
 
 about = {
     "website": "https://presearch.io",
-    "wikidiata_id": "Q7240905",
+    "wikidata_id": "Q7240905",
     "official_api_documentation": "https://docs.presearch.io/nodes/api",
     "use_official_api": False,
     "require_api_key": False,

searx/engines/quark.py

@@ -16,8 +16,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "zh",
 }
+language = "zh"
 
 # Engine Configuration
 categories = []

searx/engines/senscritique.py

@@ -13,8 +13,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": 'JSON',
-    'language': 'fr',
 }
+language = "fr"
 
 categories = ['movies']
 paging = True

searx/engines/seznam.py

@@ -19,8 +19,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "cz",
 }
+language = "cz"
 
 categories = ['general', 'web']
 base_url = 'https://search.seznam.cz/'

searx/engines/sogou.py

@@ -16,8 +16,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "zh",
 }
+language = "zh"
 
 # Engine Configuration
 categories = ["general"]

searx/engines/sogou_videos.py

@@ -11,8 +11,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "JSON",
-    "language": "zh",
 }
+language = "zh"
 
 categories = ["videos"]
 paging = True

searx/engines/sogou_wechat.py

@@ -14,8 +14,8 @@ about = {
     "use_official_api": False,
     "require_api_key": False,
     "results": "HTML",
-    "language": "zh",
 }
+language = "zh"
 
 # Engine Configuration
 categories = ["news"]

searx/engines/tagesschau.py

@@ -27,8 +27,9 @@ about = {
     'use_official_api': True,
     'require_api_key': False,
     'results': 'JSON',
-    'language': 'de',
 }
+language = "de"
+
 categories = ['general', 'news']
 paging = True
 

searx/engines/tonline.py

@@ -16,20 +16,17 @@ from lxml import html
 
 from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, ElementType
 from searx.result_types import EngineResults
+from searx.enginelib import EngineAbout
 
 if t.TYPE_CHECKING:
     from searx.extended_types import SXNG_Response
     from searx.search.processors import OnlineParams
 
-about = {
-    "website": "https://www.t-online.de",
-    "wikidata_id": "Q590940",
-    "official_api_documentation": None,
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": "HTML",
-    "language": "de",
-}
+about = EngineAbout(
+    website="https://www.t-online.de",
+    wikidata_id="Q590940",
+    results="HTML",
+)
 
 paging = True
 time_range_support = True
@@ -44,6 +41,8 @@ time_range_map = {"day": "d", "week": "w", "month": "m", "year": "y"}
 # use "tonline" to only search for results from t-online news articles
 tonline_channel_map = {"images": "flickr", "videos": "yt"}
 
+language = "de"
+
 
 def init(_):
     if tonline_categ not in ("web", "images", "videos", "news"):

searx/engines/xpath.py

@@ -76,6 +76,9 @@ from lxml import html
 from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
 from searx.network import raise_for_httperror
 from searx.result_types import EngineResults
+from searx.enginelib import EngineAbout
+
+about = EngineAbout()
 
 search_url = None
 """