Skip to content
Back to top

# Source code for searx.enginelib.traits

    # SPDX-License-Identifier: AGPL-3.0-or-later
    """Engine's traits are fetched from the origin engines and stored in a JSON file
    in the *data folder*.  Most often traits are languages and region codes and
    their mapping from SearXNG's representation to the representation in the origin
    search engine.  For new traits new properties can be added to the class
    :py:class:`EngineTraits`.

    To load traits from the persistence :py:obj:`EngineTraitsMap.from_data` can be
    used.
    """

    import dataclasses
    import json
    import pathlib
    import types
    import typing as t

    from searx import locales
    from searx.data import ENGINE_TRAITS, data_dir

    if t.TYPE_CHECKING:
        from . import Engine

    class EngineTraitsEncoder(json.JSONEncoder):
        """Encodes :class:`EngineTraits` to a serializable object, see
        :class:`json.JSONEncoder`."""

        def default(self, o: t.Any) -> t.Any:
            """Return dictionary of a :class:`EngineTraits` object."""
            if isinstance(o, EngineTraits):
                return o.__dict__
            return super().default(o)

    @dataclasses.dataclass
    class EngineTraits:
        """The class is intended to be instantiated for each engine."""

        regions: dict[str, str] = dataclasses.field(default_factory=dict)
        """Maps SearXNG's internal representation of a region to the one of the engine.

        SearXNG's internal representation can be parsed by babel and the value is
        send to the engine:

        .. code:: python

           regions ={
               'fr-BE' : <engine's region name>,
           }

           for key, egnine_region regions.items():
              searxng_region = babel.Locale.parse(key, sep='-')
              ...
        """

        languages: dict[str, str] = dataclasses.field(default_factory=dict)
        """Maps SearXNG's internal representation of a language to the one of the engine.

        SearXNG's internal representation can be parsed by babel and the value is
        send to the engine:

        .. code:: python

           languages = {
               'ca' : <engine's language name>,
           }

           for key, egnine_lang in languages.items():
              searxng_lang = babel.Locale.parse(key)
              ...
        """

        all_locale: str | None = None
        """To which locale value SearXNG's ``all`` language is mapped (shown a "Default
        language").
        """

        data_type: t.Literal["traits_v1"] = "traits_v1"
        """Data type, default is 'traits_v1'.
        """

        custom: dict[str, t.Any] = dataclasses.field(default_factory=dict)
        """A place to store engine's custom traits, not related to the SearXNG core.
        """

        def get_language(self, searxng_locale: str, default: str | None = None) -> str | None:
            """Return engine's language string that *best fits* to SearXNG's locale.

            :param searxng_locale: SearXNG's internal representation of locale
              selected by the user.

            :param default: engine's default language

            The *best fits* rules are implemented in
            :py:obj:`searx.locales.get_engine_locale`.  Except for the special value ``all``
            which is determined from :py:obj:`EngineTraits.all_locale`.
            """
            if searxng_locale == "all" and self.all_locale is not None:
                return self.all_locale
            return locales.get_engine_locale(searxng_locale, self.languages, default=default)

        def get_region(self, searxng_locale: str, default: str | None = None) -> str | None:
            """Return engine's region string that best fits to SearXNG's locale.

            :param searxng_locale: SearXNG's internal representation of locale
              selected by the user.

            :param default: engine's default region

            The *best fits* rules are implemented in
            :py:obj:`searx.locales.get_engine_locale`.  Except for the special value ``all``
            which is determined from :py:obj:`EngineTraits.all_locale`.
            """
            if searxng_locale == "all" and self.all_locale is not None:
                return self.all_locale
            return locales.get_engine_locale(searxng_locale, self.regions, default=default)

        def copy(self):
            """Create a copy of the dataclass object."""
            return EngineTraits(**dataclasses.asdict(self))

        @classmethod
        def fetch_traits(cls, engine: "Engine | types.ModuleType") -> "EngineTraits | None":
            """Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
            and set properties from the origin engine in the object ``engine_traits``.  If
            function does not exists, ``None`` is returned.
            """

            fetch_traits = getattr(engine, "fetch_traits", None)
            engine_traits = None

            if fetch_traits:
                engine_traits = cls()
                fetch_traits(engine_traits)
            return engine_traits

        def set_traits(self, engine: "Engine | types.ModuleType") -> None:
            """Set traits from self object in a :py:obj:`.Engine` namespace.

            :param engine: engine instance build by :py:func:`searx.engines.load_engine`
            """

            if self.data_type == "traits_v1":
                self._set_traits_v1(engine)  # pyright: ignore[reportArgumentType]
            else:
                raise TypeError("engine traits of type %s is unknown" % self.data_type)

        def _set_traits_v1(self, engine: "Engine") -> None:
            # For an engine, when there is `language: ...` in the YAML settings the engine
            # does support only this one language (region)::
            #
            #   - name: google italian
            #     engine: google
            #     language: it
            #     region: it-IT

            traits = self.copy()

            _msg = "settings.yml - engine: '%s' / %s: '%s' not supported"

            if engine.language:
                if engine.language_support:
                    if not len(traits.languages) > 1:
                        raise ValueError(
                            f"engine {engine.name}: activated language_support with just one or less languages"
                        )
                    if engine.language not in traits.languages:
                        raise ValueError(_msg % (engine.name, "language", engine.language))
                    traits.languages = {engine.language: traits.languages[engine.language]}

            if engine.region:
                if engine.region not in traits.regions:
                    raise ValueError(_msg % (engine.name, "region", engine.region))
                traits.regions = {engine.region: traits.regions[engine.region]}

            engine.language_support = bool(traits.languages or traits.regions)

            # set the copied & modified traits in engine's namespace
            engine.traits = traits

    class EngineTraitsMap(dict[str, EngineTraits]):
        """A python dictionary to map :class:`EngineTraits` by engine name."""

        ENGINE_TRAITS_FILE: pathlib.Path = (data_dir / "engine_traits.json").resolve()
        """File with persistence of the :py:obj:`EngineTraitsMap`."""

        def save_data(self):
            """Store EngineTraitsMap in in file :py:obj:`self.ENGINE_TRAITS_FILE`"""
            with open(self.ENGINE_TRAITS_FILE, "w", encoding="utf-8") as f:
                json.dump(self, f, indent=2, sort_keys=True, cls=EngineTraitsEncoder)

        @classmethod
        def from_data(cls) -> "EngineTraitsMap":
            """Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""
            obj = cls()
            for k, v in ENGINE_TRAITS.items():
                obj[k] = EngineTraits(**v)
            return obj

        @classmethod
        def fetch_traits(cls, log: t.Callable[[str], None]) -> "EngineTraitsMap":
            from searx import (  # pylint: disable=cyclic-import, import-outside-toplevel
                engines,
            )

            names = list(engines.engines)
            names.sort()
            obj = cls()

            for engine_name in names:
                engine: Engine | types.ModuleType = engines.engines[engine_name]
                traits = None

                # pylint: disable=broad-exception-caught
                try:
                    traits = EngineTraits.fetch_traits(engine)
                except Exception as exc:
                    log("ERROR: while fetch_traits %s: %s" % (engine_name, exc))
                    v = ENGINE_TRAITS.get(engine_name)
                    if v:
                        log("WARNING: re-use old values from fetch_traits - ENGINE_TRAITS[%s]" % engine_name)
                        traits = EngineTraits(**v)
                    else:
                        log("WARNING: no old values available for ENGINE_TRAITS[%s], skipping" % engine_name)

                if traits is not None:
                    log("%-20s: SearXNG languages --> %s " % (engine_name, len(traits.languages)))
                    log("%-20s: SearXNG regions   --> %s" % (engine_name, len(traits.regions)))
                    obj[engine_name] = traits

            return obj

        def set_traits(self, engine: "Engine | types.ModuleType"):
            """Set traits in a :py:obj:`Engine` namespace.

            :param engine: engine instance build by :py:func:`searx.engines.load_engine`
            """

            engine_traits = EngineTraits(data_type="traits_v1")
            if engine.name in self.keys():
                engine_traits = self[engine.name]

            elif engine.engine in self.keys():
                # The key of the dictionary traits_map is the *engine name*
                # configured in settings.xml.  When multiple engines are configured
                # in settings.yml to use the same origin engine (python module)
                # these additional engines can use the languages from the origin
                # engine.  For this use the configured ``engine: ...`` from
                # settings.yml
                engine_traits = self[engine.engine]

            engine_traits.set_traits(engine)