[feat] update_engine_traits.py: add option to update engines selective

Previously, `update_engine_traits.py` would fetch traits for all engines, which
is very slow and by side-effect touches engine data that are unrelated to the
engine you're currently working on.

To be faster with developing `update_engine_traits.py` supports now engine
arguments.

To test, jump into the developer environment and run the script::

    $ ./manage dev.env
    (dev.env)$ ./searxng_extra/update/update_engine_traits.py --help

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
2026-05-12 15:16:52 +02:00
committed by Markus Heiser
parent 09829b1ccc
commit 471f2b205f
+34 -11
View File
@@ -12,12 +12,15 @@ The script :origin:`searxng_extra/update/update_engine_traits.py` is called in
the :origin:`CI Update data ... <.github/workflows/data-update.yml>` the :origin:`CI Update data ... <.github/workflows/data-update.yml>`
""" """
# pylint: disable=invalid-name # pylint: disable=invalid-name
import typing as t
from unicodedata import lookup from unicodedata import lookup
from pathlib import Path from pathlib import Path
from pprint import pformat from pprint import pformat
import babel import babel
import typer
from searx import settings, searx_dir from searx import settings, searx_dir
from searx import network from searx import network
@@ -75,23 +78,45 @@ lang2emoji = {
'he': '\U0001f1ee\U0001f1f1', # Hebrew 'he': '\U0001f1ee\U0001f1f1', # Hebrew
} }
app = typer.Typer()
def main():
engines_cfg = [] @app.command()
def cli(engines: t.Annotated[list[str] | None, typer.Argument()] = None):
"""Update ``data/engine_traits.json`` and ``languages.py``.
Optionally, if arguments are provided via the command line, these are
interpreted as the names of the engines that should be updated. All other
engines will be left untouched.
"""
all_eng_names: list[str] = [_["name"] for _ in settings["engines"]]
if engines:
unknown: list[str] = [_ for _ in engines if _ not in all_eng_names]
if unknown:
print(f"ERROR: unknown engines --> {', '.join(unknown)}")
raise typer.Exit(42)
engines_cfg: list[dict[str, t.Any]] = []
for eng_data in settings["engines"]: for eng_data in settings["engines"]:
eng_data["inactive"] = False if not engines or eng_data["name"] in engines:
engines_cfg.append(eng_data) eng_data["inactive"] = False
engines_cfg.append(eng_data)
load_engines(engines_cfg) load_engines(engines_cfg)
# traits_map = EngineTraitsMap.from_data() traits_map: EngineTraitsMap = fetch_traits_map()
traits_map = fetch_traits_map() if engines:
_map = EngineTraitsMap.from_data()
_map.update(traits_map)
traits_map = _map
print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
traits_map.save_data()
sxng_tag_list = filter_locales(traits_map) sxng_tag_list = filter_locales(traits_map)
write_languages_file(sxng_tag_list) write_languages_file(sxng_tag_list)
def fetch_traits_map(): def fetch_traits_map() -> EngineTraitsMap:
"""Fetches supported languages for each engine and writes json file with those.""" """Fetches supported languages for each engine and writes json file with those."""
network.set_timeout_for_thread(10.0) network.set_timeout_for_thread(10.0)
@@ -100,8 +125,6 @@ def fetch_traits_map():
traits_map = EngineTraitsMap.fetch_traits(log=log) traits_map = EngineTraitsMap.fetch_traits(log=log)
print("fetched properties from %s engines" % len(traits_map)) print("fetched properties from %s engines" % len(traits_map))
print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
traits_map.save_data()
return traits_map return traits_map
@@ -203,4 +226,4 @@ def get_unicode_flag(locale: babel.Locale):
if __name__ == "__main__": if __name__ == "__main__":
main() app()