[mod] update_engine_traits.py: improve type annotations

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
2026-05-12 15:54:04 +02:00
committed by Markus Heiser
parent 471f2b205f
commit cf4d7e31c4
+21 -21
View File
@@ -1,11 +1,11 @@
#!/usr/bin/env python #!/usr/bin/env python
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
"""Update :py:obj:`searx.enginelib.traits.EngineTraitsMap` and :origin:`searx/languages.py` """Update :py:obj:`searx.enginelib.traits.EngineTraitsMap` and :origin:`searx/sxng_locales.py`
:py:obj:`searx.enginelib.traits.EngineTraitsMap.ENGINE_TRAITS_FILE`: :py:obj:`searx.enginelib.traits.EngineTraitsMap.ENGINE_TRAITS_FILE`:
Persistence of engines traits, fetched from the engines. Persistence of engines traits, fetched from the engines.
:origin:`searx/languages.py` :origin:`searx/sxng_locales.py`
Is generated from intersecting each engine's supported traits. Is generated from intersecting each engine's supported traits.
The script :origin:`searxng_extra/update/update_engine_traits.py` is called in The script :origin:`searxng_extra/update/update_engine_traits.py` is called in
@@ -28,8 +28,8 @@ from searx.engines import load_engines
from searx.enginelib.traits import EngineTraitsMap from searx.enginelib.traits import EngineTraitsMap
# Output files. # Output files.
languages_file = Path(searx_dir) / 'sxng_locales.py' sxng_locales_file = Path(searx_dir) / 'sxng_locales.py'
languages_file_header = """\ sxng_locales_file_header = """\
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
'''List of SearXNG's locale codes used for the search language/region. '''List of SearXNG's locale codes used for the search language/region.
@@ -42,7 +42,7 @@ languages_file_header = """\
sxng_locales = ( sxng_locales = (
""" """
languages_file_footer = """, sxng_locales_file_footer = """,
) )
''' '''
A list of five-digit tuples: A list of five-digit tuples:
@@ -83,16 +83,16 @@ app = typer.Typer()
@app.command() @app.command()
def cli(engines: t.Annotated[list[str] | None, typer.Argument()] = None): def cli(engines: t.Annotated[list[str] | None, typer.Argument()] = None):
"""Update ``data/engine_traits.json`` and ``languages.py``. """Update ``data/engine_traits.json`` and ``sxng_locales.py``.
Optionally, if arguments are provided via the command line, these are Optionally, if arguments are provided via the command line, these are
interpreted as the names of the engines that should be updated. All other interpreted as the names of the engines that should be updated. All other
engines will be left untouched. engines will be left untouched.
""" """
all_eng_names: list[str] = [_["name"] for _ in settings["engines"]] all_eng_names: list[str] = [e["name"] for e in settings["engines"]]
if engines: if engines:
unknown: list[str] = [_ for _ in engines if _ not in all_eng_names] unknown: list[str] = [e for e in engines if e not in all_eng_names]
if unknown: if unknown:
print(f"ERROR: unknown engines --> {', '.join(unknown)}") print(f"ERROR: unknown engines --> {', '.join(unknown)}")
raise typer.Exit(42) raise typer.Exit(42)
@@ -113,14 +113,14 @@ def cli(engines: t.Annotated[list[str] | None, typer.Argument()] = None):
print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE) print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
traits_map.save_data() traits_map.save_data()
sxng_tag_list = filter_locales(traits_map) sxng_tag_list = filter_locales(traits_map)
write_languages_file(sxng_tag_list) write_sxng_locales_file(sxng_tag_list)
def fetch_traits_map() -> EngineTraitsMap: def fetch_traits_map() -> EngineTraitsMap:
"""Fetches supported languages for each engine and writes json file with those.""" """Fetches supported languages for each engine and writes json file with those."""
network.set_timeout_for_thread(10.0) network.set_timeout_for_thread(10.0)
def log(msg): def log(msg: str):
print(msg) print(msg)
traits_map = EngineTraitsMap.fetch_traits(log=log) traits_map = EngineTraitsMap.fetch_traits(log=log)
@@ -128,13 +128,13 @@ def fetch_traits_map() -> EngineTraitsMap:
return traits_map return traits_map
def filter_locales(traits_map: EngineTraitsMap): def filter_locales(traits_map: EngineTraitsMap) -> set[str]:
"""Filter language & region tags by a threshold.""" """Filter language & region tags by a threshold."""
min_eng_per_region = 18 min_eng_per_region = 18
min_eng_per_lang = 22 min_eng_per_lang = 22
_ = {} _: dict[str, int] = {}
for eng in traits_map.values(): for eng in traits_map.values():
for reg in eng.regions.keys(): for reg in eng.regions.keys():
_[reg] = _.get(reg, 0) + 1 _[reg] = _.get(reg, 0) + 1
@@ -154,7 +154,7 @@ def filter_locales(traits_map: EngineTraitsMap):
languages = set(k for k, v in _.items() if v >= min_eng_per_lang) languages = set(k for k, v in _.items() if v >= min_eng_per_lang)
sxng_tag_list = set() sxng_tag_list: set[str] = set()
sxng_tag_list.update(regions) sxng_tag_list.update(regions)
sxng_tag_list.update(lang_from_region) sxng_tag_list.update(lang_from_region)
sxng_tag_list.update(languages) sxng_tag_list.update(languages)
@@ -162,9 +162,9 @@ def filter_locales(traits_map: EngineTraitsMap):
return sxng_tag_list return sxng_tag_list
def write_languages_file(sxng_tag_list): def write_sxng_locales_file(sxng_tag_list: set[str]):
language_codes = [] language_codes: list[tuple[str, str, str, str, str]] = []
for sxng_tag in sorted(sxng_tag_list): for sxng_tag in sorted(sxng_tag_list):
sxng_locale: babel.Locale = babel.Locale.parse(sxng_tag, sep='-') sxng_locale: babel.Locale = babel.Locale.parse(sxng_tag, sep='-')
@@ -173,7 +173,7 @@ def write_languages_file(sxng_tag_list):
item = ( item = (
sxng_tag, sxng_tag,
sxng_locale.get_language_name().title(), # type: ignore sxng_locale.get_language_name().title(), # pyright: ignore[reportOptionalMemberAccess]
sxng_locale.get_territory_name() or '', sxng_locale.get_territory_name() or '',
sxng_locale.english_name.split(' (')[0] if sxng_locale.english_name else '', sxng_locale.english_name.split(' (')[0] if sxng_locale.english_name else '',
UnicodeEscape(flag), UnicodeEscape(flag),
@@ -181,13 +181,13 @@ def write_languages_file(sxng_tag_list):
language_codes.append(item) language_codes.append(item)
language_codes = tuple(language_codes) _codes = tuple(language_codes)
with languages_file.open('w', encoding='utf-8') as new_file: with sxng_locales_file.open('w', encoding='utf-8') as new_file:
file_content = "{header} {language_codes}{footer}".format( file_content = "{header} {language_codes}{footer}".format(
header=languages_file_header, header=sxng_locales_file_header,
language_codes=pformat(language_codes, width=120, indent=4)[1:-1], language_codes=pformat(_codes, width=120, indent=4)[1:-1],
footer=languages_file_footer, footer=sxng_locales_file_footer,
) )
new_file.write(file_content) new_file.write(file_content)
new_file.close() new_file.close()