[mod] addition of various type hints / engine processors

Continuation of #5147 .. typification of the engine processors.

BTW:

- removed obsolete engine property https_support
- fixed & improved currency_convert
- engine instances can now implement a engine.setup method

[#5147] https://github.com/searxng/searxng/pull/5147

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/network/__init__.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # pylint: disable=missing-module-docstring, global-statement
 
-__all__ = ["initialize", "check_network_configuration", "raise_for_httperror"]
+__all__ = ["get_network", "initialize", "check_network_configuration", "raise_for_httperror"]
 
 import typing as t
 
@@ -22,6 +22,8 @@ from .network import get_network, initialize, check_network_configuration  # pyl
 from .client import get_loop
 from .raise_for_httperror import raise_for_httperror
 
+if t.TYPE_CHECKING:
+    from searx.network.network import Network
 
 THREADLOCAL = threading.local()
 """Thread-local data is data for thread specific values."""
@@ -31,7 +33,7 @@ def reset_time_for_thread():
     THREADLOCAL.total_time = 0
 
 
-def get_time_for_thread():
+def get_time_for_thread() -> float | None:
     """returns thread's total time or None"""
     return THREADLOCAL.__dict__.get('total_time')
 
@@ -45,7 +47,7 @@ def set_context_network_name(network_name: str):
     THREADLOCAL.network = get_network(network_name)
 
 
-def get_context_network():
+def get_context_network() -> "Network":
     """If set return thread's network.
 
     If unset, return value from :py:obj:`get_network`.
@@ -68,7 +70,7 @@ def _record_http_time():
             THREADLOCAL.total_time += time_after_request - time_before_request
 
 
-def _get_timeout(start_time: float, kwargs):
+def _get_timeout(start_time: float, kwargs: t.Any) -> float:
     # pylint: disable=too-many-branches
 
     timeout: float | None
@@ -91,7 +93,7 @@ def _get_timeout(start_time: float, kwargs):
     return timeout
 
 
-def request(method, url, **kwargs) -> SXNG_Response:
+def request(method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
     """same as requests/requests/api.py request(...)"""
     with _record_http_time() as start_time:
         network = get_context_network()
@@ -183,15 +185,15 @@ def head(url: str, **kwargs: t.Any) -> SXNG_Response:
     return request('head', url, **kwargs)
 
 
-def post(url: str, data=None, **kwargs: t.Any) -> SXNG_Response:
+def post(url: str, data: dict[str, t.Any] | None = None, **kwargs: t.Any) -> SXNG_Response:
     return request('post', url, data=data, **kwargs)
 
 
-def put(url: str, data=None, **kwargs: t.Any) -> SXNG_Response:
+def put(url: str, data: dict[str, t.Any] | None = None, **kwargs: t.Any) -> SXNG_Response:
     return request('put', url, data=data, **kwargs)
 
 
-def patch(url: str, data=None, **kwargs: t.Any) -> SXNG_Response:
+def patch(url: str, data: dict[str, t.Any] | None = None, **kwargs: t.Any) -> SXNG_Response:
     return request('patch', url, data=data, **kwargs)
 
 
@@ -250,7 +252,7 @@ def _close_response_method(self):
         continue
 
 
-def stream(method: str, url: str, **kwargs: t.Any) -> tuple[httpx.Response, Iterable[bytes]]:
+def stream(method: str, url: str, **kwargs: t.Any) -> tuple[SXNG_Response, Iterable[bytes]]:
     """Replace httpx.stream.
 
     Usage:

searx/network/client.py

@@ -138,7 +138,7 @@ def get_transport_for_socks_proxy(
         password=proxy_password,
         rdns=rdns,
         loop=get_loop(),
-        verify=_verify,
+        verify=_verify,  # pyright: ignore[reportArgumentType]
         http2=http2,
         local_address=local_address,
         limits=limit,

searx/network/network.py

@@ -1,8 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # pylint: disable=global-statement
 # pylint: disable=missing-module-docstring, missing-class-docstring
+
+__all__ = ["get_network"]
+
 import typing as t
-from collections.abc import Generator, AsyncIterator
+from collections.abc import Generator
+
 
 import atexit
 import asyncio
@@ -74,7 +78,7 @@ class Network:
         using_tor_proxy: bool = False,
         local_addresses: str | list[str] | None = None,
         retries: int = 0,
-        retry_on_http_error: None = None,
+        retry_on_http_error: bool = False,
         max_redirects: int = 30,
         logger_name: str = None,  # pyright: ignore[reportArgumentType]
     ):
@@ -232,14 +236,14 @@ class Network:
         return kwargs_clients
 
     @staticmethod
-    def extract_do_raise_for_httperror(kwargs):
+    def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
         do_raise_for_httperror = True
         if 'raise_for_httperror' in kwargs:
             do_raise_for_httperror = kwargs['raise_for_httperror']
             del kwargs['raise_for_httperror']
         return do_raise_for_httperror
 
-    def patch_response(self, response: httpx.Response | SXNG_Response, do_raise_for_httperror: bool) -> SXNG_Response:
+    def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
         if isinstance(response, httpx.Response):
             response = t.cast(SXNG_Response, response)
             # requests compatibility (response is not streamed)
@@ -255,7 +259,7 @@ class Network:
                     raise
         return response
 
-    def is_valid_response(self, response: SXNG_Response):
+    def is_valid_response(self, response: httpx.Response):
         # pylint: disable=too-many-boolean-expressions
         if (
             (self.retry_on_http_error is True and 400 <= response.status_code <= 599)
@@ -265,9 +269,7 @@ class Network:
             return False
         return True
 
-    async def call_client(
-        self, stream: bool, method: str, url: str, **kwargs: t.Any
-    ) -> AsyncIterator[SXNG_Response] | None:
+    async def call_client(self, stream: bool, method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
         retries = self.retries
         was_disconnected = False
         do_raise_for_httperror = Network.extract_do_raise_for_httperror(kwargs)
@@ -278,9 +280,9 @@ class Network:
             client.cookies = httpx.Cookies(cookies)
             try:
                 if stream:
-                    response = client.stream(method, url, **kwargs)  # pyright: ignore[reportAny]
+                    response = client.stream(method, url, **kwargs)
                 else:
-                    response = await client.request(method, url, **kwargs)  # pyright: ignore[reportAny]
+                    response = await client.request(method, url, **kwargs)
                 if self.is_valid_response(response) or retries <= 0:
                     return self.patch_response(response, do_raise_for_httperror)
             except httpx.RemoteProtocolError as e:
@@ -298,7 +300,7 @@ class Network:
                     raise e
             retries -= 1
 
-    async def request(self, method: str, url: str, **kwargs):
+    async def request(self, method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
         return await self.call_client(False, method, url, **kwargs)
 
     async def stream(self, method: str, url: str, **kwargs):
@@ -358,7 +360,7 @@ def initialize(
         'proxies': settings_outgoing['proxies'],
         'max_redirects': settings_outgoing['max_redirects'],
         'retries': settings_outgoing['retries'],
-        'retry_on_http_error': None,
+        'retry_on_http_error': False,
     }
 
     def new_network(params: dict[str, t.Any], logger_name: str | None = None):