[fix] fireball: all results are shown in general category

Add support for the Dogpile search engine, found at:

https://seirdy.one/posts/2021/03/10/search-engines-with-own-indexes/

It seems to use the same index as startpage because results are similar and they
share the ``qadf`` (Safe-Search) request parameter.

searx/engines/dogpile.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Dogpile is a metasearch engine by the American advertising company `System1`_.
+
+.. _System1: https://system1.com/
+"""
+
+import typing as t
+from datetime import datetime, timezone
+import html
+
+from searx.utils import format_duration, html_to_text, humanize_number
+from searx.result_types import EngineResults
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
+about = {
+    "website": "https://www.dogpile.com",
+    "wikidata_id": "Q3595363",
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+paging = True
+safesearch = True
+
+categories = ["general"]
+dogpile_categ = "search"
+"""Category to search in. Can be either "search", "images", "videos" or "news"."""
+
+
+base_url = "https://www.dogpile.com"
+safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
+
+
+def init(_):
+    if dogpile_categ not in ("search", "images", "videos", "news"):
+        raise ValueError("invalid search type: %s" % dogpile_categ)
+
+
+def request(query: str, params: "OnlineParams"):
+    params["url"] = f"{base_url}/api/{dogpile_categ}"
+    params["method"] = "POST"
+    params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}
+    return params
+
+
+def response(resp: "SXNG_Response"):
+    res = EngineResults()
+
+    json_resp = resp.json()
+
+    for result in json_resp["results"]:
+        if dogpile_categ == "search":
+            res.add(
+                res.types.MainResult(
+                    url=result["clickUrl"],
+                    title=html_to_text(result["title"]),
+                    content=html_to_text(result["description"]),
+                )
+            )
+        elif dogpile_categ == "news":
+            res.add(
+                res.types.MainResult(
+                    url=result["clickUrl"],
+                    title=html_to_text(html.unescape(result["title"])),
+                    content=html_to_text(html.unescape(result["description"])),
+                    thumbnail=result["thumbnailUrl"],
+                    publishedDate=datetime.fromtimestamp(result["date"], tz=timezone.utc),
+                )
+            )
+        elif dogpile_categ == "videos":
+            res.add(
+                res.types.LegacyResult(
+                    template="videos.html",
+                    url=result["clickUrl"],
+                    title=html_to_text(result["title"]),
+                    content=html_to_text(result["description"]),
+                    thumbnail=result["thumbnailUrl"],
+                    publishedDate=datetime.fromisoformat(result["publishDate"]),
+                    length=format_duration(result["duration"]),
+                    views=humanize_number(result["viewCount"]),
+                )
+            )
+        elif dogpile_categ == "images":
+            res.add(
+                res.types.Image(
+                    url=result["altClickUrl"],
+                    title=html_to_text(result["title"]),
+                    content=html_to_text(result["description"]),
+                    img_src=result["clickUrl"],
+                    thumbnail_src=result["thumbnailUrl"],
+                    resolution=f"{result['width']}x{result['height']}",
+                    img_format=result["format"],
+                )
+            )
+
+    return res

searx/engines/odysee.py

@@ -4,7 +4,6 @@
 .. _Odysee: https://github.com/OdyseeTeam/odysee-frontend
 """
 
-import time
 from datetime import datetime
 from urllib.parse import urlencode
 
@@ -12,6 +11,7 @@ import babel
 
 from searx.enginelib.traits import EngineTraits
 from searx.locales import language_tag
+from searx.utils import format_duration
 
 # Engine metadata
 about = {
@@ -61,15 +61,6 @@ def request(query, params):
     return params
 
 
-# Format the video duration
-def format_duration(duration):
-    seconds = int(duration)
-    length = time.gmtime(seconds)
-    if length.tm_hour:
-        return time.strftime("%H:%M:%S", length)
-    return time.strftime("%M:%S", length)
-
-
 def response(resp):
     data = resp.json()
     results = []

searx/settings.yml

@@ -803,6 +803,34 @@ engines:
     display_type: ["infobox"]
     categories: [general]
 
+  - name: dogpile
+    engine: dogpile
+    shortcut: dog
+    dogpile_categ: search
+    categories: general
+    disabled: true
+
+  - name: dogpile images
+    engine: dogpile
+    shortcut: dogi
+    dogpile_categ: images
+    categories: images
+    disabled: true
+
+  - name: dogpile videos
+    engine: dogpile
+    shortcut: dogv
+    dogpile_categ: videos
+    categories: videos
+    disabled: true
+
+  - name: dogpile news
+    engine: dogpile
+    shortcut: dogn
+    dogpile_categ: news
+    categories: news
+    disabled: true
+
   # duckduckgo uses html.duckduckgo.com,
   # duckduckgo web uses duckduckgo.com
   - name: duckduckgo
@@ -899,21 +927,21 @@ engines:
   - name: fireball
     engine: fireball
     shortcut: fire
-    category: general
+    categories: general
     fireball_category: web
     disabled: true
 
   - name: fireball news
     engine: fireball
     shortcut: firen
-    category: news
+    categories: news
     fireball_category: news
     disabled: true
 
   - name: fireball videos
     engine: fireball
     shortcut: firev
-    category: videos
+    categories: videos
     fireball_category: videos
     disabled: true
 

searx/utils.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Utility functions for the engines"""
 
+import time
 
 import re
 import importlib
@@ -809,3 +810,12 @@ def parse_duration_string(duration_str: str) -> timedelta | None:
         pass
 
     return None
+
+
+# Format the video duration
+def format_duration(duration: str | int) -> str:
+    seconds = int(duration)
+    length = time.gmtime(seconds)
+    if length.tm_hour:
+        return time.strftime("%H:%M:%S", length)
+    return time.strftime("%M:%S", length)