mirror of
https://github.com/searxng/searxng.git
synced 2026-06-22 09:38:34 +02:00
Compare commits
4 Commits
5c38d2feab
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 75c1b1dade | |||
| 097ab64c70 | |||
| 0e9f513efc | |||
| fd42d4fda1 |
Generated
+12
-35
@@ -20,7 +20,7 @@
|
||||
"browserslist": "^4.28.2",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.6.4",
|
||||
"less": "^4.6.6",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.1",
|
||||
"sort-package-json": "^4.0.0",
|
||||
@@ -2890,9 +2890,9 @@
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/less": {
|
||||
"version": "4.6.4",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-4.6.4.tgz",
|
||||
"integrity": "sha512-OJmO5+HxZLLw0RLzkqaNHzcgEAQG7C0y3aMbwtCzIUFZsLMNNq/1IdAdHEycQ58CwUO3jPTHmoN+tE5I7FQxNg==",
|
||||
"version": "4.6.6",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-4.6.6.tgz",
|
||||
"integrity": "sha512-ooPSwQGQ2sVe8Dh1jVsbKKsRR2gd8lFK72BDkeSzjnD1T5aIHL65hCMfO0GVmtriKgDKrQv6xp9UrihUsWuAzA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -2909,7 +2909,7 @@
|
||||
"errno": "^0.1.1",
|
||||
"graceful-fs": "^4.1.2",
|
||||
"image-size": "~0.5.0",
|
||||
"make-dir": "^2.1.0",
|
||||
"make-dir": "^5.1.0",
|
||||
"mime": "^1.4.1",
|
||||
"needle": "^3.1.0",
|
||||
"source-map": "~0.6.0"
|
||||
@@ -3191,18 +3191,17 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/make-dir": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
|
||||
"integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-5.1.0.tgz",
|
||||
"integrity": "sha512-IfpFq6UM39dUNiphpA6uDezNx/AvWyhwfICWPR3t1VspkgkMZrL+Rk1RbN1bx+aeNYwOrqGJgEgV3yotk+ZUVw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"pify": "^4.0.1",
|
||||
"semver": "^5.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/mathjs": {
|
||||
@@ -3491,17 +3490,6 @@
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/pify": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
|
||||
"integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/pluralize": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",
|
||||
@@ -3861,17 +3849,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "5.7.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
|
||||
"integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"optional": true,
|
||||
"bin": {
|
||||
"semver": "bin/semver"
|
||||
}
|
||||
},
|
||||
"node_modules/sharp": {
|
||||
"version": "0.35.1",
|
||||
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.1.tgz",
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"browserslist": "^4.28.2",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.6.4",
|
||||
"less": "^4.6.6",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.1",
|
||||
"sort-package-json": "^4.0.0",
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
.. _aol engine:
|
||||
|
||||
===
|
||||
AOL
|
||||
===
|
||||
|
||||
.. automodule:: searx.engines.aol
|
||||
:members:
|
||||
@@ -2,16 +2,16 @@ mock==5.2.0
|
||||
nose2[coverage_plugin]==0.16.0
|
||||
cov-core==1.15.0
|
||||
black==25.9.0
|
||||
pylint==4.0.5
|
||||
pylint==4.0.6
|
||||
splinter==0.21.0
|
||||
selenium==4.44.0
|
||||
selenium==4.45.0
|
||||
Sphinx==8.2.3;python_version <= "3.11"
|
||||
Sphinx==9.1.0; python_version > "3.11"
|
||||
sphinx-issues==6.0.0
|
||||
sphinx-jinja==2.0.2
|
||||
sphinx-tabs==3.5.0
|
||||
furo==2025.12.19
|
||||
sphinxcontrib-programoutput==0.19
|
||||
sphinxcontrib-programoutput==0.20
|
||||
sphinx-autobuild==2025.8.25
|
||||
sphinx-notfound-page==1.1.0
|
||||
myst-parser==5.0.0
|
||||
@@ -24,5 +24,5 @@ docutils>=0.21.2;python_version <= "3.11"
|
||||
docutils>=0.22.4; python_version > "3.11"
|
||||
parameterized==0.9.0
|
||||
granian[reload]==2.7.6
|
||||
basedpyright==1.39.7
|
||||
basedpyright==1.39.8
|
||||
types-lxml==2026.2.16
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
certifi==2026.5.20
|
||||
certifi==2026.6.17
|
||||
babel==2.18.0
|
||||
flask-babel==4.0.0
|
||||
flask==3.1.3
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""AOL supports WEB, image, and video search. Internally, it uses the Bing
|
||||
index.
|
||||
|
||||
AOL doesn't seem to support setting the language via request parameters, instead
|
||||
the results are based on the URL. For example, there is
|
||||
|
||||
- `search.aol.com <https://search.aol.com>`_ for English results
|
||||
- `suche.aol.de <https://suche.aol.de>`_ for German results
|
||||
|
||||
However, AOL offers its services only in a few regions:
|
||||
|
||||
- en-US: search.aol.com
|
||||
- de-DE: suche.aol.de
|
||||
- fr-FR: recherche.aol.fr
|
||||
- en-GB: search.aol.co.uk
|
||||
- en-CA: search.aol.ca
|
||||
|
||||
In order to still offer sufficient support for language and region, the `search
|
||||
keywords`_ known from Bing, ``language`` and ``loc`` (region), are added to the
|
||||
search term (AOL is basically just a proxy for Bing).
|
||||
|
||||
.. _search keywords:
|
||||
https://support.microsoft.com/en-us/topic/advanced-search-keywords-ea595928-5d63-4a0b-9c6b-0b769865e78a
|
||||
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode, unquote_plus
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
from dateutil import parser
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath_list, eval_xpath, extract_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://www.aol.com",
|
||||
"wikidata_id": "Q27585",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
categories = ["general"]
|
||||
search_type = "search" # supported: search, image, video
|
||||
|
||||
paging = True
|
||||
safesearch = True
|
||||
time_range_support = True
|
||||
results_per_page = 10
|
||||
|
||||
|
||||
base_url = "https://search.aol.com"
|
||||
time_range_map = {"day": "1d", "week": "1w", "month": "1m", "year": "1y"}
|
||||
safesearch_map = {0: "p", 1: "r", 2: "i"}
|
||||
|
||||
enable_http2 = False
|
||||
|
||||
|
||||
def init(_):
|
||||
if search_type not in ("search", "image", "video"):
|
||||
raise ValueError(f"unsupported search type {search_type}")
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
language, region = (params["searxng_locale"].split("-") + [None])[:2]
|
||||
if language and language != "all":
|
||||
query = f"{query} language:{language}"
|
||||
if region:
|
||||
query = f"{query} loc:{region}"
|
||||
|
||||
args: dict[str, str | int | None] = {
|
||||
"q": query,
|
||||
"b": params["pageno"] * results_per_page + 1, # page is 1-indexed
|
||||
"pz": results_per_page,
|
||||
}
|
||||
|
||||
if params["time_range"]:
|
||||
args["fr2"] = "time"
|
||||
args["age"] = params["time_range"]
|
||||
else:
|
||||
args["fr2"] = "sb-top-search"
|
||||
|
||||
params["cookies"]["sB"] = f"vm={safesearch_map[params['safesearch']]}"
|
||||
params["url"] = f"{base_url}/aol/{search_type}?{urlencode(args)}"
|
||||
logger.debug(params)
|
||||
|
||||
|
||||
def _deobfuscate_url(obfuscated_url: str) -> str | None:
|
||||
# URL looks like "https://search.aol.com/click/_ylt=AwjFSDjd;_ylu=JfsdjDFd/RV=2/RE=1774058166/RO=10/RU=https%3a%2f%2fen.wikipedia.org%2fwiki%2fTree/RK=0/RS=BP2CqeMLjscg4n8cTmuddlEQA2I-" # pylint: disable=line-too-long
|
||||
if not obfuscated_url:
|
||||
return None
|
||||
|
||||
for part in obfuscated_url.split("/"):
|
||||
if part.startswith("RU="):
|
||||
return unquote_plus(part[3:])
|
||||
# pattern for de-obfuscating URL not found, fall back to Yahoo's tracking link
|
||||
return obfuscated_url
|
||||
|
||||
|
||||
def _general_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//div[@id='web']//ol/li[not(contains(@class, 'first'))]"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, ".//h3/a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=url,
|
||||
title=extract_text(eval_xpath(result, ".//h3/a")) or "",
|
||||
content=extract_text(eval_xpath(result, ".//div[contains(@class, 'compText')]")) or "",
|
||||
thumbnail=extract_text(eval_xpath(result, ".//a[contains(@class, 'thm')]/img/@data-src")) or "",
|
||||
)
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
def _video_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//div[contains(@class, 'results')]//ol/li"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, ".//a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
published_date_raw = extract_text(eval_xpath(result, ".//div[contains(@class, 'v-age')]"))
|
||||
try:
|
||||
published_date = parser.parse(published_date_raw or "")
|
||||
except parser.ParserError:
|
||||
published_date = None
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
{
|
||||
"template": "videos.html",
|
||||
"url": url,
|
||||
"title": extract_text(eval_xpath(result, ".//h3")),
|
||||
"content": extract_text(eval_xpath(result, ".//div[contains(@class, 'compText')]")),
|
||||
"thumbnail": extract_text(eval_xpath(result, ".//img[contains(@class, 'thm')]/@src")),
|
||||
"length": extract_text(eval_xpath(result, ".//span[contains(@class, 'v-time')]")),
|
||||
"publishedDate": published_date,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def _image_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//section[@id='results']//ul/li"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, "./a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
{
|
||||
"template": "images.html",
|
||||
# results don't have an extra URL, only the image source
|
||||
"url": url,
|
||||
"title": extract_text(eval_xpath(result, ".//a/@aria-label")),
|
||||
"thumbnail_src": extract_text(eval_xpath(result, ".//img/@src")),
|
||||
"img_src": url,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
doc = html.fromstring(resp.text)
|
||||
|
||||
match search_type:
|
||||
case "search":
|
||||
results = _general_results(doc)
|
||||
case "image":
|
||||
results = _image_results(doc)
|
||||
case "video":
|
||||
results = _video_results(doc)
|
||||
case _:
|
||||
raise ValueError("unsupported search type")
|
||||
|
||||
for suggestion in eval_xpath_list(doc, ".//ol[contains(@class, 'searchRightBottom')]//table//a"):
|
||||
results.add(results.types.LegacyResult({"suggestion": extract_text(suggestion)}))
|
||||
|
||||
return results
|
||||
@@ -14,7 +14,6 @@ from searx.extended_types import SXNG_Response
|
||||
from searx.network import get, post
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
from searx.enginelib import EngineCache
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search.processors import OnlineParams
|
||||
@@ -42,21 +41,7 @@ search_index = "cw22"
|
||||
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
|
||||
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache to store session info (i.e. api key, csrf token, session id)."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _obtain_api_key() -> tuple[str, str, str]:
|
||||
cached_session = CACHE.get("session")
|
||||
if cached_session:
|
||||
return tuple(cached_session.split("|"))
|
||||
|
||||
home_resp = get(base_url)
|
||||
if not home_resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain api key")
|
||||
@@ -76,10 +61,6 @@ def _obtain_api_key() -> tuple[str, str, str]:
|
||||
session_id = token_resp.cookies["sessionid"]
|
||||
scraped_api_key = token_resp.json()["token"]["token"]
|
||||
|
||||
# session keys seem to become rate-limited very fast, so only remembering
|
||||
# for 1 minute here
|
||||
CACHE.set("session", f"{csrf_token}|{session_id}|{scraped_api_key}", expire=60)
|
||||
|
||||
return csrf_token, session_id, scraped_api_key
|
||||
|
||||
|
||||
|
||||
@@ -444,27 +444,6 @@ engines:
|
||||
shortcut: conda
|
||||
disabled: true
|
||||
|
||||
- name: aol
|
||||
engine: aol
|
||||
search_type: search
|
||||
categories: [general]
|
||||
shortcut: aol
|
||||
disabled: true
|
||||
|
||||
- name: aol images
|
||||
engine: aol
|
||||
search_type: image
|
||||
categories: [images]
|
||||
shortcut: aoli
|
||||
disabled: true
|
||||
|
||||
- name: aol videos
|
||||
engine: aol
|
||||
search_type: video
|
||||
categories: [videos]
|
||||
shortcut: aolv
|
||||
disabled: true
|
||||
|
||||
- name: arch linux wiki
|
||||
engine: archlinux
|
||||
shortcut: al
|
||||
|
||||
Reference in New Issue
Block a user