[fix] pexels: circumvent botdetection by passing referer header

As a side effect, Cloudscraper is no longer needed.
It probably only ever worked by setting the correct request headers,
so we don't really need it since we can just set the right request
headers and ciphersuites ourselves.
This commit is contained in:
Bnyro
2026-04-05 11:50:05 +02:00
parent fb68010f72
commit 20fc6fe80d
2 changed files with 12 additions and 6 deletions
-1
View File
@@ -17,4 +17,3 @@ typer==0.24.1
isodate==0.7.2 isodate==0.7.2
whitenoise==6.12.0 whitenoise==6.12.0
typing-extensions==4.15.0 typing-extensions==4.15.0
cloudscraper==1.2.71
+12 -5
View File
@@ -6,10 +6,8 @@ import re
from urllib.parse import urlencode from urllib.parse import urlencode
from lxml import html from lxml import html
import cloudscraper
from searx.result_types import EngineResults from searx.result_types import EngineResults
from searx.utils import eval_xpath_list from searx.utils import eval_xpath_list, gen_useragent
from searx.enginelib import EngineCache from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException from searx.exceptions import SearxEngineAPIException
from searx.network import get from searx.network import get
@@ -40,6 +38,8 @@ SECRET_KEY_DB_KEY = "secret-key"
CACHE: EngineCache CACHE: EngineCache
"""Cache to store the secret API key for the engine.""" """Cache to store the secret API key for the engine."""
enable_http2 = False
def init(engine_settings): def init(engine_settings):
global CACHE # pylint: disable=global-statement global CACHE # pylint: disable=global-statement
@@ -47,8 +47,15 @@ def init(engine_settings):
def _get_secret_key(): def _get_secret_key():
scraper = cloudscraper.create_scraper() resp = get(
resp = scraper.get(base_url) base_url,
headers={
# circumvents Cloudflare bot protections
"User-Agent": gen_useragent(),
"Referer": base_url,
},
)
if resp.status_code != 200: if resp.status_code != 200:
raise SearxEngineAPIException("failed to obtain secret key") raise SearxEngineAPIException("failed to obtain secret key")