fix(morpho): retry on transient GraphQL INTERNAL_SERVER_ERROR errors

protocols/compound/proposals.py

@@ -5,6 +5,7 @@
 
 from utils.alert import Alert, AlertSeverity, send_alert
 from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file
+from utils.http_client import request_with_retry
 from utils.logger import get_logger
 from utils.telegram import escape_markdown, send_error_message
 
@@ -86,13 +87,13 @@ def get_proposals():
     }
 
     try:
-        response = requests.post(
+        response = request_with_retry(
+            "post",
             TALLY_API_URL,
             json={"query": query, "variables": variables},
             headers=headers,
             timeout=30,
         )
-        response.raise_for_status()
         data = response.json()
 
         if "errors" in data:

protocols/ethena/ethena.py

@@ -1,9 +1,8 @@
 from datetime import datetime, timedelta, timezone
 
-import requests
-
 from utils.abi import load_abi
 from utils.alert import Alert, AlertSeverity, send_alert
+from utils.http_client import fetch_json
 from utils.logger import get_logger
 from utils.telegram import send_error_message
 from utils.web3_wrapper import Chain, ChainManager
@@ -29,19 +28,6 @@
 ETHENA_SOURCE = "Ethena API"
 
 
-def fetch_json(url: str) -> dict | None:
-    """Helper that fetches JSON with basic error handling."""
-    try:
-        resp = requests.get(url, timeout=REQUEST_TIMEOUT)
-        if resp.status_code != 200:
-            logger.error("HTTP %s for %s\n%s", resp.status_code, url, resp.text)
-            return None
-        return resp.json()
-    except Exception as e:
-        logger.error("Failed to fetch %s: %s", url, e)
-        return None
-
-
 def _parse_timestamp(ts: str) -> datetime | None:
     """Parse the timestamp formats returned by Ethena's transparency API."""
     formats = [
@@ -75,7 +61,7 @@ def is_stale_timestamp(ts: str, max_age_hours: int = 3) -> bool:
 
 def get_usde_supply() -> float | None:
     """Return total circulating USDe supply in USD terms (raw token amount / 1e18)."""
-    data = fetch_json(SUPPLY_URL)
+    data = fetch_json(SUPPLY_URL, timeout=REQUEST_TIMEOUT)
     if not data:
         return None
 
@@ -98,7 +84,7 @@ def get_total_collateral_usd() -> float | None:
     e.g. $4.14B vs $4.03B) but is a stale snapshot (items lag several hours). We
     use the fresh net figure and add the reserve fund as the buffer.
     """
-    data = fetch_json(COLLATERAL_URL)
+    data = fetch_json(COLLATERAL_URL, timeout=REQUEST_TIMEOUT)
     if not data:
         return None
 
@@ -112,7 +98,7 @@ def get_reserve_fund() -> float | None:
     ``{timestamp, value}`` points; we take the most recent one and treat stale
     data (older than 3 hours) as unavailable.
     """
-    data = fetch_json(RESERVE_FUND_URL)
+    data = fetch_json(RESERVE_FUND_URL, timeout=REQUEST_TIMEOUT)
     if not data:
         return None
 

protocols/fluid/proposals.py

@@ -4,6 +4,7 @@
 
 from utils.alert import Alert, AlertSeverity, send_alert
 from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file
+from utils.http_client import request_with_retry
 from utils.logger import get_logger
 from utils.telegram import escape_markdown, send_error_message
 
@@ -62,8 +63,7 @@ def get_proposals():
     """Fetch and process Fluid governance proposals"""
     try:
         # Fetch queued proposals
-        response = requests.get(f"{FLUID_API_URL}?status=queued", timeout=30)
-        response.raise_for_status()
+        response = request_with_retry("get", f"{FLUID_API_URL}?status=queued", timeout=30)
         data = response.json()
 
         if "data" not in data or not data["data"]:

protocols/infinifi/main.py

@@ -1,6 +1,5 @@
 from decimal import Decimal
 
-import requests
 from web3 import Web3
 
 from utils.abi import load_abi
@@ -15,6 +14,7 @@
 )
 from utils.chains import Chain
 from utils.config import Config
+from utils.http_client import fetch_json
 from utils.logger import get_logger
 from utils.telegram import send_error_message
 from utils.web3_wrapper import ChainManager
@@ -57,20 +57,10 @@
 
 def fetch_api_data():
     """Fetches data from the Infinifi API."""
-    url = f"{API_BASE_URL}{API_PROTOCOL_DATA}"
-    try:
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-        }
-        response = requests.get(url, headers=headers, timeout=10)
-        if response.status_code == 200:
-            return response.json()
-        else:
-            logger.error("API Error %s for %s: %s", response.status_code, url, response.text[:200])
-            return None
-    except Exception as e:
-        logger.error("API Request Failed for %s: %s", url, e)
-        return None
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    }
+    return fetch_json(f"{API_BASE_URL}{API_PROTOCOL_DATA}", headers=headers, timeout=10)
 
 
 def to_float(value, default=0.0):

protocols/maker/proposals.py

@@ -4,6 +4,7 @@
 
 from utils.alert import Alert, AlertSeverity, send_alert
 from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file
+from utils.http_client import request_with_retry
 from utils.logger import get_logger
 from utils.telegram import escape_markdown, send_error_message
 
@@ -16,8 +17,7 @@
 
 def fetch_executive_proposals() -> list[dict]:
     """Fetch executive proposals from the Sky governance API."""
-    response = requests.get(SKY_EXECUTIVE_API, timeout=30)
-    response.raise_for_status()
+    response = request_with_retry("get", SKY_EXECUTIVE_API, timeout=30)
     return response.json()
 
 

protocols/morpho/_shared.py

@@ -1,11 +1,13 @@
 """Shared helpers used by both v1 and v2 Morpho monitors."""
 
+import time
 from dataclasses import dataclass
 from typing import Any, Dict, Iterable, List, Optional
 
 import requests
 
 from utils.chains import Chain
+from utils.config import Config
 from utils.http_client import request_with_retry
 from utils.logger import get_logger
 
@@ -24,30 +26,59 @@ class MorphoV2MonitoringError(MorphoMonitoringError):
     """Raised when configured Morpho Vault V2 monitoring is incomplete."""
 
 
+def _is_retryable_graphql_error(error: dict[str, Any]) -> bool:
+    """Return True for transient GraphQL error statuses that may resolve on retry."""
+    return error.get("status") == "INTERNAL_SERVER_ERROR"
+
+
 def execute_graphql(
     query: str,
     variables: dict[str, Any],
     context: str,
     *,
     error_type: type[MorphoMonitoringError] = MorphoMonitoringError,
 ) -> dict[str, Any]:
-    """Execute a strict Morpho GraphQL request and return its data object."""
-    try:
-        response = request_with_retry("post", API_URL, json={"query": query, "variables": variables})
-    except requests.RequestException as exc:
-        raise error_type(f"Failed to fetch {context}: {exc}") from exc
+    """Execute a strict Morpho GraphQL request and return its data object.
 
-    try:
-        payload = response.json()
-    except ValueError as exc:
-        raise error_type(f"Morpho returned invalid JSON while fetching {context}") from exc
-
-    if payload.get("errors"):
-        raise error_type(f"Morpho GraphQL errors fetching {context}: {payload['errors']}")
-    data = payload.get("data")
-    if not isinstance(data, dict):
-        raise error_type(f"Morpho GraphQL returned no data while fetching {context}")
-    return data
+    Retries on transient GraphQL body-level errors (e.g. INTERNAL_SERVER_ERROR)
+    in addition to the HTTP-level retries performed by ``request_with_retry``.
+    """
+    retries = Config.get_retry_count()
+    backoff_factor = Config.get_backoff_factor()
+
+    for attempt in range(retries + 1):
+        try:
+            response = request_with_retry("post", API_URL, json={"query": query, "variables": variables})
+        except requests.RequestException as exc:
+            raise error_type(f"Failed to fetch {context}: {exc}") from exc
+
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise error_type(f"Morpho returned invalid JSON while fetching {context}") from exc
+
+        errors = payload.get("errors")
+        if errors:
+            if all(_is_retryable_graphql_error(e) for e in errors) and attempt < retries:
+                wait_time = backoff_factor * (2**attempt)
+                logger.warning(
+                    "Morpho GraphQL transient error (attempt %d/%d): %s. Retrying in %.1fs...",
+                    attempt + 1,
+                    retries + 1,
+                    errors,
+                    wait_time,
+                )
+                time.sleep(wait_time)
+                continue
+            raise error_type(f"Morpho GraphQL errors fetching {context}: {errors}")
+
+        data = payload.get("data")
+        if not isinstance(data, dict):
+            raise error_type(f"Morpho GraphQL returned no data while fetching {context}")
+        return data
+
+    # Unreachable — loop always returns or raises.
+    raise error_type(f"Morpho GraphQL failed after {retries + 1} attempts for {context}")
 
 
 def require_configured_keys(

protocols/spark/proposals.py

@@ -10,6 +10,7 @@
 
 from utils.alert import Alert, AlertSeverity, send_alert
 from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file
+from utils.http_client import request_with_retry
 from utils.logger import get_logger
 from utils.telegram import escape_markdown, send_error_message
 
@@ -42,12 +43,12 @@
 
 def fetch_proposals() -> list[dict]:
     """Fetch the most recent proposals from the Snapshot hub GraphQL API."""
-    response = requests.post(
+    response = request_with_retry(
+        "post",
         SNAPSHOT_GRAPHQL_URL,
         json={"query": PROPOSALS_QUERY, "variables": {"space": SNAPSHOT_SPACE}},
         timeout=30,
     )
-    response.raise_for_status()
     payload = response.json()
     if payload.get("errors"):
         raise ValueError(f"Snapshot GraphQL errors: {payload['errors']}")

protocols/yearn/kong.py

@@ -5,6 +5,7 @@
 import requests
 
 from utils.chains import Chain
+from utils.http_client import request_with_retry
 from utils.logger import get_logger
 
 logger = get_logger("yearn.kong")
@@ -55,12 +56,15 @@ class KongRequestError(requests.RequestException):
 
 def _post_graphql(query: str, variables: Dict[str, object]) -> Dict[str, Any]:
     """Execute a Kong GraphQL query and return the response data."""
-    response = requests.post(
-        KONG_GQL_URL,
-        json={"query": query, "variables": variables},
-        timeout=30,
-    )
-    response.raise_for_status()
+    try:
+        response = request_with_retry(
+            "post",
+            KONG_GQL_URL,
+            json={"query": query, "variables": variables},
+            timeout=30,
+        )
+    except requests.RequestException as e:
+        raise KongRequestError(f"Kong request failed: {e}") from e
 
     try:
         payload = response.json()

tests/test_compound_proposals.py

@@ -38,14 +38,14 @@ def test_compound_alert_uses_timeout_escapes_title_and_updates_reported_id():
     }
 
     with (
-        patch("protocols.compound.proposals.requests.post", return_value=_Response(payload)) as mock_post,
+        patch("protocols.compound.proposals.request_with_retry", return_value=_Response(payload)) as mock_request,
         patch("protocols.compound.proposals.get_last_queued_id_from_file", return_value=40),
         patch("protocols.compound.proposals.send_alert") as mock_send,
         patch("protocols.compound.proposals.write_last_queued_id_to_file") as mock_write,
     ):
         get_proposals()
 
-    assert mock_post.call_args.kwargs["timeout"] == 30
+    assert mock_request.call_args.kwargs["timeout"] == 30
     mock_send.assert_called_once()
     alert = mock_send.call_args.args[0]
     assert alert.protocol == "comp"
@@ -57,7 +57,7 @@ def test_compound_alert_uses_timeout_escapes_title_and_updates_reported_id():
 
 def test_compound_processing_error_alert_uses_plain_text():
     with (
-        patch("protocols.compound.proposals.requests.post", return_value=_Response({"data": {}})),
+        patch("protocols.compound.proposals.request_with_retry", return_value=_Response({"data": {}})),
         patch("protocols.compound.proposals.send_error_message") as mock_send,
     ):
         get_proposals()
@@ -71,7 +71,7 @@ def test_compound_processing_error_alert_uses_plain_text():
 
 def test_compound_fetch_error_alert_uses_plain_text():
     with (
-        patch("protocols.compound.proposals.requests.post", side_effect=requests.Timeout("timed out")),
+        patch("protocols.compound.proposals.request_with_retry", side_effect=requests.RequestException("timed out")),
         patch("protocols.compound.proposals.send_error_message") as mock_send,
     ):
         get_proposals()

tests/test_fluid_proposals.py

@@ -40,7 +40,7 @@ def test_fluid_proposal_alert_escapes_api_markdown_and_keeps_link():
     }
 
     with (
-        patch("protocols.fluid.proposals.requests.get", return_value=_Response(payload)),
+        patch("protocols.fluid.proposals.request_with_retry", return_value=_Response(payload)),
         patch("protocols.fluid.proposals.get_last_queued_id_from_file", return_value=130),
         patch("protocols.fluid.proposals.write_last_queued_id_to_file") as mock_write,
         patch("protocols.fluid.proposals.send_alert") as mock_send,
@@ -59,7 +59,7 @@ def test_fluid_proposal_alert_escapes_api_markdown_and_keeps_link():
 
 def test_fluid_proposal_fetch_error_routes_to_errors_channel():
     with (
-        patch("protocols.fluid.proposals.requests.get", side_effect=Exception("bad TYPE_1 payload")),
+        patch("protocols.fluid.proposals.request_with_retry", side_effect=Exception("bad TYPE_1 payload")),
         patch("protocols.fluid.proposals.send_error_message") as mock_send,
     ):
         get_proposals()