retry more capacity-style upstream failures

gateway.mjs

@@ -2419,10 +2419,6 @@ function truncateRetryableMessage(value, maxLength = 280) {
 }
 
 function findRetryableUpstreamErrorMatch(config, upstreamStatusCode, parsedBody, bodyText) {
-  if (!isUpstreamErrorStatus(upstreamStatusCode)) {
-    return null;
-  }
-
   const retryableMessages = normalizePhraseList(
     config.retryable_error_messages,
     DEFAULT_CONFIG.retryable_error_messages,
@@ -2435,7 +2431,11 @@ function findRetryableUpstreamErrorMatch(config, upstreamStatusCode, parsedBody,
     config.retryable_status_codes,
     DEFAULT_CONFIG.retryable_status_codes,
   );
-  if (retryableStatusCodes.length > 0 && !retryableStatusCodes.includes(upstreamStatusCode)) {
+  const statusEligible = isUpstreamErrorStatus(upstreamStatusCode) && (
+    retryableStatusCodes.length === 0 ||
+    retryableStatusCodes.includes(upstreamStatusCode)
+  );
+  if (!statusEligible && !isRetryableErrorPayloadShape(parsedBody)) {
     return null;
   }
 
@@ -2480,27 +2480,64 @@ function matchRetryableMessage(config, parsedBody, bodyText) {
   return null;
 }
 
-function isRetryableStreamErrorShape(parsedBody, eventName = "") {
-  if (`${eventName || ""}`.trim().toLowerCase() === "error") {
+function hasRetryableFailureKeyword(value) {
+  const normalized = `${value || ""}`.trim().toLowerCase();
+  if (!normalized) {
+    return false;
+  }
+  return (
+    normalized.includes("error") ||
+    normalized.includes("failed") ||
+    normalized.includes("failure") ||
+    normalized.includes("cancelled") ||
+    normalized.includes("canceled") ||
+    normalized.includes("rate_limit") ||
+    normalized.includes("overloaded") ||
+    normalized.includes("unavailable")
+  );
+}
+
+function hasRetryableFailureShape(value, depth = 0) {
+  if (!value || depth > 4) {
+    return false;
+  }
+  if (Array.isArray(value)) {
+    return value.some((item) => hasRetryableFailureShape(item, depth + 1));
+  }
+  if (typeof value !== "object") {
+    return false;
+  }
+  if (Object.hasOwn(value, "error")) {
+    return true;
+  }
+  if (Number.isInteger(value.status) && value.status >= 400) {
+    return true;
+  }
+  for (const key of ["type", "event", "status", "state", "result", "code"]) {
+    if (typeof value[key] === "string" && hasRetryableFailureKeyword(value[key])) {
+      return true;
+    }
+  }
+  for (const key of ["response", "data", "meta", "details", "detail"]) {
+    if (hasRetryableFailureShape(value[key], depth + 1)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function isRetryableErrorPayloadShape(parsedBody, eventName = "") {
+  if (hasRetryableFailureKeyword(eventName)) {
     return true;
   }
   if (!parsedBody || typeof parsedBody !== "object" || Array.isArray(parsedBody)) {
     return false;
   }
-  if (Object.hasOwn(parsedBody, "error")) {
-    return true;
-  }
-  if (typeof parsedBody.type === "string" && parsedBody.type.toLowerCase().includes("error")) {
-    return true;
-  }
-  if (typeof parsedBody.event === "string" && parsedBody.event.toLowerCase().includes("error")) {
-    return true;
-  }
-  return Number.isInteger(parsedBody.status) && parsedBody.status >= 400;
+  return hasRetryableFailureShape(parsedBody);
 }
 
 function findRetryableStreamErrorMatch(config, parsedBody, bodyText, eventName = "") {
-  if (!isRetryableStreamErrorShape(parsedBody, eventName)) {
+  if (!isRetryableErrorPayloadShape(parsedBody, eventName)) {
     return null;
   }
   return matchRetryableMessage(config, parsedBody, bodyText);
@@ -2564,14 +2601,6 @@ async function fetchUpstreamWithRetry(upstreamUrl, init, config, logger, request
         };
       }
 
-      if (!isUpstreamErrorStatus(response.status)) {
-        return {
-          response,
-          attempt_count: attempt,
-          retryable_upstream_error: null,
-        };
-      }
-
       const contentType = response.headers.get("content-type");
       if (!isJsonContentType(contentType)) {
         return {