429 vs 503: retrying LLM API requests with Retry-After

import random
import time

import openai
from openai import OpenAI

client = OpenAI(
    base_url="https://api.gtokens.io/api/v1",
    api_key="sk-sl-your-key",
    max_retries=0,  # this loop is the only retry policy
)

MAX_ATTEMPTS = 4


def complete(prompt, model="anthropic/claude-sonnet-5"):
    for attempt in range(1, MAX_ATTEMPTS + 1):
        try:
            return client.chat.completions.create(
                model=model,
                messages=[{"role": "user", "content": prompt}],
            )
        except openai.APIStatusError as e:
            # 429 and 503 are waited out; everything else goes to the caller,
            # and so does the last refusal once the attempts run out.
            if e.status_code not in (429, 503) or attempt == MAX_ATTEMPTS:
                raise
            # Whole seconds, present on every 429 and 503.
            delay = int(e.response.headers.get("retry-after", "30"))
            # Never sooner than the header, doubled on each repeat, jitter on top.
            delay *= 2 ** (attempt - 1)
            time.sleep(delay + random.uniform(0, delay / 10))

import OpenAI, { APIError } from 'openai'

const client = new OpenAI({
  baseURL: 'https://api.gtokens.io/api/v1',
  apiKey: 'sk-sl-your-key',
  maxRetries: 0, // this loop is the only retry policy
})

const MAX_ATTEMPTS = 4
const sleep = (s: number) => new Promise((done) => setTimeout(done, s * 1000))

// Whole seconds. openai v4 hands error headers over as a plain object,
// v5 and later as a Headers instance; this reads either.
function retryAfter(err: APIError): number {
  const h: any = err.headers
  const value = typeof h?.get === 'function' ? h.get('retry-after') : h?.['retry-after']
  return Number(value) || 30
}

export async function complete(prompt: string, model = 'anthropic/claude-sonnet-5') {
  for (let attempt = 1; ; attempt++) {
    try {
      return await client.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
      })
    } catch (err) {
      if (!(err instanceof APIError)) throw err
      // 429 and 503 are waited out; everything else goes to the caller,
      // and so does the last refusal once the attempts run out.
      const retryable = err.status === 429 || err.status === 503
      if (!retryable || attempt === MAX_ATTEMPTS) throw err
      // Never sooner than the header, doubled on each repeat, jitter on top.
      const delay = retryAfter(err) * 2 ** (attempt - 1)
      await sleep(delay + Math.random() * (delay / 10))
    }
  }
}

import random
import time

import anthropic
from anthropic import Anthropic

client = Anthropic(
    base_url="https://api.gtokens.io",       # the root: the library appends /v1/messages
    auth_token="sk-sl-your-key",   # sent as Authorization: Bearer
    max_retries=0,                 # this loop is the only retry policy
)

MAX_ATTEMPTS = 4


def ask(prompt, model="anthropic/claude-haiku-4-5"):
    for attempt in range(1, MAX_ATTEMPTS + 1):
        try:
            return client.messages.create(
                model=model,
                max_tokens=1024,
                messages=[{"role": "user", "content": prompt}],
            )
        except anthropic.APIStatusError as e:
            # The same two statuses as on the other routes.
            if e.status_code not in (429, 503) or attempt == MAX_ATTEMPTS:
                raise
            delay = int(e.response.headers.get("retry-after", "30"))
            delay *= 2 ** (attempt - 1)
            time.sleep(delay + random.uniform(0, delay / 10))

# client, MAX_ATTEMPTS and the imports as in the loop above

# Worth a wait: the typed codes of 429 and 503, and provider_unavailable —
# the code of a failure the model reports mid-answer.
WAIT_TYPES = ("rate_limit_exceeded", "provider_overloaded", "provider_unavailable")


def stream_reply(prompt, model="anthropic/claude-sonnet-5"):
    for attempt in range(1, MAX_ATTEMPTS + 1):
        text = ""
        delay = 10  # your own base: inside a stream there is no header
        try:
            stream = client.chat.completions.create(
                model=model,
                messages=[{"role": "user", "content": prompt}],
                stream=True,
            )
            for chunk in stream:
                if chunk.choices and chunk.choices[0].delta.content:
                    text += chunk.choices[0].delta.content
            if text:
                return text
            # A 200 that carried nothing: a failure, worth retrying.
        except openai.APIStatusError as e:
            # Before the first byte: a real status, with Retry-After on 429 and 503.
            if e.status_code not in (429, 503) or attempt == MAX_ATTEMPTS:
                raise
            delay = int(e.response.headers.get("retry-after", "30"))
        except openai.APIError as e:
            # After the first byte the status was 200: the library raises the
            # error chunk, and its error_type says what failed.
            body = e.body if isinstance(e.body, dict) else {}
            error_type = (body.get("metadata") or {}).get("error_type")
            if error_type not in WAIT_TYPES or attempt == MAX_ATTEMPTS:
                raise
        if attempt < MAX_ATTEMPTS:
            delay *= 2 ** (attempt - 1)
            time.sleep(delay + random.uniform(0, delay / 10))
    raise RuntimeError("the stream ended without any content")

// client, MAX_ATTEMPTS, sleep and retryAfter as in the loop above

// Worth a wait: the typed codes of 429 and 503, and provider_unavailable —
// the code of a failure the model reports mid-answer.
const WAIT_TYPES = ['rate_limit_exceeded', 'provider_overloaded', 'provider_unavailable']

export async function streamReply(prompt: string, model = 'anthropic/claude-sonnet-5'): Promise<string> {
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
    let text = ''
    let delay = 10 // your own base: inside a stream there is no header
    try {
      const stream = await client.chat.completions.create({
        model,
        messages: [{ role: 'user', content: prompt }],
        stream: true,
      })
      for await (const chunk of stream) text += chunk.choices[0]?.delta?.content ?? ''
      if (text) return text
      // A 200 that carried nothing: a failure, worth retrying.
    } catch (err) {
      if (!(err instanceof APIError) || attempt === MAX_ATTEMPTS) throw err
      if (err.status !== undefined) {
        // Before the first byte: a real status, with Retry-After on 429 and 503.
        if (err.status !== 429 && err.status !== 503) throw err
        delay = retryAfter(err)
      } else {
        // After the first byte the status was 200: the library throws the
        // error chunk, and its error_type says what failed.
        const errorType = (err.error as any)?.metadata?.error_type
        if (!WAIT_TYPES.includes(errorType)) throw err
      }
    }
    if (attempt < MAX_ATTEMPTS) {
      delay *= 2 ** (attempt - 1)
      await sleep(delay + Math.random() * (delay / 10))
    }
  }
  throw new Error('the stream ended without any content')
}