llm-retry

import { retryWithValidation, heuristicValidator } from 'llm-retry';

// Wrap your LLM SDK call -- any provider works
const callLLM = async (messages, context) => {
  const response = await openai.chat.completions.create({
    model: context?.model ?? 'gpt-4o-mini',
    temperature: context?.temperature ?? 0.2,
    messages,
  });
  return response.choices[0].message.content;
};

const validator = heuristicValidator({
  type: 'object',
  required: ['name', 'age'],
});

const result = await retryWithValidation(callLLM, validator, {
  messages: [{ role: 'user', content: 'Return a JSON object with name and age.' }],
  maxRetries: 3,
});

if (result.success) {
  console.log(result.data); // { name: "Alice", age: 30 }
} else {
  console.error(result.error);
  console.error(`Failed after ${result.attempts.length} attempts`);
}

function retryWithValidation<T>(
  callLLM: CallLLMFunction,
  validator: ValidatorFunction<T>,
  options?: RetryOptions & { messages: Message[] }
): Promise<RetryResult<T>>

const result = await retryWithValidation(callLLM, validator, {
  messages: [{ role: 'user', content: 'Extract the person as JSON.' }],
  maxRetries: 5,
  repair: { level: 'aggressive' },
  feedbackStrategy: 'full',
  temperature: 0.3,
  model: 'gpt-4o',
  onAttempt: (record) => {
    console.log(`Attempt ${record.attempt}: ${record.success ? 'ok' : 'failed'}`);
  },
});

function createRetrier<T>(
  config: RetrierConfig & { validator: ValidatorFunction<T> }
): Retrier<T>

import { createRetrier, heuristicValidator } from 'llm-retry';

const retrier = createRetrier({
  validator: heuristicValidator({ type: 'object', required: ['id', 'title'] }),
  maxRetries: 5,
  repair: { level: 'aggressive' },
  feedbackStrategy: 'truncated',
});

// Reuse across multiple calls
const result1 = await retrier.retry(callLLM, [
  { role: 'user', content: 'Extract article 1 as JSON.' },
]);
const result2 = await retrier.retry(callLLM, [
  { role: 'user', content: 'Extract article 2 as JSON.' },
]);

// Inspect config
console.log(retrier.config.maxRetries); // 5

function heuristicValidator(
  schema?: Record<string, unknown>
): ValidatorFunction<unknown>

import { heuristicValidator } from 'llm-retry';

// Accept any valid JSON
const anyJson = heuristicValidator();

// Require a specific shape
const personValidator = heuristicValidator({
  type: 'object',
  required: ['name', 'age', 'email'],
});

function repairOutput(raw: string, level: RepairLevel): string

function extractCodeFence(text: string, lang?: string): string | null

function removeTrailingCommas(json: string): string

function extractJsonBlock(text: string): string | null

function buildFeedbackMessage(
  rawOutput: string,
  errors: ValidationError[],
  strategy: 'errors-only' | 'full' | 'truncated' | 'none'
): string

function wrapErrors(error: unknown): ValidationError[]

function formatValidationErrors(errors: ValidationError[]): string

interface RetryOptions {
  maxRetries?: number;
  repair?: RepairConfig;
  feedbackStrategy?: 'errors-only' | 'full' | 'truncated' | 'none';
  temperature?: number;
  model?: string;
  onAttempt?: (record: AttemptRecord) => void;
  systemPrompt?: string;
}

interface RepairConfig {
  level?: RepairLevel;
  custom?: (text: string) => string;
}

const result = await retryWithValidation(callLLM, validator, {
  messages,
  repair: {
    level: 'standard',
    custom: (text) => text.replace(/\/\/.*/g, ''), // strip JS-style line comments
  },
});

interface RetryResult<T> {
  success: boolean;
  data?: T;
  error?: string;
  attempts: AttemptRecord[];
  totalDurationMs: number;
  finalOutput?: string;
}

interface AttemptRecord {
  attempt: number;
  rawOutput: string;
  repairedOutput?: string;
  validationErrors?: ValidationError[];
  durationMs: number;
  success: boolean;
}

interface ValidationError {
  path: string;
  message: string;
  code: string;
  expected?: unknown;
  received?: unknown;
}

import type { ValidatorFunction, ValidationResult } from 'llm-retry';
import { retryWithValidation } from 'llm-retry';

interface Person {
  name: string;
  age: number;
}

const personValidator: ValidatorFunction<Person> = (data): ValidationResult<Person> => {
  if (typeof data !== 'object' || data === null) {
    return {
      success: false,
      errors: [{ path: '', message: 'expected an object', code: 'invalid_type' }],
    };
  }

  const obj = data as Record<string, unknown>;
  const errors = [];

  if (typeof obj.name !== 'string') {
    errors.push({
      path: 'name',
      message: 'expected a string',
      code: 'invalid_type',
      expected: 'string',
      received: typeof obj.name,
    });
  }

  if (typeof obj.age !== 'number' || obj.age < 0 || obj.age > 150) {
    errors.push({
      path: 'age',
      message: 'expected a number between 0 and 150',
      code: 'invalid_type',
      expected: 'number (0-150)',
      received: String(obj.age),
    });
  }

  if (errors.length > 0) {
    return { success: false, errors };
  }

  return { success: true, data: obj as Person };
};

const result = await retryWithValidation(callLLM, personValidator, {
  messages: [{ role: 'user', content: 'Return a person object with name and age.' }],
});

if (result.success) {
  // result.data is typed as Person
  console.log(result.data.name, result.data.age);
}

const result = await retryWithValidation(callLLM, validator, {
  messages,
  maxRetries: 5,
  onAttempt: (record) => {
    console.log(JSON.stringify({
      attempt: record.attempt,
      success: record.success,
      durationMs: record.durationMs,
      errorCount: record.validationErrors?.length ?? 0,
    }));

    if (!record.success && record.validationErrors) {
      for (const err of record.validationErrors) {
        console.warn(`  [${err.code}] ${err.path}: ${err.message}`);
      }
    }
  },
});

import OpenAI from 'openai';

const openai = new OpenAI();

const callLLM = async (messages, context) => {
  const response = await openai.chat.completions.create({
    model: context?.model ?? 'gpt-4o-mini',
    temperature: context?.temperature ?? 0.2,
    messages,
  });
  return response.choices[0].message.content ?? '';
};

import Anthropic from '@anthropic-ai/sdk';

const client = new Anthropic();

const callLLM = async (messages, context) => {
  const response = await client.messages.create({
    model: context?.model ?? 'claude-sonnet-4-20250514',
    max_tokens: context?.maxTokens ?? 1024,
    messages,
  });
  return response.content[0].type === 'text' ? response.content[0].text : '';
};

const callLLM = async (messages, context) => {
  const response = await fetch('http://localhost:11434/api/chat', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: context?.model ?? 'llama3',
      messages,
      stream: false,
      options: { temperature: context?.temperature ?? 0.2 },
    }),
  });
  const data = await response.json();
  return data.message.content;
};

import { extractCodeFence, removeTrailingCommas, extractJsonBlock, repairOutput } from 'llm-retry';

// Extract from a ```json fence
const json = extractCodeFence('```json\n{"key": "value"}\n```', 'json');
// '{"key": "value"}'

// Remove trailing commas
const fixed = removeTrailingCommas('{"a": 1, "b": 2,}');
// '{"a": 1, "b": 2}'

// Extract first balanced JSON block from prose
const block = extractJsonBlock('The result is {"name": "Alice"} as requested.');
// '{"name": "Alice"}'

// Apply full repair pipeline
const repaired = repairOutput('```json\n{"x": 1,}\n```', 'standard');
// '{"x": 1}'

import type {
  Message,
  RetryContext,
  CallLLMFunction,
  ValidationError,
  ValidationResult,
  ValidatorFunction,
  RepairLevel,
  RepairConfig,
  AttemptRecord,
  RetryResult,
  RetryOptions,
  RetrierConfig,
  Retrier,
} from 'llm-retry';

// Message format (standard chat completion format)
interface Message {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

// The function you provide to call your LLM
type CallLLMFunction = (
  messages: Message[],
  context?: RetryContext
) => Promise<string>;

// Context passed to callLLM on each attempt
interface RetryContext {
  attempt: number;
  temperature?: number;
  model?: string;
  maxTokens?: number;
  escalated: boolean;
}

// Your validator must return this discriminated union
type ValidationResult<T> =
  | { success: true; data: T }
  | { success: false; errors: ValidationError[] };

// Validator function signature
type ValidatorFunction<T> = (data: unknown) => ValidationResult<T>;

// Repair level setting
type RepairLevel = 'none' | 'minimal' | 'standard' | 'aggressive';

// Reusable retrier instance
interface Retrier<T> {
  retry(callLLM: CallLLMFunction, messages: Message[]): Promise<RetryResult<T>>;
  readonly config: RetrierConfig;
}