typia.llm.evaluation — typed questions for evaluation models

export namespace llm {
  export function evaluation<T extends Record<string, any>>(): ILlmEvaluation<T>;
}

import { IValidation } from "./IValidation";
 
/**
 * LLM evaluation questions with a checked answer decoder.
 *
 * `ILlmEvaluation<T>` is generated by `typia.llm.evaluation<T>()`. Each leaf
 * property of `T` becomes one question for an evaluation model, such as
 * TypeSafe's Jev, or OpenAI, Anthropic, and Google through Vercel AI SDK's
 * `experimental_evaluate()`:
 *
 * - `boolean` becomes an {@link ILlmEvaluation.IBoolean} question
 * - A string enum or string literal union becomes an
 *   {@link ILlmEvaluation.IChoice} question
 * - A numeric enum or numeric literal union becomes an
 *   {@link ILlmEvaluation.IScore} question
 * - An array of a string literal union or string enum becomes one
 *   {@link ILlmEvaluation.IBoolean} question per member, asking in English
 *   whether that member applies
 * - A nested object is flattened, one question per leaf
 *
 * Workflow:
 *
 * 1. Pass {@link questions} to the evaluation model with a shared state
 * 2. Receive the answer map keyed by the same question IDs
 * 3. Call {@link decode} to check the answers and fold them back into `T`
 *
 * Probabilities are calibrated only on native evaluation models such as Jev.
 * The AI SDK adapters for OpenAI, Anthropic, and Google ask a language model to
 * write each number itself, and return no choice or score distribution.
 *
 * This interface is experimental. It follows Vercel AI SDK's evaluation model
 * specification, which is itself experimental and may change in patch releases.
 * Refinements that the answer decoder cannot enforce are compile errors, so a
 * successful {@link decode} does not require a second `typia.validate<T>()`.
 *
 * @author Jeongho Nam - https://github.com/samchon
 * @template T The decision type
 */
export interface ILlmEvaluation<T = unknown> {
  /**
   * Question map, keyed by readable property paths.
   *
   * Pass it as the `questions` of Vercel AI SDK's `experimental_evaluate()` as
   * it is. Jev's own wire format, shared by TypeSafe's API and OpenRouter's
   * Decisions API, spells the boolean question type `"noul"`;
   * `toJevQuestions()` from `@typia/jev` converts the questions.
   *
   * Jev evaluates each question independently, while the AI SDK adapters for
   * language models only instruct the model to. A key is not a part of the
   * question text. The `instructions` come from the property's JSDoc
   * description, and choice or score criteria from the enum members' JSDoc
   * descriptions.
   */
  questions: Record<string, ILlmEvaluation.IQuestion>;
 
  /**
   * Check an answer map and decode it into `T`.
   *
   * Unlike `typia.validate<T>()` or `ILlmStructuredOutput.validate()`, this
   * method transforms a model answer map into a different shape. It checks the
   * answers keyed by {@link questions} and returns the converted `T` on success.
   * It accepts both the neutral boolean answer (`{ type: "boolean", probability
   * }`) and TypeSafe's native one (`{ type: "noul", noul }`).
   *
   * Conversion rules:
   *
   * - A boolean is `true` when its probability reaches the threshold of
   *   `tags.Probability<N>` or `@probability N`, `0.5` by default
   * - A choice is the returned option
   * - A score is the level with the highest probability when the answer has
   *   `probabilities`, otherwise the level nearest to `score`
   * - A choice option or score level with a probability requirement fails
   *   validation when it is selected below that requirement, or when the answer
   *   has no `probabilities` to prove it
   * - Once one choice, score, or set member declares a probability requirement,
   *   every member must declare one or inherit a property `@probability`
   *   default
   * - An optional choice or score `probabilities` map must be complete, sum to
   *   one, and agree with the selected choice or score, following AI SDK's
   *   evaluation answer contract and any declared rounding precision
   *
   * @param answers Answer map from the evaluation model
   * @param rounding Optional provider-declared precision. Pass AI SDK's
   *   `result.rounding` here; without it, distribution checks remain strict.
   * @returns Validation result with the converted `T` on success
   */
  decode: (
    answers: unknown,
    rounding?: ILlmEvaluation.IRounding,
  ) => IValidation<T>;
}
export namespace ILlmEvaluation {
  /** Decimal precision declared by an evaluation model for its answer values. */
  export interface IRounding {
    probabilityDecimals?: number;
    scoreDecimals?: number;
  }
 
  /** Any evaluation question. */
  export type IQuestion = IChoice | IScore | IBoolean;
 
  /** Question selecting exactly one of the declared options. */
  export interface IChoice {
    /** Discriminator. */
    type: "choice";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * The alternatives the evaluation model must distinguish.
     *
     * Each key is one string literal value that the model may select, and its
     * value explains when that option applies. The explanation comes from the
     * enum member's JSDoc description. It is `null` when the member has no
     * description, in which case the model sees only the literal key.
     *
     * For example, an enum member `billing = "billing"` documented as `Payments
     * and refunds` becomes `{ billing: "Payments and refunds" }`.
     */
    criteria: Record<string, string | null>;
  }
 
  /** Question placing the state on ordered levels. */
  export interface IScore {
    /** Discriminator. */
    type: "score";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * The ordered levels the evaluation model uses to place the state.
     *
     * Each entry explains when its level applies and comes from that numeric
     * enum member's JSDoc description. The entries are ordered by the members'
     * numeric values, not by declaration order. The wire-level score is a
     * possibly fractional position on the array's index scale;
     * {@link ILlmEvaluation.decode} selects a level and maps it back to the
     * corresponding numeric value of `T`.
     *
     * For example, members `calm = 10` and `angry = 30`, documented as `No
     * frustration` and `Clearly frustrated`, produce `["No frustration",
     * "Clearly frustrated"]`. A level without a description uses its numeric
     * value as text. There are always at least two levels, because a single
     * numeric literal is a compile error.
     */
    criteria: [string, string, ...string[]];
  }
 
  /** Question estimating the probability of "yes". */
  export interface IBoolean {
    /** Discriminator. */
    type: "boolean";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
  }
}

import { TagBase } from "./TagBase";
 
/**
 * Probability requirement for an LLM evaluation decision.
 *
 * `Probability<N>` attaches a probability requirement to a decision that
 * `typia.llm.evaluation<T>()` asks an evaluation model to make. `N` must be in
 * `[0, 1]`:
 *
 * - On a `boolean` property, it is the decision threshold: the value becomes
 *   `true` only when the model's P(true) is at least `N`. Without the tag the
 *   threshold is `0.5`.
 * - On every member of a literal union, it is that member's acceptance minimum:
 *   `decode()` fails when the model selects the member with a probability below
 *   `N`, instead of falling back to a less likely member.
 * - On every member of a literal union used as an array element, where every
 *   member is its own yes/no decision, it is that member's inclusion threshold,
 *   as on a `boolean`.
 *
 * TypeScript enum members cannot carry type tags, so write `@probability N` in
 * the member's JSDoc comment instead. Once one member has a probability
 * requirement, every member must have one. The same comment tag on a property
 * sets the boolean threshold, or supplies the default for every member that
 * carries none: a minimum for a choice or score, a threshold for an array set.
 *
 * The tag carries metadata only. It adds no runtime check to `typia.is()` or
 * `typia.validate()` and no JSON Schema keyword, because it constrains how an
 * evaluation answer is converted, not the value itself.
 *
 * @author Jeongho Nam - https://github.com/samchon
 * @example
 *   interface ITicketTriage {
 *     // true only when P(true) >= 0.8
 *     refund: boolean & Probability<0.8>;
 *   }
 *
 * @template Value The probability requirement, in `[0, 1]`
 */
export type Probability<Value extends number> = TagBase<{
  target: "boolean" | "string" | "number";
  kind: "probability";
  value: Value;
  exclusive: true;
}>;

import { TypeSafeClient } from "@typesafe-ai/sdk";
import { toJevQuestions } from "@typia/jev";
import typia, { tags } from "typia";
 
enum Department {
  /**
   * Payments, invoicing, refunds
   *
   * @probability 0.5
   */
  billing = "billing",
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  technical = "technical",
  /**
   * Pricing, upgrades, new accounts
   *
   * @probability 0.5
   */
  sales = "sales",
}
 
enum Frustration {
  /** Calm or neutral */
  calm = 0,
  /** Annoyed but cooperative */
  annoyed = 1,
  /** Angry or threatening to leave */
  angry = 2,
}
 
interface ITicketTriage {
  /** Does the customer convey urgency? */
  urgent: boolean;
 
  /** Which team should handle this ticket? */
  department: Department;
 
  /** How frustrated is the customer? */
  frustration: Frustration;
 
  /** Which products does the customer mention? */
  products: Array<"card" | "loan" | "deposit">;
 
  refund: {
    /** Does the customer ask for a refund? */
    requested: boolean & tags.Probability<0.8>;
  };
}
 
const main = async (): Promise<void> => {
  // Generate the questions and checked answer decoder.
  const triage = typia.llm.evaluation<ITicketTriage>();
 
  // Ask TypeSafe's Jev in its own wire format
  const client = new TypeSafeClient(); // reads TYPESAFE_API_KEY
  const { answers } = await client.systemOne({
    model: "jev-1.13.0", // pinned: thresholds are tuned per model version
    state: "I was charged twice this morning. Refund it now, or I leave.",
    questions: toJevQuestions(triage.questions),
  });
 
  // Check and decode the answers into ITicketTriage.
  // The direct TypeSafe SDK does not declare rounding precision.
  const result = triage.decode(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
  console.log("Department probabilities:", answers.department);
};
main().catch(console.error);

import { TypeSafeClient } from "@typesafe-ai/sdk";
import { toJevQuestions } from "@typia/jev";
import typia from "typia";
import * as _createLlmEvaluation_1 from "typia/lib/internal/_createLlmEvaluation";
 
var Department;
(function (Department) {
  /**
   * Payments, invoicing, refunds
   *
   * @probability 0.5
   */
  Department["billing"] = "billing";
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  Department["technical"] = "technical";
  /**
   * Pricing, upgrades, new accounts
   *
   * @probability 0.5
   */
  Department["sales"] = "sales";
})(Department || (Department = {}));
var Frustration;
(function (Frustration) {
  /** Calm or neutral */
  Frustration[(Frustration["calm"] = 0)] = "calm";
  /** Annoyed but cooperative */
  Frustration[(Frustration["annoyed"] = 1)] = "annoyed";
  /** Angry or threatening to leave */
  Frustration[(Frustration["angry"] = 2)] = "angry";
})(Frustration || (Frustration = {}));
const main = async () => {
  // Generate the questions and checked answer decoder.
  const triage = _createLlmEvaluation_1._createLlmEvaluation([
    {
      instructions: "Does the customer convey urgency?",
      kind: "boolean",
      path: ["urgent"],
      threshold: 0.5,
    },
    {
      instructions: "Which team should handle this ticket?",
      kind: "choice",
      options: [
        {
          minimum: 0.5,
          description: "Payments, invoicing, refunds",
          value: "billing",
        },
        {
          minimum: 0.5,
          description: "Pricing, upgrades, new accounts",
          value: "sales",
        },
        {
          minimum: 0.75,
          description: "Bugs, outages, integrations",
          value: "technical",
        },
      ],
      path: ["department"],
    },
    {
      instructions: "How frustrated is the customer?",
      kind: "score",
      levels: [
        {
          description: "Calm or neutral",
          value: 0,
        },
        {
          description: "Annoyed but cooperative",
          value: 1,
        },
        {
          description: "Angry or threatening to leave",
          value: 2,
        },
      ],
      path: ["frustration"],
    },
    {
      instructions: "Which products does the customer mention?",
      kind: "set",
      members: [
        {
          threshold: 0.5,
          value: "card",
        },
        {
          threshold: 0.5,
          value: "deposit",
        },
        {
          threshold: 0.5,
          value: "loan",
        },
      ],
      path: ["products"],
    },
    {
      instructions: "Does the customer ask for a refund?",
      kind: "boolean",
      path: ["refund", "requested"],
      threshold: 0.8,
    },
  ]);
  // Ask TypeSafe's Jev in its own wire format
  const client = new TypeSafeClient(); // reads TYPESAFE_API_KEY
  const { answers } = await client.systemOne({
    model: "jev-1.13.0", // pinned: thresholds are tuned per model version
    state: "I was charged twice this morning. Refund it now, or I leave.",
    questions: toJevQuestions(triage.questions),
  });
  // Check and decode the answers into ITicketTriage.
  // The direct TypeSafe SDK does not declare rounding precision.
  const result = triage.decode(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
  console.log("Department probabilities:", answers.department);
};
main().catch(console.error);