import { ChatOpenAI } from '@langchain/openai';
import zevals from '@zevals/core';
import { langChainZEvalsJudge } from '@zevals/langchain';

test('Simple example', async () => {
  const agent: zevals.Agent = {
    async invoke(messages) {
      // Run your application logic to generate a response for the user
      return { response: { role: 'assistant', content: 'What kind of vitamin?' } };
    },
  };

  const judge = langChainZEvalsJudge({
    model: new ChatOpenAI({
      modelName: 'gpt-4.1-mini',
      temperature: 0,
    }),
  });

  const followupAssertion = zevals.aiAssertion({
    judge,
    prompt: 'The assistant asked a followup question',
  });

  const { getResultOrThrow } = await zevals.evaluate({
    agent,
    segments: [
      // The user wants vitamins
      zevals.message({ role: 'user', content: 'I want another bottle of the vitamin' }),

      // The agent responds
      zevals.agentResponse(),

      // We judge the above
      zevals.aiEval(followupAssertion),
    ],
  });

  // Run your assertions on type-safe outputs
  expect(getResultOrThrow(followupAssertion).output).toBe(true);
});

import zevals from '@zevals/core';
import { simpleExampleAgent } from './simple-example-agent.js';

test('Tool calls example', { timeout: 20000 }, async () => {
  const agent = simpleExampleAgent();

  // We expect the `get_current_date` tool to be called and to return the current date
  const dateToolCalledAssertion = zevals.aiToolCalls({
    assertion(toolCalls) {
      const date = new Date();

      expect(toolCalls).toEqual(
        expect.arrayContaining([
          {
            name: 'get_current_date',
            args: {},
            result: {
              day: date.getDate(),
              month: date.getMonth(),
              year: date.getFullYear(),
            },
          },
        ]),
      );
    },
  });

  const { getResultOrThrow } = await zevals.evaluate({
    agent,
    segments: [
      zevals.message({ role: 'user', content: 'What day of month are we at?' }),

      zevals.agentResponse(),

      zevals.aiEval(dateToolCalledAssertion),
    ],
  });

  const error = getResultOrThrow(dateToolCalledAssertion).error;
  if (error) throw error;
});

import { BaseMessage } from '@langchain/core/messages';
import { RunnableLambda } from '@langchain/core/runnables';
import { ChatOpenAI } from '@langchain/openai';
import zevals from '@zevals/core';
import { langChainZEvalsJudge, langChainZEvalsSyntheticUser } from '@zevals/langchain';
import { simpleExampleAgent } from './simple-example-agent.js';

test('User simulation example', { timeout: 60000 }, async () => {
  const agent = simpleExampleAgent();
  const model = new ChatOpenAI({ model: 'gpt-4.1-mini', temperature: 0 });
  const judge = langChainZEvalsJudge({ model });

  const user = langChainZEvalsSyntheticUser({
    runnable: RunnableLambda.from((messages: BaseMessage[]) =>
      model.invoke([
        {
          role: 'system',
          content: `You will ask three questions, each in a separate message. Do not repeat the same question.
             Question 1) What is the capital of France? 
             Question 2) What is the capital of Germany? 
             Question 3) What is the capital of Italy?`,
        },
        ...messages,
      ]),
    ),
  });

  const { messages, success } = await zevals.evaluate({
    agent,
    segments: [
      zevals.userSimulation({
        user,
        until: zevals.aiAssertion({
          judge,
          prompt: 'The assistant has answered THREE questions',
        }),
      }),
    ],
  });

  expect(success).toBe(true);
  expect(messages.length).toBe(6);
});

zevals.dynamicMessage(({ messages }) => {
  const lastAssistant = messages.findLast((m) => m.role === 'assistant');

  return {
    role: 'user',
    content: lastAssistant?.content.includes('name') ? 'Jane Doe' : 'MG-12345678',
  };
}),
zevals.agentResponse(),

zevals.message({
  role: 'user',
  content: 'I have attached the letter',
  context: { attachments: [{ url: 'https://example.com/letter.pdf' }] },
});

zevals.aiAssertion({ judge, prompt: 'The agent transferred the chat', scope: 'lastAssistantTurn' });

zevals.Criterion.scoped({ criterion: myCriterion, scope: 'lastAssistantTurn' });

import zevals from '@zevals/core';

const client = zevals.openRouterJevClient(); // reads OPENROUTER_API_KEY

zevals.aiEval(
  zevals.aiAssertion({
    judge: client,
    prompt: 'The agent transferred the chat to a human',
    scope: 'lastAssistantTurn',
    // Jev-only, optional:
    threshold: 0.5,
    criteria: { true: 'A handoff was performed', false: 'The agent only promised a handoff' },
    explainFailures: judge, // any zevals Judge; called on failures only
  }),
);

const client = zevals.vercelJevClient(); // reads AI_GATEWAY_API_KEY
// or
const client = zevals.cloudflareJevClient({ model: 'typesafe/jev' });

const { success, iterations, failedIterations } = await zevals.repeat({
  count: 5,
  scenario: async () => ({
    agent: await makeFreshAgent(),
    segments: [
      /* ... */
    ],
  }),
});

expect(success).toBe(true); // All 5 iterations must pass

const res = await zevals.evaluate({ agent, segments });

expect(res.success, zevals.formatTranscript(res)).toBe(true);
// [user] Cancel my order
// [assistant] Cancelling now
//   [tool call] cancel_order({"orderId":"42"}) => "not_found"
// [eval FAIL] Order cancelled — The order was never cancelled

import { openai } from '@ai-sdk/openai';
import zevals from '@zevals/core';
import { vercelZEvalsAgent, vercelZEvalsJudge } from '@zevals/vercel';
import { generateText } from 'ai';

test('Vercel AI SDK integration', { timeout: 10000 }, async () => {
  const model = openai.chat('gpt-4.1-mini');

  const agent = vercelZEvalsAgent({
    runnable({ messages }) {
      // Put any agent logic here
      return generateText({ model, messages });
    },
  });

  const judge = vercelZEvalsJudge({ model });

  const { messages, success } = await zevals.evaluate({
    agent,
    segments: [
      zevals.message({ role: 'user', content: 'Hello' }),

      zevals.agentResponse(),

      zevals.aiEval(zevals.aiAssertion({ judge, prompt: 'The assistant greeted the user' })),
    ],
  });

  expect(success).toBe(true);
  expect(messages).toHaveLength(2);
});