local ef = require("evalframe")

local s = ef.suite "math" {
  provider = ef.providers.claude_cli { model = "haiku" },

  ef.bind { ef.graders.exact_match, weight = 0.5 },
  ef.bind { ef.graders.contains,    weight = 0.5 },

  cases = {
    ef.case "add" { input = "What is 2+2?",  expected = "4",  tags = { "basic" } },
    ef.case "mul" { input = "What is 7*8?",  expected = "56", tags = { "basic" } },
    ef.case "div" { input = "What is 10/3?", expected = "3.33", tags = { "basic" } },
  },
}

local report = s:run()
print(report:summary())

-- Minimal
ef.case { input = "What is 2+2?", expected = "4" }

-- Named with tags
ef.case "capital" { input = "Capital of Japan?", expected = {"Tokyo"}, tags = {"geo"} }

-- Open-ended (no expected)
ef.case { input = "Write a poem about the sea" }

-- Load from file
local cases = ef.load_cases("cases/math.lua")

-- cases/math.lua
return {
  { input = "2+2?",  expected = "4",  tags = {"basic"} },
  { input = "3*3?",  expected = "9",  tags = {"basic"} },
}

local g = ef.graders

g.exact_match    -- response.text == expected (any of)
g.contains       -- response.text contains expected
g.starts_with    -- response.text starts with expected
g.regex          -- response.text matches Lua pattern (case.context.pattern)
g.json_valid     -- response.text is valid JSON
g.not_empty      -- response.text is non-empty
g.length         -- returns #response.text (pair with linear scorer)
g.latency        -- returns response.latency_ms (pair with inverse scorer)

local no_apology = ef.grader "no_apology" {
  check = function(resp, case)
    return not resp.text:lower():find("sorry")
  end
}

local llm = ef.llm_graders
local provider = ef.providers.claude_cli { model = "sonnet" }

-- Rubric-based rating (returns number 1-5)
local quality = llm.rubric("Rate the answer for accuracy and clarity, 1-5", {
  provider = provider,
})
ef.bind { quality, ef.scorers.linear_1_5 }

-- Yes/No question (returns bool)
local polite = llm.yes_no("Is the response polite and professional?", {
  provider = provider,
})
ef.bind { polite }

-- Factuality check (returns number 1-5)
local factual = llm.factuality({ provider = provider })
ef.bind { factual, ef.scorers.linear_1_5 }

local s = ef.scorers

-- Built-in
s.bool            -- true->1.0, false->0.0
s.linear_1_5      -- [1,5] -> [0,1]
s.linear_1_10     -- [1,10] -> [0,1]
s.linear_0_100    -- [0,100] -> [0,1]
s.pass_50         -- >=0.5 -> 1.0
s.pass_80         -- >=0.8 -> 1.0
s.inverse_linear  -- lower is better (latency, token count)

-- Custom
ef.scorer "log_scale" {
  score = function(v)
    return math.log(v + 1) / math.log(11)
  end
}

-- Grader only (default bool scorer)
ef.bind { ef.graders.exact_match }

-- Grader + Scorer
ef.bind { ef.graders.length, ef.scorers.inverse_linear }

-- Weighted (default weight = 1.0)
ef.bind { ef.graders.exact_match, weight = 0.7 }
ef.bind { ef.graders.contains,    weight = 0.3 }

-- Order doesn't matter (type-dispatched)
ef.bind { ef.scorers.linear_1_5, some_grader }

-- Claude Code CLI
local p = ef.providers.claude_cli()
local p = ef.providers.claude_cli { model = "haiku" }
local p = ef.providers.claude_cli {
  model  = "sonnet",
  system = "You are a math tutor. Answer concisely.",
}

-- Mock (testing)
local p = ef.providers.mock.static("always this")
local p = ef.providers.mock.map({ ["2+2"] = "4", ["3*3"] = "9" })
local p = ef.providers.mock.fn(function(input) return "Echo: " .. input end)

-- Recording mock (captures inputs)
local p, log = ef.providers.mock.recording({ "a", "b", "c" })
p("hello")
print(log.calls[1])  -- "hello"

-- Custom provider (any function with the right signature)
local p = function(input)
  local resp = my_api_call(input)
  return { text = resp.content, model = "my-model" }
end

local s = ef.suite "my_eval" {
  provider = provider,
  ef.bind { ef.graders.exact_match, weight = 0.5 },
  ef.bind { ef.graders.contains,    weight = 0.5 },
  cases = {
    ef.case { input = "...", expected = "..." },
  },
}

local report = s:run()

-- Summary text
print(report:summary())

-- Programmatic access
report.aggregated.total       -- 50
report.aggregated.passed      -- 42
report.aggregated.pass_rate   -- 0.84
report.aggregated.pass_at_1   -- 0.84
report.aggregated.pass_at_5   -- 0.97
report.aggregated.scores.mean -- 0.82
report.aggregated.ci_95       -- { lower = 0.76, upper = 0.88 }
report.aggregated.by_tag      -- { math = { pass = 20, fail = 2, rate = 0.91 } }

-- Failures
for _, f in ipairs(report.failures) do
  print(report.format_result(f))
end

local configs = ef.variants {
  base = { temperature = 0.7 },

  ef.vary "model" {
    { model = "gpt-4",  name = "gpt4" },
    { model = "claude", name = "claude" },
  },

  ef.vary "temp" {
    { temperature = 0.0, name = "cold" },
    { temperature = 1.0, name = "hot" },
  },

  mode = "cross",  -- "cross" (default) or "zip"
}
-- → 4 variants: gpt4_cold, gpt4_hot, claude_cold, claude_hot

for _, cfg in ipairs(configs) do
  local report = ef.suite(cfg.name) {
    provider = make_provider(cfg),
    ef.bind { ef.graders.exact_match },
    cases = my_cases,
  }:run()
  print(report:summary())
end

local sw = require("evalframe.swarm")

-- 1. Environment
local env = sw.env "troubleshooting" {
  scenario = "memory_leak",
  services = { "user-service", "db-service" },
}

-- 2. Action space
local actions = sw.actions {
  sw.action "CheckStatus"    { description = "Check service health" },
  sw.action "ReadLogs"       { description = "Read service logs" },
  sw.action "RestartService" { description = "Restart a service" },
}

-- 3. Swarm config
local swarm_cfg = sw.swarm { workers = 3, max_ticks = 20, strategy = "ucb1" }

-- 4. Runner → Provider adapter
local provider = sw.provider(my_runner, {
  env = env, actions = actions, swarm = swarm_cfg,
})

-- 5. Suite with Swarm graders
local report = ef.suite "swarm_eval" {
  provider = provider,

  ef.bind { sw.graders.completed, weight = 0.3 },
  ef.bind { sw.graders.efficiency { max_ticks = 20, optimal_ticks = 5 }, weight = 0.2 },
  ef.bind { sw.graders.action_sequence { "CheckStatus", "ReadLogs", "RestartService" } },
  ef.bind { sw.graders.metric("throughput", { min = 1.0 }) },

  cases = { ef.case { input = "Diagnose the memory leak" } },
}:run()

local function my_runner(config)
  -- config.input, config.env, config.actions, config.swarm
  -- ... your tick loop / LLM orchestration here ...
  return {
    text        = "Resolved",
    success     = true,
    ticks       = 12,
    termination = "success",  -- "success" | "failure" | "timeout"
    actions     = {
      { tick = 1, worker = "w-0", action = "CheckStatus", result = "ok" },
    },
    metrics     = { task_completion = 1.0 },
  }
end

local custom = ef.grader "my_check" {
  check = function(resp, _case)
    return resp.success == true and resp.ticks < 15
  end,
}

-- Available fields:
-- resp.success      boolean
-- resp.ticks        number
-- resp.termination  "success" | "failure" | "timeout"
-- resp.actions      { { tick, worker, action, result?, ... }, ... }
-- resp.metrics      { [name] = value }
-- resp.text         string
-- resp.latency_ms   number | nil

local analysis = sw.analysis

-- Action sequence frequency (n-gram, per-trace deduplicated)
local freq = analysis.action_sequences(traces, 2)
-- { ["Check,Read"] = { count = 5, success = 3, rate = 0.6 }, ... }

-- Convergence: tick count distribution across traces
local conv = analysis.convergence(traces)
-- conv.mean, conv.std_dev, conv.ci_lower, conv.ci_upper

-- Exploration efficiency (per trace)
local eff = analysis.exploration_efficiency(trace)
-- { total = 15, unique = 8, unique_ratio = 0.53, duplicate_rate = 0.47 }

-- Multi-worker coordination (per trace)
local coord = analysis.worker_coordination(trace)
-- { worker_count = 3, overlap_rate = 0.2, workers = { ... } }

-- Action validity with user-defined predicate
local qual = analysis.action_validity(trace, function(a)
  return a.result ~= "parse_error"
end)
-- { total = 20, valid = 18, rate = 0.9 }