평가 (Evaluation)

# Verified against: 표준 평가 패턴
# Verified at: 2026-06-02
from dataclasses import dataclass

@dataclass
class Sample:
  input: str
  expected: str

def exact_match(pred: str, expected: str) -> float:
  return 1.0 if pred.strip() == expected.strip() else 0.0

def evaluate(samples: list[Sample], run) -> dict:
  correct = 0
  for s in samples:
      pred = run(s.input)
      correct += exact_match(pred, s.expected)
  return {"accuracy": correct / len(samples), "n": len(samples)}

samples = [
  Sample("2+2=?", "4"),
  Sample("KST 시간대 오프셋?", "+9"),
]
print(evaluate(samples, lambda x: "4" if "2+2" in x else "+9"))

// Verified against: 표준 평가 패턴
// Verified at: 2026-06-02
type Sample = { input: string; expected: string }

const exactMatch = (pred: string, expected: string) =>
pred.trim() === expected.trim() ? 1 : 0

export function evaluate(samples: Sample[], run: (q: string) => string) {
let correct = 0
for (const s of samples) correct += exactMatch(run(s.input), s.expected)
return { accuracy: correct / samples.length, n: samples.length }
}

const samples: Sample[] = [
{ input: '2+2=?', expected: '4' },
{ input: 'KST 시간대 오프셋?', expected: '+9' },
]
console.log(evaluate(samples, (x) => (x.includes('2+2') ? '4' : '+9')))

# Verified against: 본문 LLM-as-Judge 패턴
# Verified at: 2026-06-02
import json
from anthropic import Anthropic

client = Anthropic()

JUDGE = """엄격한 채점자다. 0~3점으로 채점하라.
질문: {q}
응답: {a}
JSON으로만: {{"accuracy": n, "clarity": n, "safety": n}}"""

def judge_once(q: str, a: str) -> dict:
  r = client.messages.create(
      model="claude-sonnet-4-6",
      max_tokens=200,
      messages=[{"role": "user", "content": JUDGE.format(q=q, a=a)}],
  )
  return json.loads(r.content[0].text)

def judge_ensemble(q: str, a: str, n: int = 3) -> dict:
  rounds = [judge_once(q, a) for _ in range(n)]
  keys = ["accuracy", "clarity", "safety"]
  return {k: sum(r[k] for r in rounds) / n for k in keys}

// Verified against: 본문 LLM-as-Judge 패턴
// Verified at: 2026-06-02
import Anthropic from '@anthropic-ai/sdk'

const client = new Anthropic()
const JUDGE = (q: string, a: string) =>
`엄격한 채점자다. 0~3점으로 채점하라.
질문: ${q}
응답: ${a}
JSON으로만: {"accuracy": n, "clarity": n, "safety": n}`

async function judgeOnce(q: string, a: string) {
const r = await client.messages.create({
  model: 'claude-sonnet-4-6',
  max_tokens: 200,
  messages: [{ role: 'user', content: JUDGE(q, a) }],
})
const block = r.content[0]
return JSON.parse(block.type === 'text' ? block.text : '{}')
}

export async function judgeEnsemble(q: string, a: string, n = 3) {
const rounds = await Promise.all(Array.from({ length: n }, () => judgeOnce(q, a)))
const keys = ['accuracy', 'clarity', 'safety'] as const
return Object.fromEntries(
  keys.map((k) => [k, rounds.reduce((acc, r) => acc + r[k], 0) / n]),
)
}

# Verified against: https://www.promptfoo.dev/docs/getting-started/
# Verified at: 2026-06-02
# promptfoo는 Node CLI. Python 코드에서는 subprocess로 호출.
import subprocess, json

def run_promptfoo(config: str = "promptfooconfig.yaml") -> dict:
  out = subprocess.run(
      ["npx", "promptfoo", "eval", "-c", config, "-o", "json"],
      capture_output=True, text=True, check=True,
  )
  return json.loads(out.stdout)

# CI에서: results = run_promptfoo()
# assert results["pass_rate"] > 0.9

// Verified against: https://www.promptfoo.dev/docs/getting-started/
// Verified at: 2026-06-02
import { execSync } from 'node:child_process'

export function runPromptfoo(config = 'promptfooconfig.yaml') {
const out = execSync(`npx promptfoo eval -c ${config} -o json`).toString()
return JSON.parse(out)
}

// const results = runPromptfoo()
// if (results.passRate < 0.9) process.exit(1)

# Verified against: 본문 회귀 패턴
# Verified at: 2026-06-02
import sys, json
from pathlib import Path

def main():
  results = json.loads(Path("eval-result.json").read_text())
  baseline = json.loads(Path("eval-baseline.json").read_text())
  delta = results["accuracy"] - baseline["accuracy"]
  print(f"delta: {delta:+.3f}")
  if delta < -0.02:
      print("FAIL: 정확도가 2%p 이상 하락", file=sys.stderr)
      sys.exit(1)

if __name__ == "__main__":
  main()

// Verified against: 본문 회귀 패턴
// Verified at: 2026-06-02
import fs from 'node:fs'

const results = JSON.parse(fs.readFileSync('eval-result.json', 'utf8'))
const baseline = JSON.parse(fs.readFileSync('eval-baseline.json', 'utf8'))
const delta = results.accuracy - baseline.accuracy
console.log(`delta: ${delta >= 0 ? '+' : ''}${delta.toFixed(3)}`)
if (delta < -0.02) {
console.error('FAIL: 정확도가 2%p 이상 하락')
process.exit(1)
}