llmgw/internal/store/eval.go

package store

import (
	"context"
	"database/sql"
	"errors"
	"time"
)

// EvalRun is one quality suite run against a model, as `llmgw ctl eval`
// and the gate on an update keep it.
type EvalRun struct {
	ID string `json:"id"`
	// Model is the model the run is about; Target the catalog model that
	// answered, when that was another — its canary, or the new weights an
	// update tried before moving to them.
	Model  string `json:"model"`
	Target string `json:"target,omitempty"`
	// Repo and Revision are where the weights that answered came from,
	// when that is known.
	Repo     string `json:"repo,omitempty"`
	Revision string `json:"revision,omitempty"`
	// Suite is its name, SuiteHash the sha256 of its text.
	Suite     string    `json:"suite"`
	SuiteHash string    `json:"suite_hash"`
	At        time.Time `json:"at"`
	TookMs    int64     `json:"took_ms"`
	// Score is the cases' scores weighed, 0 to 1; Passed whether it
	// reached Threshold.
	Score     float64 `json:"score"`
	Threshold float64 `json:"threshold"`
	Passed    bool    `json:"passed"`
	// Cases were asked; Below scored under half; Errors got no answer.
	Cases  int    `json:"cases"`
	Below  int    `json:"below"`
	Errors int    `json:"errors"`
	Judge  string `json:"judge,omitempty"`
	By     string `json:"by,omitempty"`
	// Job is the update or promotion the run gated, if one did.
	Job string `json:"job,omitempty"`
	// Error is why the run could not be scored at all.
	Error   string     `json:"error,omitempty"`
	Results []EvalCase `json:"results,omitempty"`
}

// EvalCase is one case of a run: how its answer scored, and why.
type EvalCase struct {
	ID     string  `json:"id"`
	Kind   string  `json:"kind"`
	Weight float64 `json:"weight"`
	Score  float64 `json:"score"`
	Passed bool    `json:"passed"`
	Answer string  `json:"answer,omitempty"`
	Detail string  `json:"detail,omitempty"`
	Error  string  `json:"error,omitempty"`
	TookMs int64   `json:"took_ms"`
}

// PutEvalRun keeps a finished run and its cases.
func (s *DB) PutEvalRun(ctx context.Context, r EvalRun) error {
	return s.inTx(ctx, func(q conn) error {
		if _, err := q.ExecContext(ctx, `INSERT INTO eval_runs (id, model, target, repo, revision, suite,
			suite_hash, started_at, took_ms, score, threshold, passed, cases, below, errors, judge, actor, job, error)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
			r.ID, r.Model, r.Target, r.Repo, r.Revision, r.Suite, r.SuiteHash, millis(r.At), r.TookMs,
			r.Score, r.Threshold, boolInt(r.Passed), r.Cases, r.Below, r.Errors, r.Judge, r.By, r.Job, r.Error); err != nil {
			return err
		}
		for i, c := range r.Results {
			if _, err := q.ExecContext(ctx, `INSERT INTO eval_cases (run_id, idx, case_id, kind, weight, score,
				passed, answer, detail, error, took_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
				r.ID, i, c.ID, c.Kind, c.Weight, c.Score, boolInt(c.Passed), c.Answer, c.Detail, c.Error,
				c.TookMs); err != nil {
				return err
			}
		}
		return nil
	})
}

func boolInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

const evalRunColumns = `id, model, target, repo, revision, suite, suite_hash, started_at, took_ms, score,
	threshold, passed, cases, below, errors, judge, actor, job, error`

func scanEvalRun(row interface{ Scan(...any) error }) (EvalRun, error) {
	var r EvalRun
	var at int64
	var passed int
	err := row.Scan(&r.ID, &r.Model, &r.Target, &r.Repo, &r.Revision, &r.Suite, &r.SuiteHash, &at, &r.TookMs,
		&r.Score, &r.Threshold, &passed, &r.Cases, &r.Below, &r.Errors, &r.Judge, &r.By, &r.Job, &r.Error)
	r.At, r.Passed = fromMillis(at), passed == 1
	return r, err
}

// EvalRuns are a model's runs, newest first and without their cases; every
// model's for an empty name.
func (s *DB) EvalRuns(ctx context.Context, model string, limit int) ([]EvalRun, error) {
	if limit <= 0 || limit > 1000 {
		limit = 20
	}
	q := `SELECT ` + evalRunColumns + ` FROM eval_runs`
	var args []any
	if model != "" {
		q += ` WHERE model = ? OR target = ?`
		args = append(args, model, model)
	}
	q += ` ORDER BY started_at DESC, id DESC LIMIT ?`
	args = append(args, limit)
	return queryAll(ctx, s.db, q, args, func(rows *sql.Rows) (EvalRun, error) { return scanEvalRun(rows) })
}

// EvalRunByID is one run with its cases; ErrNotFound when there is none.
func (s *DB) EvalRunByID(ctx context.Context, id string) (EvalRun, error) {
	r, err := scanEvalRun(s.db.QueryRowContext(ctx, `SELECT `+evalRunColumns+` FROM eval_runs WHERE id = ?`, id))
	if errors.Is(err, sql.ErrNoRows) {
		return EvalRun{}, ErrNotFound
	}
	if err != nil {
		return EvalRun{}, err
	}
	r.Results, err = queryAll(ctx, s.db, `SELECT case_id, kind, weight, score, passed, answer, detail, error, took_ms
		FROM eval_cases WHERE run_id = ? ORDER BY idx`, []any{id}, func(rows *sql.Rows) (EvalCase, error) {
		var c EvalCase
		var passed int
		err := rows.Scan(&c.ID, &c.Kind, &c.Weight, &c.Score, &passed, &c.Answer, &c.Detail, &c.Error, &c.TookMs)
		c.Passed = passed == 1
		return c, err
	})
	return r, err
}