llmgw/internal/control/modelserve/gate.go

package modelserve

import (
	"context"
	"errors"
	"fmt"

	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// A gate: new weights asked a quality suite before they replace a model's,
// and refused below its score. A promotion asks the canary, which is
// already running; an update starts the new weights as a canary given none
// of the model's requests, asks it, and promotes it or takes it out — so
// the old weights serve until the new have passed.

// Evaluator runs quality suites: the control plane's eval service.
type Evaluator interface {
	// Check says whether a suite can be run, before anything is started
	// for it.
	Check(model string, r eval.Request) error
	// Run asks a model a suite and waits for its score.
	Run(ctx context.Context, model string, r eval.Request, by, job string) (store.EvalRun, error)
}

// Change is how an update or a promotion is held to quality.
type Change struct {
	// Eval is a suite the new weights must pass first. Nil takes the
	// model's own (eval: in its catalog entry), if it has one.
	Eval *eval.Request `json:"eval,omitempty"`
	// SkipEval moves without the model's own suite; the job says so.
	SkipEval bool `json:"skip_eval,omitempty"`
}

// gateFor is the suite a change is held to, and whether one was skipped.
func gateFor(m catalog.Model, ch Change) (r *eval.Request, skipped bool) {
	switch {
	case ch.Eval != nil:
		return ch.Eval, false
	case m.Eval == nil:
		return nil, false
	case ch.SkipEval:
		return nil, true
	}
	return &eval.Request{Suite: m.Eval.Suite, MinScore: m.Eval.MinScore, Judge: m.Eval.Judge}, false
}

// checkGate refuses, before a job starts, a gate that cannot be run.
func (s *Server) checkGate(name string, r *eval.Request) error {
	if r == nil {
		return nil
	}
	if s.deps.Eval == nil {
		return errors.New("this control plane cannot run quality suites: it routes no requests")
	}
	if err := s.deps.Eval.Check(name, *r); err != nil {
		return fmt.Errorf("the quality suite: %w", err)
	}
	return nil
}

// gate asks the new weights, running as target, a suite, and says whether
// they may replace the old; when they may not, the job fails saying why,
// and what refused does about the new weights, which it says too.
func (s *Server) gate(ctx context.Context, j *Job, target string, r *eval.Request, skipped bool, refused func() string) bool {
	if skipped {
		s.step(j, "eval", Skipped, "the model's quality suite was skipped by "+orUnknown(j.Request.By))
		return true
	}
	if r == nil {
		return true
	}
	s.step(j, "eval", Running, fmt.Sprintf("asking %s the cases of %s", target, r.Named()))
	run, err := s.deps.Eval.Run(ctx, target, *r, j.Request.By, j.ID)
	if err != nil {
		s.fail(j, "eval", fmt.Errorf("%w; %s", err, refused()))
		return false
	}
	run.Results = nil
	s.mu.Lock()
	j.Eval = &run
	s.mu.Unlock()
	detail := fmt.Sprintf("%s scored %.2f on %s, %d of %d cases passed; at least %.2f is asked", target,
		run.Score, run.Suite, run.Cases-run.Below, run.Cases, run.Threshold)
	if run.Error != "" {
		detail += "; " + run.Error
	}
	if !run.Passed {
		s.fail(j, "eval", fmt.Errorf("%s; %s. `llmgw ctl eval --run %s` shows each case", detail, refused(), run.ID))
		return false
	}
	s.step(j, "eval", Done, detail)
	return true
}