llmgw/internal/cli/serverinit_judges.go

package cli

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
	"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)

// Machines of the judges' own. Under load on a 4-core control plane that was
// also a node, 38 of 60 requests were refused for want of a judgment; with the
// judges on a node of their own the same test refused 2. A judge machine is
// added as a node carrying judgeRoleLabel, and the judge's profile selects
// that label, so nothing else is placed beside it and it on nothing else.

// judgeRoleLabel marks a node as the judges'.
const (
	judgeRoleLabel = placement.RoleLabel
	judgeRoleValue = "judge"
)

// judgeMachine is a machine for the judges: user@host, and the connection the
// setup screen already made to it, when it made one.
type judgeMachine struct {
	target string
	remote onboard.Remote
}

// ensureJudgeNodes adds each judge machine as a node labelled for the
// judges, and says how many cores the smallest of them has.
func (in *serverInit) ensureJudgeNodes(ctx context.Context) error {
	if len(in.judgeMachines) == 0 {
		return nil
	}
	na := nodeAdd{serverCfg: in.configPath(), plain: true}
	na.opts.PKI = in.pkiDir
	na.opts.NoFirewall = in.noFirewall
	if err := na.prepare(); err != nil {
		return err
	}
	in.judgeCPUs = nil
	for _, jm := range in.judgeMachines {
		o := na.target(jm.target)
		fmt.Fprintf(in.out, "  ..    adding %s as a node for the judges\n", o.Addr)
		r := onboard.New(o, nil)
		if jm.remote != nil {
			r.Remote = jm.remote
		}
		res, err := r.Run(ctx)
		if err != nil {
			return fmt.Errorf("%s: %w", o.Addr, err)
		}
		if err := na.watch(ctx, res.Node, func(onboard.Event) {}); err != nil && err != errNoAdmin {
			return fmt.Errorf("%s: %w", res.Node, err)
		}
		if out, err := r.Remote.Run(ctx, "nproc", nil); err == nil {
			if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil {
				in.judgeCPUs = append(in.judgeCPUs, n)
			}
		}
		if _, err := changeStoredSettings(in.configPath(), "a node for the judges, from init", func(doc *settings.Doc) error {
			return doc.SetYAML([]string{"agents", "nodes", res.Node, "labels", judgeRoleLabel}, judgeRoleValue)
		}); err != nil {
			return fmt.Errorf("labelling %s for the judges: %w", res.Node, err)
		}
		in.done("%s is a node for the judges alone (%s: %s)", res.Node, judgeRoleLabel, judgeRoleValue)
	}
	return nil
}

// judgePlacement is the judge's cores and where it may run: on the judge
// machines when there are some, by init's rule over their cores; otherwise
// on any node, with this machine's plan.
func (in *serverInit) judgePlacement() (cores int, selector map[string]string) {
	if len(in.judgeMachines) == 0 {
		return in.cpuPlan().judge, nil
	}
	return refitCores(in.judgeFor().Replicas, in.judgeCPUs), map[string]string{judgeRoleLabel: judgeRoleValue}
}

// judgeMachineFlag is --judge-node, given again for each machine.
type judgeMachineFlag struct{ to *[]judgeMachine }

func (f judgeMachineFlag) String() string {
	if f.to == nil {
		return ""
	}
	var out []string
	for _, m := range *f.to {
		out = append(out, m.target)
	}
	return strings.Join(out, ",")
}

func (f judgeMachineFlag) Set(v string) error {
	if strings.TrimSpace(v) == "" {
		return fmt.Errorf("--judge-node wants user@host")
	}
	*f.to = append(*f.to, judgeMachine{target: strings.TrimSpace(v)})
	return nil
}

func (judgeMachineFlag) Type() string { return "user@host" }

// judgeReadyWait is how long init waits for its judge to load before timing
// it: the weights come from the mirror, and llama.cpp loads in seconds.
const judgeReadyWait = 15 * time.Minute

// checkJudge times the judge once it is up, and keeps what it found for the
// closing text: a machine too slow for its judge is said before its first
// refused request, not after. The budget is not changed without asking.
func (in *serverInit) checkJudge(ctx context.Context) {
	c := in.adminAPI()
	deadline := time.Now().Add(judgeReadyWait)
	for {
		var judges []adminapi.JudgeView
		if err := c.get(ctx, "/api/v1/guard/judges", &judges); err == nil {
			ready := false
			for _, j := range judges {
				ready = ready || (j.Model == guardModelName && j.Ready > 0)
			}
			if ready {
				break
			}
		}
		if time.Now().After(deadline) || ctx.Err() != nil {
			in.judgeNote = "  !!    the judge was not up to be timed; once it is: llmgw ctl guard judge check\n"
			return
		}
		time.Sleep(10 * time.Second)
	}
	fmt.Fprintf(in.out, "  ..    timing the judge\n")
	v, err := checkJudge(ctx, c, guardModelName, 0)
	if err != nil {
		in.judgeNote = "  !!    the judge could not be timed (" + err.Error() + "); later: llmgw ctl guard judge check\n"
		return
	}
	if v.Verdict == "ok" {
		in.done("the judge reads a prompt in %.1f s, of the %.1f s a request waits; about %.1f requests a second",
			float64(v.SlowestMs)/1000, float64(v.BudgetMs)/1000, v.PerSecond/2)
		return
	}
	in.judgeNote = "The judge is " + v.Verdict + " on this cluster:\n" + indent(judgeCheckText(v), "  ") + "\n"
}