Merge features-c2: the docs portal and its links in English and Turkish

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:15:25 +02:00
commit e8a62b23b7
201 files changed
+26276 -630

No files matched your search

+85 -9
View File
@@ -23,6 +23,8 @@ import (
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"golang.org/x/net/http2"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
@@ -33,6 +35,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -419,6 +422,17 @@ func int32Slice(v []int) []int32 {
return out
}
func int32Ints(v []int32) []int {
out := make([]int, len(v))
for i, x := range v {
out[i] = int(x)
}
return out
}
// tracerName names the spans the agent opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent"
// scan refreshes the node's hardware inventory.
func (a *Agent) scan(ctx context.Context) error {
// Carried from the agent's own configuration rather than duplicated into
@@ -786,15 +800,23 @@ func (a *Agent) removeWeights(paths []string) {
}
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
// The control plane's span that sent this, when it is tracing: what is
// done here for a start or a stop goes into the same trace.
tctx := obs.Extract(ctx, msg.GetTrace())
switch m := msg.Msg.(type) {
case *llmgwv1.ControlMessage_Start:
go a.startShard(ctx, m.Start, out)
go a.startShard(tctx, m.Start, out)
case *llmgwv1.ControlMessage_Stop:
go a.stopShard(ctx, m.Stop.GetShardId(), out)
go a.stopShard(tctx, m.Stop.GetShardId(), out)
case *llmgwv1.ControlMessage_Drain:
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
_, span := obs.Tracer(tracerName).Start(tctx, "drain shard",
trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(attribute.String("llmgw.shard", m.Drain.GetShardId())))
err := a.sup.Drain(m.Drain.GetShardId())
if err != nil {
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
}
obs.End(span, err)
case *llmgwv1.ControlMessage_RemoveWeights:
a.removeWeights(m.RemoveWeights.GetPaths())
case *llmgwv1.ControlMessage_Provision:
@@ -927,6 +949,25 @@ func localAddr(ip netip.Addr) bool {
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
// This node's whole part of the start, from the command arriving to the
// engine answering, beneath the control plane's span that sent it. The
// steps — weights, launch, readiness — are its children.
ctx, span := obs.Tracer(tracerName).Start(ctx, "start shard",
trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("llmgw.node", a.cfg.NodeID),
attribute.String("llmgw.replica", replicaID),
attribute.String("llmgw.shard", shardID),
attribute.Int("llmgw.rank", int(m.GetRank())),
attribute.String("llmgw.model", m.GetProfile().GetServedModelName()),
attribute.String("llmgw.profile", m.GetProfile().GetName()),
attribute.String("llmgw.engine", m.GetProfile().GetEngine()),
attribute.String("llmgw.gpu_class", m.GetProfile().GetGpuClass()),
attribute.IntSlice("llmgw.gpus", int32Ints(m.GetGpuIndices())),
))
var failed error
defer func() { obs.End(span, failed) }()
a.mu.Lock()
a.shardReplica[shardID] = replicaID
a.mu.Unlock()
@@ -939,6 +980,9 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
}
}
report := func(st supervisor.State, errMsg string, tail []string) {
if st == supervisor.Failed {
failed = errors.New(errMsg)
}
send(&llmgwv1.ShardStatus{
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
})
@@ -1109,9 +1153,18 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
delete(a.shardReplica, shardID)
a.mu.Unlock()
if err := a.sup.Stop(ctx, shardID); err != nil {
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop shard",
trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("llmgw.node", a.cfg.NodeID),
attribute.String("llmgw.replica", replicaID),
attribute.String("llmgw.shard", shardID),
))
err := a.sup.Stop(ctx, shardID)
if err != nil {
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
}
obs.End(span, err)
a.sup.Remove(shardID)
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
@@ -1202,18 +1255,20 @@ const (
//
// The interface NCCL bootstraps over is this node's, not the cluster's: the
// control plane sends one name for every node, and a node whose management
// network is on ens3 rather than eth0 would have NCCL bind nothing. And on
// RoCE, which GID entry routes is a property of these adapters.
// network is on ens3 rather than eth0 would have NCCL bind nothing. On RoCE,
// which GID entry routes is a property of these adapters. And a MIG instance
// is opened by its UUID, which only this node knows.
func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
if a.cfg.MgmtInterface != "" {
spec.MgmtInterface = a.cfg.MgmtInterface
}
a.mu.Lock()
hcas, gpus := a.inv.HCAs, a.inv.GPUs
a.mu.Unlock()
spec.GPUUUIDs = migUUIDs(gpus, spec.GPUIndices)
if len(spec.HCAs) == 0 {
return
}
a.mu.Lock()
hcas := a.inv.HCAs
a.mu.Unlock()
want := map[string]bool{}
for _, h := range spec.HCAs {
want[h] = true
@@ -1232,6 +1287,27 @@ func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
}
}
// migUUIDs names a shard's devices by UUID when one of them is a MIG
// instance, and is nil otherwise: whole cards keep being given by index, as
// they always were. A device this node does not have is left to fail where
// it always has, in the engine, rather than guessed at here.
func migUUIDs(gpus []discovery.GPU, indices []int) []string {
var out []string
mig := false
for _, i := range indices {
k := slices.IndexFunc(gpus, func(g discovery.GPU) bool { return g.Index == i })
if k < 0 || gpus[k].UUID == "" {
return nil
}
mig = mig || gpus[k].IsMIG()
out = append(out, gpus[k].UUID)
}
if !mig {
return nil
}
return out
}
// ResolveManagement is the address and interface this node's agent would use,
// found the way normalize finds them, for a tool that reports them.
func ResolveManagement(c Config) (addr, iface string) {
+8
View File
@@ -14,6 +14,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -94,6 +95,13 @@ type Config struct {
Engines engines.Config `koanf:"engines"`
Superv supervisor.Config `koanf:"supervisor"`
// Tracing sends this node's part of each replica start — the weights
// fetched, the engine launched, the wait for it to answer — to an
// OpenTelemetry collector, inside the control plane's trace of that
// start. Give it the collector server.yaml's tracing names. Off unless
// an endpoint is set here or in OTEL_EXPORTER_OTLP_ENDPOINT.
Tracing obs.Tracing `koanf:"tracing"`
Version string
}
+29 -2
View File
@@ -47,6 +47,11 @@ type GPU struct {
Fabric *GPUFabric `json:"fabric,omitempty"`
// Health carries anything that should keep the GPU out of placement.
Health GPUHealth `json:"health"`
// MIG is set when this is a MIG instance rather than a whole card: its
// Index is then MIGIndex's, its UUID the instance's (MIG-…), and its
// memory the instance's share. Utilisation stays unreported, because no
// driver measures it per instance.
MIG *MIGDevice `json:"mig,omitempty"`
}
// GPUHealth records conditions that make a GPU unsafe to schedule onto.
@@ -94,8 +99,30 @@ func (g GPU) Class() string {
}
// Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card.
gb := int(math.Round(float64(g.MemoryTotalMB) / 1024))
return fmt.Sprintf("%s-%dg", model, gb)
// A MIG instance is named from its card's memory and then its profile,
// so two instances are one class only when both the card and the slice
// are the same.
mem := g.MemoryTotalMB
if g.MIG != nil {
mem = g.MIG.ParentMemoryTotalMB
}
gb := int(math.Round(float64(mem) / 1024))
class := fmt.Sprintf("%s-%dg", model, gb)
if g.MIG != nil {
return migClass(class, g.MIG.Profile)
}
return class
}
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
// call and lowercase from another, and sometimes with a shortened domain.
func normalizeBusID(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
if i := strings.Index(s, ":"); i > 0 {
s = strings.TrimLeft(s[:i], "0") + s[i:]
}
return strings.TrimPrefix(s, ":")
}
// MarshalJSON adds the derived class, so a consumer of the JSON does not have
+77 -11
View File
@@ -14,7 +14,6 @@ package discovery
import (
"context"
"fmt"
"strings"
"github.com/NVIDIA/go-nvml/pkg/nvml"
)
@@ -83,11 +82,18 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
}
assessHealth(&g)
gpus = append(gpus, g)
// A card in MIG mode is its instances, not itself: CUDA will not
// open the whole card while MIG is on.
devs, _ := migFromNVML(g, nvmlCard{dev})
gpus = append(gpus, devs...)
}
// A second pass, because a link's remote end may be a GPU enumerated later.
// A second pass, because a link's remote end may be a GPU enumerated
// later. MIG instances have no NVLink of their own to use.
for i := range gpus {
if gpus[i].IsMIG() {
continue
}
dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index)
if ret != nvml.SUCCESS {
continue
@@ -136,13 +142,73 @@ func busIDString(b []int8) string {
return string(buf)
}
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
// call and lowercase from another, and sometimes with a shortened domain.
func normalizeBusID(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
if i := strings.Index(s, ":"); i > 0 {
s = strings.TrimLeft(s[:i], "0") + s[i:]
// nvmlCard adapts an NVML device to what MIG discovery reads of it.
type nvmlCard struct{ dev nvml.Device }
func (c nvmlCard) MIGMode() (current, pending, ok bool) {
cur, pend, ret := c.dev.GetMigMode()
if ret != nvml.SUCCESS {
// NOT_SUPPORTED on a card without MIG, which is most of them.
return false, false, false
}
return strings.TrimPrefix(s, ":")
return cur == nvml.DEVICE_MIG_ENABLE, pend == nvml.DEVICE_MIG_ENABLE, true
}
func (c nvmlCard) MaxMIGDevices() int {
n, ret := c.dev.GetMaxMigDeviceCount()
if ret != nvml.SUCCESS {
return 0
}
return n
}
func (c nvmlCard) MIGDevice(i int) (nvmlMIGInstance, bool) {
h, ret := c.dev.GetMigDeviceHandleByIndex(i)
if ret != nvml.SUCCESS {
// NOT_FOUND for an index with no instance: the count is the most a
// card can hold, not how many it has.
return nil, false
}
return nvmlMIG{h}, true
}
// nvmlMIG adapts a MIG device handle.
type nvmlMIG struct{ dev nvml.Device }
func (m nvmlMIG) UUID() string {
u, _ := m.dev.GetUUID()
return u
}
func (m nvmlMIG) Name() string {
n, _ := m.dev.GetName()
return n
}
func (m nvmlMIG) Memory() (totalMB, usedMB int, ok bool) {
mem, ret := m.dev.GetMemoryInfo()
if ret != nvml.SUCCESS {
return 0, 0, false
}
return int(mem.Total / (1 << 20)), int(mem.Used / (1 << 20)), true
}
func (m nvmlMIG) Instances() (gpuInstance, computeInstance int) {
gi, ret := m.dev.GetGpuInstanceId()
if ret != nvml.SUCCESS {
gi = -1
}
ci, ret := m.dev.GetComputeInstanceId()
if ret != nvml.SUCCESS {
ci = -1
}
return gi, ci
}
func (m nvmlMIG) Attributes() (slices, memoryMB int, ok bool) {
a, ret := m.dev.GetAttributes()
if ret != nvml.SUCCESS {
return 0, 0, false
}
return int(a.GpuInstanceSliceCount), int(a.MemorySizeMB), true
}
+27 -1
View File
@@ -60,7 +60,33 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseSMI(string(out))
cards, err := parseSMI(string(out))
if err != nil || len(cards) == 0 {
return cards, err
}
// The query lists whole cards, MIG mode or not. A card in MIG mode is
// replaced by its instances, which only -L names and only the full
// report sizes; asked only when some card could be in MIG mode, since
// the report is the slowest thing nvidia-smi does.
if !anyMIGCapable(cards) {
return cards, nil
}
listed, report := readMIG(ctx, binary)
return expandMIG(cards, listed, report), nil
}
// anyMIGCapable reports whether any card could be in MIG mode: MIG began
// with compute capability 8.0 (the A100). The consumer cards of the same
// generations have none, but telling them apart by name is guesswork, and
// asking one costs only two more nvidia-smi runs at registration.
func anyMIGCapable(cards []GPU) bool {
for _, c := range cards {
major, _, _ := strings.Cut(c.ComputeCapability, ".")
if n, err := strconv.Atoi(major); err == nil && n >= 8 {
return true
}
}
return false
}
func parseSMI(out string) ([]GPU, error) {
+47 -25
View File
@@ -5,6 +5,7 @@ import (
"encoding/xml"
"fmt"
"os/exec"
"slices"
"strconv"
"strings"
)
@@ -34,9 +35,16 @@ func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
return parseGPUProcesses(out)
}
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
// GPUMemory is one device's memory as nvidia-smi reports it at a moment: what
// is allocated on it, and which processes hold how much of that.
//
// A device is what discovery reports: a whole card, or each MIG instance of a
// card in MIG mode, in the same order, so the readings line up with the
// inventory one for one.
type GPUMemory struct {
// Index is the device's number as placement knows it: the card's
// nvidia-smi index, or MIGIndex's for an instance.
Index int
UsedMB int
Procs []GPUProcess
}
@@ -66,48 +74,62 @@ func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error)
return parseGPUMemory(out)
}
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
// index order, its memory and the processes on it.
type smiLog struct {
GPUs []struct {
Used string `xml:"fb_memory_usage>used"`
Processes []struct {
PID string `xml:"pid"`
Type string `xml:"type"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
} `xml:"gpu"`
}
// parseGPUProcesses reads the report. GPUs are listed in index order, which
// is nvidia-smi's own numbering — the one placement assigns by.
// parseGPUProcesses reads the report, by device as placement numbers them:
// nvidia-smi's index for a whole card, MIGIndex's for a MIG instance.
func parseGPUProcesses(report []byte) (map[int][]int, error) {
mem, err := parseGPUMemory(report)
if err != nil {
return nil, err
}
out := map[int][]int{}
for i, g := range mem {
for _, g := range mem {
for _, p := range g.Procs {
out[i] = append(out[i], p.PID)
out[g.Index] = append(out[g.Index], p.PID)
}
}
return out, nil
}
// parseGPUMemory reads the report into each GPU's memory and its holders.
// parseGPUMemory reads the report into each device's memory and its holders.
//
// Cards are listed in index order, which is nvidia-smi's own numbering. A
// card in MIG mode is read as its instances, each with its own memory and
// the processes nvidia-smi attributes to its GPU and compute instance: a
// process on one instance is not on its neighbours, and the card's total
// would have charged every instance with all of them.
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
var log smiLog
var log migLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := make([]GPUMemory, len(log.GPUs))
out := make([]GPUMemory, 0, len(log.GPUs))
for i, g := range log.GPUs {
out[i].UsedMB = mib(g.Used)
for _, p := range g.Processes {
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
procs := func(gi, ci int) []GPUProcess {
var ps []GPUProcess
for _, p := range g.Processes {
pid, err := strconv.Atoi(strings.TrimSpace(p.PID))
if err != nil {
continue
}
if gi >= 0 && (atoiOr(p.GPUInstance, -1) != gi || atoiOr(p.ComputeInstance, -1) != ci) {
continue
}
ps = append(ps, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
}
return ps
}
if !enabled(g.Current) || len(g.Devices) == 0 {
out = append(out, GPUMemory{Index: i, UsedMB: mib(g.Used), Procs: procs(-1, -1)})
continue
}
devs := slices.Clone(g.Devices)
slices.SortFunc(devs, func(a, b migLogDevice) int { return atoiOr(a.Index, -1) - atoiOr(b.Index, -1) })
for _, d := range devs {
out = append(out, GPUMemory{
Index: MIGIndex(i, atoiOr(d.Index, 0)),
UsedMB: mib(d.Used),
Procs: procs(atoiOr(d.GPUInstance, -1), atoiOr(d.ComputeInstance, -1)),
})
}
}
return out, nil
+2 -2
View File
@@ -57,8 +57,8 @@ func TestGPUMemoryIsReadPerProcess(t *testing.T) {
t.Fatal(err)
}
want := []GPUMemory{
{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{},
{Index: 0, UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{Index: 1},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %+v, want %+v", got, want)
+560
View File
@@ -0,0 +1,560 @@
package discovery
// MIG — Multi-Instance GPU — carves one A100, H100 or later card into up to
// seven instances, each with its own memory, its own share of the
// multiprocessors and its own UUID. To CUDA an instance is a device of its
// own, and to placement it is one too: a card in MIG mode is reported as its
// instances instead of as the whole card, each a GPU of a class named from
// the card's and the instance's profile ("a100-80g-mig-3g.40gb").
//
// llmgw does not create instances. Which split a card gets is an operator's
// decision, made with nvidia-smi mig; discovery reads what is there.
//
// Two facts about MIG shape everything below. A process is given one MIG
// device: CUDA enumerates a single instance however many are made visible.
// And instances have no NVLink or peer-to-peer access between them. A replica
// on MIG therefore runs on exactly one instance, never a tensor-parallel group
// of them, and the catalog refuses a profile that asks for more.
import (
"context"
"encoding/xml"
"fmt"
"math"
"os/exec"
"regexp"
"slices"
"strconv"
"strings"
)
// MIGDevice is what a MIG instance carries beyond any GPU's fields: which
// card it is carved from, and how.
type MIGDevice struct {
// ParentIndex is the card's nvidia-smi index, which is what its sensors,
// topology and wear are reported by.
ParentIndex int `json:"parent_index"`
ParentUUID string `json:"parent_uuid,omitempty"`
// ParentMemoryTotalMB is the whole card's memory: the class is named from
// the card, so an instance of an 80 GB card is not mistaken for one of a
// 40 GB card that happens to have the same profile name.
ParentMemoryTotalMB int `json:"parent_memory_total_mb"`
// Device is the instance's number on its card, as nvidia-smi -L lists it.
Device int `json:"device"`
// Profile is NVIDIA's name for the slice, like "3g.40gb": three of the
// card's seven compute slices and forty gigabytes of its memory.
Profile string `json:"profile"`
// GPUInstance and ComputeInstance are the instance's ids in nvidia-smi
// mig -lgi and -lci, which is how nvidia-smi attributes processes to it.
GPUInstance int `json:"gpu_instance"`
ComputeInstance int `json:"compute_instance"`
}
// migIndexBase starts the numbers MIG instances are given.
//
// Placement, the agent protocol and every running shard name a node's devices
// by an integer, which for a whole card is its nvidia-smi index. An instance
// has none of its own, so it is given 1000 + 100 × card + device: card 0's
// instance 2 is 1002, card 3's instance 0 is 1300. Readable, stable while the
// instances are, and never one a whole card could have.
const migIndexBase = 1000
// MIGIndex is the device number placement knows a MIG instance by.
func MIGIndex(card, device int) int { return migIndexBase + 100*card + device }
// IsMIGIndex reports whether a device number names a MIG instance.
func IsMIGIndex(i int) bool { return i >= migIndexBase }
// DeviceLabel is how a device number is shown to people: "3" for a whole
// card, "0/mig2" for card 0's MIG instance 2.
func DeviceLabel(i int) string {
if !IsMIGIndex(i) {
return strconv.Itoa(i)
}
i -= migIndexBase
return fmt.Sprintf("%d/mig%d", i/100, i%100)
}
// DeviceLabels is DeviceLabel for each of a list.
func DeviceLabels(is []int) []string {
out := make([]string, len(is))
for k, i := range is {
out[k] = DeviceLabel(i)
}
return out
}
// IsMIG reports whether this GPU is a MIG instance rather than a whole card.
func (g GPU) IsMIG() bool { return g.MIG != nil }
// CardIndex is the nvidia-smi index of the physical card this GPU is on: its
// own index for a whole card, its parent's for a MIG instance.
func (g GPU) CardIndex() int {
if g.MIG != nil {
return g.MIG.ParentIndex
}
return g.Index
}
// migClassMarker joins a card's class to an instance's profile in a MIG
// class: "a100-80g" + "-mig-" + "3g.40gb".
const migClassMarker = "-mig-"
// IsMIGClass reports whether a GPU class names MIG instances.
func IsMIGClass(class string) bool { return strings.Contains(class, migClassMarker) }
// migProfileClean keeps a profile name to what a class may hold: "1g.10gb+me"
// becomes "1g.10gb-me".
var migProfileClean = regexp.MustCompile(`[^a-z0-9.]+`)
// migClass is the class of a MIG instance: the card's class and the profile.
func migClass(parentClass, profile string) string {
p := migProfileClean.ReplaceAllString(strings.ToLower(strings.TrimSpace(profile)), "-")
return parentClass + migClassMarker + strings.Trim(p, "-")
}
// MIGProfileMemoryGB reads the memory out of a MIG class or profile:
// 40 from "a100-80g-mig-3g.40gb" or "3g.40gb". Zero when there is none.
func MIGProfileMemoryGB(s string) int {
if i := strings.LastIndex(s, migClassMarker); i >= 0 {
s = s[i+len(migClassMarker):]
}
m := migProfileMemory.FindStringSubmatch(strings.ToLower(s))
if m == nil {
return 0
}
gb, _ := strconv.Atoi(m[1])
return gb
}
var migProfileMemory = regexp.MustCompile(`^\d+g\.(\d+)gb`)
// --- nvidia-smi -L ---
// listedMIG is one MIG instance as nvidia-smi -L lists it.
type listedMIG struct {
Device int
Profile string
UUID string
}
var (
// "GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-...)"
smiListGPU = regexp.MustCompile(`^GPU (\d+):`)
// " MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-...)". Drivers before
// R470 name the instance MIG-GPU-<card uuid>/<gi>/<ci>, which CUDA of that
// era takes in the same place, so the UUID is read as whatever it is.
smiListMIG = regexp.MustCompile(`^\s+MIG\s+(\S+)\s+Device\s+(\d+):\s*\(UUID:\s*([^)\s]+)\)`)
)
// parseSMIList reads nvidia-smi -L into each card's MIG instances, by the
// card's index, in device order. A card with none is absent.
func parseSMIList(out string) map[int][]listedMIG {
res := map[int][]listedMIG{}
card := -1
for _, line := range strings.Split(out, "\n") {
if m := smiListGPU.FindStringSubmatch(line); m != nil {
card, _ = strconv.Atoi(m[1])
continue
}
m := smiListMIG.FindStringSubmatch(line)
if m == nil || card < 0 {
continue
}
dev, _ := strconv.Atoi(m[2])
res[card] = append(res[card], listedMIG{Device: dev, Profile: m[1], UUID: m[3]})
}
for c := range res {
slices.SortFunc(res[c], func(a, b listedMIG) int { return a.Device - b.Device })
}
return res
}
// --- nvidia-smi -q -x ---
// migLog is the part of nvidia-smi's XML report MIG is read from: each card's
// mode, and each instance's ids and memory.
type migLog struct {
GPUs []migLogGPU `xml:"gpu"`
}
type migLogGPU struct {
ID string `xml:"id,attr"`
Current string `xml:"mig_mode>current_mig"`
Pending string `xml:"mig_mode>pending_mig"`
Devices []migLogDevice `xml:"mig_devices>mig_device"`
// Processes carry the GPU and compute instance they run in on a card
// in MIG mode, and "N/A" for both otherwise.
Processes []struct {
GPUInstance string `xml:"gpu_instance_id"`
ComputeInstance string `xml:"compute_instance_id"`
PID string `xml:"pid"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
Used string `xml:"fb_memory_usage>used"`
}
type migLogDevice struct {
Index string `xml:"index"`
GPUInstance string `xml:"gpu_instance_id"`
ComputeInstance string `xml:"compute_instance_id"`
Total string `xml:"fb_memory_usage>total"`
Used string `xml:"fb_memory_usage>used"`
}
// migState is one card's MIG mode and instances from the report.
type migState struct {
BusID string
Enabled bool
PendingEnabled bool
Devices []migReported
}
// migReported is one instance as the report describes it.
type migReported struct {
Device, GPUInstance, ComputeInstance int
TotalMB, UsedMB int
}
// enabled reads nvidia-smi's mode words: "Enabled", "Disabled", "N/A".
func enabled(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "Enabled") }
func atoiOr(s string, def int) int {
n, err := strconv.Atoi(strings.TrimSpace(s))
if err != nil {
return def
}
return n
}
// parseMIGReport reads each card's MIG state from the XML report, in the
// report's order, which is nvidia-smi's index order.
func parseMIGReport(report []byte) ([]migState, error) {
var log migLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := make([]migState, len(log.GPUs))
for i, g := range log.GPUs {
out[i] = migState{BusID: g.ID, Enabled: enabled(g.Current), PendingEnabled: enabled(g.Pending)}
for _, d := range g.Devices {
out[i].Devices = append(out[i].Devices, migReported{
Device: atoiOr(d.Index, -1), GPUInstance: atoiOr(d.GPUInstance, -1),
ComputeInstance: atoiOr(d.ComputeInstance, -1),
TotalMB: mib(d.Total), UsedMB: mib(d.Used),
})
}
slices.SortFunc(out[i].Devices, func(a, b migReported) int { return a.Device - b.Device })
}
return out, nil
}
// --- putting it together ---
// migProblemNoInstances keeps a card in MIG mode with nothing carved from it
// out of placement: CUDA cannot use the whole card while MIG is on, and
// there is no instance to use instead.
const migProblemNoInstances = "MIG is enabled with no instances created: nothing can run on it " +
"until instances are created (nvidia-smi mig -cgi … -C) or MIG is turned off"
// migProblemUnnamed is a card whose instances exist but could not be named:
// an engine is given an instance by its UUID, and without one it cannot be.
const migProblemUnnamed = "MIG is enabled and has instances, but nvidia-smi -L did not name them, " +
"so none can be given to an engine"
// expandMIG replaces each card in MIG mode with its instances.
//
// listed is nvidia-smi -L's instances by card, which names them; report is
// the XML report's state per card, which gives their memory and ids. Either
// may be missing: -L alone still says which instances exist, and the report
// alone still says a card is in MIG mode, so a card it says has instances
// that -L did not name is kept as the whole card with the reason, rather
// than placed as a card CUDA will not open.
func expandMIG(cards []GPU, listed map[int][]listedMIG, report []migState) []GPU {
if len(listed) == 0 && !slices.ContainsFunc(report, func(s migState) bool { return s.Enabled }) {
return cards
}
byBus := map[string]migState{}
for _, s := range report {
byBus[normalizeBusID(s.BusID)] = s
}
out := make([]GPU, 0, len(cards))
for pos, c := range cards {
st, ok := byBus[normalizeBusID(c.PCIBusID)]
if !ok && pos < len(report) {
st = report[pos]
}
devs := listed[c.Index]
if !st.Enabled && len(devs) == 0 {
out = append(out, c)
continue
}
if len(devs) == 0 {
p := migProblemNoInstances
if len(st.Devices) > 0 {
p = migProblemUnnamed
}
c.Health.Problems = append(c.Health.Problems, p)
out = append(out, c)
continue
}
for _, d := range devs {
var rep migReported
found := false
for _, r := range st.Devices {
if r.Device == d.Device {
rep, found = r, true
}
}
mig := migInstance(c, d.Device, d.Profile, d.UUID)
if found {
mig.MIG.GPUInstance, mig.MIG.ComputeInstance = rep.GPUInstance, rep.ComputeInstance
mig.MemoryTotalMB, mig.MemoryUsedMB = rep.TotalMB, rep.UsedMB
}
if mig.MemoryTotalMB == 0 {
// The profile says it to the gigabyte, which is close enough
// to place by when the report could not be read.
mig.MemoryTotalMB = MIGProfileMemoryGB(d.Profile) * 1024
}
assessHealth(&mig)
out = append(out, mig)
}
}
return out
}
// migInstance builds a MIG instance of card: everything about the card that
// is shared — driver, compute capability, NUMA node, bus, temperature and
// ECC errors — and nothing that is not. Not NVLink, which an instance cannot
// use; not utilisation, which no driver reports per instance; and not the
// card's memory or problems, which the instance's own replace.
func migInstance(card GPU, device int, profile, uuid string) GPU {
g := GPU{
Index: MIGIndex(card.Index, device), UUID: uuid,
Name: strings.TrimSpace(card.Name + " MIG " + profile),
ComputeCapability: card.ComputeCapability, NUMANode: card.NUMANode,
PCIBusID: card.PCIBusID, DriverVersion: card.DriverVersion, CUDAVersion: card.CUDAVersion,
NICPaths: card.NICPaths, Fabric: card.Fabric,
Health: GPUHealth{ECCErrors: card.Health.ECCErrors, TemperatureC: card.Health.TemperatureC},
MIG: &MIGDevice{
ParentIndex: card.Index, ParentUUID: card.UUID, ParentMemoryTotalMB: card.MemoryTotalMB,
Device: device, Profile: profile, GPUInstance: -1, ComputeInstance: -1,
},
}
return g
}
// readMIG asks nvidia-smi for what expandMIG needs. A failure of either
// command leaves its half empty, which expandMIG is written to take.
func readMIG(ctx context.Context, binary string) (map[int][]listedMIG, []migState) {
var listed map[int][]listedMIG
if out, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil {
listed = parseSMIList(string(out))
}
var report []migState
if out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output(); err == nil {
report, _ = parseMIGReport(out)
}
return listed, report
}
// --- NVML ---
// nvmlMIGCard is the part of an NVML device MIG discovery reads, behind an
// interface so the logic is tested without a driver: gpu_nvml.go adapts
// nvml.Device to it.
type nvmlMIGCard interface {
// MIGMode is nvmlDeviceGetMigMode; ok is false on a card without MIG.
MIGMode() (current, pending, ok bool)
// MaxMIGDevices is nvmlDeviceGetMaxMigDeviceCount.
MaxMIGDevices() int
// MIGDevice is nvmlDeviceGetMigDeviceHandleByIndex; ok is false for an
// index with no instance behind it, which is how NVML says so.
MIGDevice(i int) (nvmlMIGInstance, bool)
}
// nvmlMIGInstance is one MIG device handle.
type nvmlMIGInstance interface {
UUID() string
// Name is "NVIDIA A100-SXM4-80GB MIG 3g.40gb" on the drivers that name
// MIG devices, which is where the profile is read from.
Name() string
Memory() (totalMB, usedMB int, ok bool)
Instances() (gpuInstance, computeInstance int)
// Slices and MemoryMB are the instance's attributes, which name the
// profile on a driver whose device name does not.
Attributes() (slices, memoryMB int, ok bool)
}
var nvmlMIGName = regexp.MustCompile(`MIG\s+(\S+)\s*$`)
// migFromNVML is what one card is to placement under NVML: its instances if
// it is in MIG mode, itself if it is not. isMIG says which.
func migFromNVML(card GPU, dev nvmlMIGCard) (out []GPU, isMIG bool) {
current, _, ok := dev.MIGMode()
if !ok || !current {
return []GPU{card}, false
}
n := dev.MaxMIGDevices()
for i := range n {
h, ok := dev.MIGDevice(i)
if !ok {
continue
}
profile := ""
if m := nvmlMIGName.FindStringSubmatch(h.Name()); m != nil {
profile = m[1]
} else if n, mem, ok := h.Attributes(); ok {
// The convention NVIDIA's names follow: compute slices, then the
// memory in whole gigabytes.
profile = fmt.Sprintf("%dg.%dgb", n, int(math.Round(float64(mem)/1024)))
}
g := migInstance(card, i, profile, h.UUID())
g.MIG.GPUInstance, g.MIG.ComputeInstance = h.Instances()
if total, used, ok := h.Memory(); ok {
g.MemoryTotalMB, g.MemoryUsedMB = total, used
} else {
g.MemoryTotalMB = MIGProfileMemoryGB(profile) * 1024
}
assessHealth(&g)
out = append(out, g)
}
if len(out) == 0 {
card.Health.Problems = append(card.Health.Problems, migProblemNoInstances)
return []GPU{card}, true
}
return out, true
}
// --- what the doctor reports ---
// MIGCard is one card's MIG state, as `llmgw doctor` reports it.
type MIGCard struct {
Index int `json:"index"`
Name string `json:"name,omitempty"`
// Enabled is the mode in force; PendingEnabled the mode after the next
// reset of the card, which differs from it until the card is reset.
Enabled bool `json:"enabled"`
PendingEnabled bool `json:"pending_enabled"`
// Instances are the MIG devices an engine can be given, as nvidia-smi -L
// lists them.
Instances []MIGListed `json:"instances,omitempty"`
// GPUInstances and ComputeInstances are nvidia-smi mig -lgi and -lci: a
// GPU instance with no compute instance in it is not a device yet.
GPUInstances []MIGInstance `json:"gpu_instances,omitempty"`
ComputeInstances []MIGInstance `json:"compute_instances,omitempty"`
}
// MIGListed is one instance an engine can be given.
type MIGListed struct {
Device int `json:"device"`
Profile string `json:"profile"`
UUID string `json:"uuid"`
}
// MIGInstance is one row of nvidia-smi mig -lgi or -lci.
type MIGInstance struct {
GPU int `json:"gpu"`
Profile string `json:"profile"`
// ProfileID is NVIDIA's number for the profile.
ProfileID int `json:"profile_id"`
// GPUInstance is the GPU instance's id; ComputeInstance the compute
// instance's within it, -1 in a -lgi row.
GPUInstance int `json:"gpu_instance"`
ComputeInstance int `json:"compute_instance"`
Placement string `json:"placement,omitempty"`
}
var (
// -lgi: "| 0 MIG 3g.40gb 9 2 4:4 |"
migLGIRow = regexp.MustCompile(`^\|\s*(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`)
// -lci: "| 0 1 MIG 3g.40gb 2 0 0:3 |"
migLCIRow = regexp.MustCompile(`^\|\s*(\d+)\s+(\d+)\s+MIG\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+:\d+)\s*\|`)
)
// ParseMIGInstances reads nvidia-smi mig -lgi or -lci, whichever it is
// given: the two tables differ by the GPU instance column -lci adds. "No GPU
// instances found" and the like are no rows, not an error.
func ParseMIGInstances(out string) []MIGInstance {
var res []MIGInstance
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if m := migLCIRow.FindStringSubmatch(line); m != nil {
res = append(res, MIGInstance{
GPU: atoiOr(m[1], -1), GPUInstance: atoiOr(m[2], -1), Profile: m[3],
ProfileID: atoiOr(m[4], -1), ComputeInstance: atoiOr(m[5], -1), Placement: m[6],
})
continue
}
if m := migLGIRow.FindStringSubmatch(line); m != nil {
res = append(res, MIGInstance{
GPU: atoiOr(m[1], -1), Profile: m[2], ProfileID: atoiOr(m[3], -1),
GPUInstance: atoiOr(m[4], -1), ComputeInstance: -1, Placement: m[5],
})
}
}
return res
}
// DiscoverMIG reads every card's MIG state for the doctor: the mode from the
// XML report, the devices from -L, the instances from nvidia-smi mig. Only
// cards that support MIG are listed, and an error is returned only when the
// report itself cannot be read.
func DiscoverMIG(ctx context.Context, binary string) ([]MIGCard, error) {
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s -q -x: %w", binary, err)
}
var list, lgi, lci string
if b, err := exec.CommandContext(ctx, binary, "-L").Output(); err == nil {
list = string(b)
}
// nvidia-smi mig exits non-zero when there is nothing to list, with the
// table or the reason on its output either way.
b, _ := exec.CommandContext(ctx, binary, "mig", "-lgi").CombinedOutput()
lgi = string(b)
b, _ = exec.CommandContext(ctx, binary, "mig", "-lci").CombinedOutput()
lci = string(b)
return buildMIGCards(out, list, lgi, lci)
}
// buildMIGCards is DiscoverMIG from the commands' output.
func buildMIGCards(report []byte, list, lgi, lci string) ([]MIGCard, error) {
var log struct {
GPUs []struct {
Name string `xml:"product_name"`
Current string `xml:"mig_mode>current_mig"`
Pending string `xml:"mig_mode>pending_mig"`
} `xml:"gpu"`
}
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
listed := parseSMIList(list)
gis, cis := ParseMIGInstances(lgi), ParseMIGInstances(lci)
var out []MIGCard
for i, g := range log.GPUs {
cur, pend := strings.TrimSpace(g.Current), strings.TrimSpace(g.Pending)
// A card without MIG says N/A, or nothing on a driver that predates
// it; neither is worth a line.
if !enabled(cur) && !enabled(pend) && !strings.EqualFold(cur, "Disabled") {
continue
}
c := MIGCard{Index: i, Name: strings.TrimSpace(g.Name), Enabled: enabled(cur), PendingEnabled: enabled(pend)}
for _, l := range listed[i] {
c.Instances = append(c.Instances, MIGListed(l))
}
for _, gi := range gis {
if gi.GPU == i && gi.ComputeInstance < 0 {
c.GPUInstances = append(c.GPUInstances, gi)
}
}
for _, ci := range cis {
if ci.GPU == i && ci.ComputeInstance >= 0 {
c.ComputeInstances = append(c.ComputeInstances, ci)
}
}
out = append(out, c)
}
return out, nil
}
+418
View File
@@ -0,0 +1,418 @@
package discovery
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
// Two A100 80GB cards: card 0 in MIG mode, cut into two 3g.40gb instances,
// card 1 whole. What nvidia-smi prints for that machine, by command. None of
// it is from real MIG hardware, which this was written without; the shapes
// follow NVIDIA's documentation and the output of drivers R470 to R550.
const (
migQuery = `0, GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:07:00.0, 550.54.15, 31, 0, [N/A]
1, GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:0F:00.0, 550.54.15, 33, 0, 0
`
migList = `GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)
MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f)
MIG 3g.40gb Device 1: (UUID: MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d)
GPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718)
`
migReport = `<?xml version="1.0" ?>
<nvidia_smi_log>
<driver_version>550.54.15</driver_version>
<gpu id="00000000:07:00.0">
<product_name>NVIDIA A100-SXM4-80GB</product_name>
<mig_mode><current_mig>Enabled</current_mig><pending_mig>Enabled</pending_mig></mig_mode>
<mig_devices>
<mig_device>
<index>1</index>
<gpu_instance_id>1</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<device_attributes><shared><multiprocessor_count>42</multiprocessor_count></shared></device_attributes>
<fb_memory_usage><total>40192 MiB</total><reserved>0 MiB</reserved><used>21000 MiB</used><free>19192 MiB</free></fb_memory_usage>
</mig_device>
<mig_device>
<index>0</index>
<gpu_instance_id>2</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<fb_memory_usage><total>40192 MiB</total><reserved>0 MiB</reserved><used>37 MiB</used><free>40155 MiB</free></fb_memory_usage>
</mig_device>
</mig_devices>
<fb_memory_usage><total>81920 MiB</total><used>21037 MiB</used></fb_memory_usage>
<processes>
<process_info>
<gpu_instance_id>1</gpu_instance_id>
<compute_instance_id>0</compute_instance_id>
<pid>4100</pid><type>C</type><process_name>vllm</process_name>
<used_memory>20900 MiB</used_memory>
</process_info>
</processes>
</gpu>
<gpu id="00000000:0F:00.0">
<product_name>NVIDIA A100-SXM4-80GB</product_name>
<mig_mode><current_mig>Disabled</current_mig><pending_mig>Disabled</pending_mig></mig_mode>
<mig_devices>None</mig_devices>
<fb_memory_usage><total>81920 MiB</total><used>300 MiB</used></fb_memory_usage>
<processes>
<process_info>
<gpu_instance_id>N/A</gpu_instance_id>
<compute_instance_id>N/A</compute_instance_id>
<pid>5200</pid><type>C</type><used_memory>250 MiB</used_memory>
</process_info>
</processes>
</gpu>
</nvidia_smi_log>`
migLGI = `+-------------------------------------------------------+
| GPU instances: |
| GPU Name Profile Instance Placement |
| ID ID Start:Size |
|=======================================================|
| 0 MIG 3g.40gb 9 1 4:4 |
+-------------------------------------------------------+
| 0 MIG 3g.40gb 9 2 0:4 |
+-------------------------------------------------------+
`
migLCI = `+--------------------------------------------------------------------+
| Compute instances: |
| GPU GPU Name Profile Instance Placement |
| Instance ID ID Start:Size |
| ID |
|====================================================================|
| 0 1 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
| 0 2 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
`
)
func migMachine(t *testing.T) []GPU {
t.Helper()
cards, err := parseSMI(migQuery)
if err != nil {
t.Fatal(err)
}
report, err := parseMIGReport([]byte(migReport))
if err != nil {
t.Fatal(err)
}
return expandMIG(cards, parseSMIList(migList), report)
}
func TestACardInMIGModeIsReportedAsItsInstancesInsteadOfItself(t *testing.T) {
gpus := migMachine(t)
var got []string
for _, g := range gpus {
got = append(got, DeviceLabel(g.Index)+" "+g.Class())
}
want := []string{"0/mig0 a100-80g-mig-3g.40gb", "0/mig1 a100-80g-mig-3g.40gb", "1 a100-80g"}
if !slices.Equal(got, want) {
t.Fatalf("devices = %q, want %q", got, want)
}
mig := gpus[1]
if mig.Index != 1001 || mig.UUID != "MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d" {
t.Errorf("instance 1 is %d %s; an engine is given it by that UUID", mig.Index, mig.UUID)
}
if mig.MemoryTotalMB != 40192 || mig.MemoryUsedMB != 21000 {
t.Errorf("instance 1 memory %d/%d, want its own 21000/40192 rather than the card's",
mig.MemoryUsedMB, mig.MemoryTotalMB)
}
if m := mig.MIG; m == nil || m.ParentIndex != 0 || m.Device != 1 || m.Profile != "3g.40gb" ||
m.GPUInstance != 1 || m.ComputeInstance != 0 || m.ParentUUID != "GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77" {
t.Errorf("instance 1's parentage: %+v", mig.MIG)
}
if mig.ComputeCapability != "8.0" || mig.DriverVersion != "550.54.15" || mig.Health.TemperatureC != 31 {
t.Errorf("what the card and its instances share was not carried: %+v", mig)
}
// Twenty gigabytes in use on an instance is a foreign tenant there, and
// only there: instance 0 next to it is free.
if mig.Health.OK() || !gpus[0].Health.OK() {
t.Errorf("foreign memory judged per instance: 0 %v, 1 %v", gpus[0].Health.Problems, mig.Health.Problems)
}
// No driver reports utilisation per instance, and none is made up.
if mig.UtilizationPercent != 0 {
t.Errorf("an instance reported utilisation %d", mig.UtilizationPercent)
}
if gpus[2].IsMIG() || gpus[2].Index != 1 || gpus[2].CardIndex() != 1 || gpus[0].CardIndex() != 0 {
t.Errorf("the whole card beside it changed: %+v", gpus[2])
}
}
func TestAMIGInstanceSurvivesTheTripToTheControlPlaneWithItsClass(t *testing.T) {
g := migMachine(t)[0]
b, err := json.Marshal(g)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), `"class":"a100-80g-mig-3g.40gb"`) {
t.Errorf("the JSON carries no class for the instance: %s", b)
}
var back GPU
if err := json.Unmarshal(b, &back); err != nil {
t.Fatal(err)
}
if back.Class() != g.Class() || !reflect.DeepEqual(back.MIG, g.MIG) {
t.Errorf("came back as %s %+v", back.Class(), back.MIG)
}
}
func TestMIGClassesAreNamedFromTheCardAndTheProfile(t *testing.T) {
for _, c := range []struct {
name string
cardMB int
profile string
want string
}{
{"NVIDIA A100-SXM4-80GB MIG 3g.40gb", 81920, "3g.40gb", "a100-80g-mig-3g.40gb"},
// The same profile name on a 40 GB card is another class: its
// instance is the same size, but the card is not, and neither is
// the set of profiles that card can be cut to.
{"NVIDIA A100-SXM4-40GB MIG 3g.20gb", 40960, "3g.20gb", "a100-40g-mig-3g.20gb"},
{"NVIDIA H100 80GB HBM3 MIG 1g.10gb+me", 81559, "1g.10gb+me", "h100-80g-mig-1g.10gb-me"},
{"NVIDIA H100 80GB HBM3 MIG 7g.80gb", 81559, "7g.80gb", "h100-80g-mig-7g.80gb"},
} {
g := GPU{Name: c.name, MemoryTotalMB: 10000, MIG: &MIGDevice{ParentMemoryTotalMB: c.cardMB, Profile: c.profile}}
if got := g.Class(); got != c.want {
t.Errorf("%s -> %s, want %s", c.name, got, c.want)
}
if !IsMIGClass(g.Class()) {
t.Errorf("%s is not recognised as a MIG class", g.Class())
}
}
if IsMIGClass("a100-80g") || IsMIGClass("6000-24g") {
t.Error("a whole card's class was taken for a MIG one")
}
if MIGProfileMemoryGB("a100-80g-mig-3g.40gb") != 40 || MIGProfileMemoryGB("1g.10gb+me") != 10 || MIGProfileMemoryGB("a100-80g") != 0 {
t.Error("the instance's memory is misread from its profile")
}
}
func TestDeviceNumbersAreShownAsCardAndInstance(t *testing.T) {
for i, want := range map[int]string{0: "0", 7: "7", MIGIndex(0, 2): "0/mig2", MIGIndex(3, 0): "3/mig0", 1302: "3/mig2"} {
if got := DeviceLabel(i); got != want {
t.Errorf("DeviceLabel(%d) = %s, want %s", i, got, want)
}
}
if IsMIGIndex(15) || !IsMIGIndex(MIGIndex(0, 0)) {
t.Error("whole cards and instances are told apart wrongly")
}
}
func TestACardInMIGModeWithNoInstancesIsKeptOutOfPlacement(t *testing.T) {
cards, _ := parseSMI(migQuery)
report, _ := parseMIGReport([]byte(strings.Replace(migReport,
migReport[strings.Index(migReport, "<mig_devices>"):strings.Index(migReport, "</mig_devices>")+len("</mig_devices>")],
"<mig_devices>None</mig_devices>", 1)))
gpus := expandMIG(cards, parseSMIList("GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-x)\nGPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-y)\n"), report)
if len(gpus) != 2 || gpus[0].IsMIG() {
t.Fatalf("got %+v", gpus)
}
// CUDA will not open the whole card while MIG is on, and there is no
// instance to open instead: placing anything here fails at start.
if gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "no instances") {
t.Errorf("card 0's problems: %v", gpus[0].Health.Problems)
}
if !gpus[1].Health.OK() {
t.Errorf("card 1, not in MIG mode, was flagged: %v", gpus[1].Health.Problems)
}
// Instances the report sees but -L could not name cannot be given to an
// engine either; the card says why.
report, _ = parseMIGReport([]byte(migReport))
gpus = expandMIG(cards, nil, report)
if len(gpus) != 2 || gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "did not name") {
t.Errorf("with -L failing: %+v", gpus)
}
}
func TestInstancesAreFoundFromTheListAloneWhenTheReportCannotBeRead(t *testing.T) {
cards, _ := parseSMI(migQuery)
gpus := expandMIG(cards, parseSMIList(migList), nil)
if len(gpus) != 3 || !gpus[0].IsMIG() || gpus[0].MemoryTotalMB != 40*1024 {
t.Errorf("without the report: %+v", gpus)
}
}
func TestAnOlderDriversInstanceUUIDIsTakenAsItIs(t *testing.T) {
// R450 and R460 name an instance by its card and ids, and CUDA of that
// era takes exactly that in CUDA_VISIBLE_DEVICES.
l := parseSMIList("GPU 0: A100-SXM4-40GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)\n" +
" MIG 1g.5gb Device 0: (UUID: MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0)\n")
if len(l[0]) != 1 || l[0][0].UUID != "MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0" || l[0][0].Profile != "1g.5gb" {
t.Errorf("parsed %+v", l)
}
}
func TestProcessesOnAMIGCardAreChargedToTheirOwnInstance(t *testing.T) {
mem, err := parseGPUMemory([]byte(migReport))
if err != nil {
t.Fatal(err)
}
want := []GPUMemory{
{Index: 1000, UsedMB: 37},
{Index: 1001, UsedMB: 21000, Procs: []GPUProcess{{PID: 4100, UsedMB: 20900}}},
{Index: 1, UsedMB: 300, Procs: []GPUProcess{{PID: 5200, UsedMB: 250}}},
}
if !reflect.DeepEqual(mem, want) {
t.Errorf("got %+v\nwant %+v", mem, want)
}
// In the order discovery reports the devices, so a heartbeat's readings
// line up with the inventory one for one.
gpus := migMachine(t)
for i := range gpus {
if mem[i].Index != gpus[i].Index {
t.Errorf("reading %d is for device %d, inventory has %d", i, mem[i].Index, gpus[i].Index)
}
}
procs, _ := parseGPUProcesses([]byte(migReport))
if !reflect.DeepEqual(procs, map[int][]int{1001: {4100}, 1: {5200}}) {
t.Errorf("processes by device: %v", procs)
}
}
func TestAMIGInstanceTakesItsCardsPlaceInTheTopologyButNoNVLink(t *testing.T) {
gpus := migMachine(t)
topo := Topology{
NVLink: map[int]map[int]int{0: {1: 12}, 1: {0: 12}},
NUMA: map[int]int{0: 0, 1: 1},
NICPath: map[int]map[string]string{0: {"mlx5_0": "PIX"}},
}
topo.Apply(gpus)
if len(gpus[0].NVLinkPeers) != 0 || len(gpus[1].NVLinkPeers) != 0 {
t.Errorf("MIG instances were given NVLink peers: %v %v", gpus[0].NVLinkPeers, gpus[1].NVLinkPeers)
}
if gpus[1].NUMANode != 0 || gpus[1].NICPaths["mlx5_0"] != "PIX" {
t.Errorf("instance 1 did not take its card's NUMA node and adapters: %+v", gpus[1])
}
if gpus[2].NUMANode != 1 || !slices.Equal(gpus[2].NVLinkPeers, []int{0}) {
t.Errorf("the whole card's topology: %+v", gpus[2])
}
}
func TestMIGInstancesAreDiscoveredThroughNvidiaSMI(t *testing.T) {
// The real code path against a stub binary that answers each command as
// the machine above would.
dir := t.TempDir()
stub := filepath.Join(dir, "nvidia-smi")
script := "#!/bin/sh\ncase \"$1\" in\n" +
"-L) cat <<'EOF'\n" + migList + "EOF\n;;\n" +
"-q) cat <<'EOF'\n" + migReport + "\nEOF\n;;\n" +
"*) cat <<'EOF'\n" + migQuery + "EOF\n;;\nesac\n"
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
gpus, err := DiscoverGPUsSMI(t.Context(), stub)
if err != nil {
t.Fatal(err)
}
inv := Inventory{GPUs: gpus}
if !slices.Equal(inv.Classes(), []string{"a100-80g", "a100-80g-mig-3g.40gb"}) || len(gpus) != 3 {
t.Errorf("classes %v from %d devices", inv.Classes(), len(gpus))
}
}
// fakeMIGCard is NVML's view of one card, as nvmlDeviceGetMigMode,
// nvmlDeviceGetMaxMigDeviceCount and nvmlDeviceGetMigDeviceHandleByIndex
// would give it.
type fakeMIGCard struct {
mode, pending, supported bool
max int
devs map[int]fakeMIGInstance
}
func (c fakeMIGCard) MIGMode() (bool, bool, bool) { return c.mode, c.pending, c.supported }
func (c fakeMIGCard) MaxMIGDevices() int { return c.max }
func (c fakeMIGCard) MIGDevice(i int) (nvmlMIGInstance, bool) {
d, ok := c.devs[i]
return d, ok
}
type fakeMIGInstance struct {
uuid, name string
total, used int
gi, ci int
slices, attrMem int
}
func (m fakeMIGInstance) UUID() string { return m.uuid }
func (m fakeMIGInstance) Name() string { return m.name }
func (m fakeMIGInstance) Memory() (int, int, bool) {
return m.total, m.used, m.total > 0
}
func (m fakeMIGInstance) Instances() (int, int) { return m.gi, m.ci }
func (m fakeMIGInstance) Attributes() (int, int, bool) {
return m.slices, m.attrMem, m.slices > 0
}
func TestNVMLReportsACardInMIGModeAsItsInstances(t *testing.T) {
card := GPU{Index: 2, UUID: "GPU-card", Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
ComputeCapability: "9.0", NUMANode: 1, Health: GPUHealth{TemperatureC: 40}}
// Seven slots, three filled, one of them on a driver that does not put
// the profile in the device's name.
dev := fakeMIGCard{mode: true, pending: true, supported: true, max: 7, devs: map[int]fakeMIGInstance{
0: {uuid: "MIG-aaa", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 2, ci: 0},
1: {uuid: "MIG-bbb", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 1, ci: 0},
4: {uuid: "MIG-ccc", name: "NVIDIA H100 80GB HBM3", total: 9984, gi: 9, ci: 0, slices: 1, attrMem: 9984},
}}
gpus, isMIG := migFromNVML(card, dev)
if !isMIG || len(gpus) != 3 {
t.Fatalf("got %d devices, MIG %v", len(gpus), isMIG)
}
var classes []string
for _, g := range gpus {
classes = append(classes, DeviceLabel(g.Index)+" "+g.UUID+" "+g.Class())
}
want := []string{"2/mig0 MIG-aaa h100-80g-mig-3g.40gb", "2/mig1 MIG-bbb h100-80g-mig-3g.40gb",
"2/mig4 MIG-ccc h100-80g-mig-1g.10gb"}
if !slices.Equal(classes, want) {
t.Errorf("devices %q, want %q", classes, want)
}
if g := gpus[1]; g.MemoryTotalMB != 40448 || g.MIG.GPUInstance != 1 || g.NUMANode != 1 ||
g.Health.TemperatureC != 40 || len(g.NVLinkPeers) != 0 {
t.Errorf("instance 1: %+v", g)
}
// MIG on and nothing carved: the card, kept out of placement.
gpus, isMIG = migFromNVML(card, fakeMIGCard{mode: true, supported: true, max: 7})
if !isMIG || len(gpus) != 1 || gpus[0].IsMIG() || gpus[0].Health.OK() {
t.Errorf("an empty MIG card: %+v", gpus)
}
// MIG off, or a card without it: the card, as it always was.
for _, d := range []fakeMIGCard{{supported: true, max: 7}, {}} {
gpus, isMIG = migFromNVML(card, d)
if isMIG || len(gpus) != 1 || !reflect.DeepEqual(gpus[0], card) {
t.Errorf("a card not in MIG mode came back as %+v", gpus)
}
}
}
func TestMIGInstanceTablesAreRead(t *testing.T) {
gis := ParseMIGInstances(migLGI)
if len(gis) != 2 || gis[0] != (MIGInstance{GPU: 0, Profile: "3g.40gb", ProfileID: 9, GPUInstance: 1,
ComputeInstance: -1, Placement: "4:4"}) {
t.Errorf("-lgi: %+v", gis)
}
cis := ParseMIGInstances(migLCI)
if len(cis) != 2 || cis[1] != (MIGInstance{GPU: 0, GPUInstance: 2, Profile: "3g.40gb", ProfileID: 2,
ComputeInstance: 0, Placement: "0:3"}) {
t.Errorf("-lci: %+v", cis)
}
if got := ParseMIGInstances("No GPU instances found: Not Found\n"); len(got) != 0 {
t.Errorf("nothing to list read as %+v", got)
}
cards, err := buildMIGCards([]byte(migReport), migList, migLGI, migLCI)
if err != nil {
t.Fatal(err)
}
if len(cards) != 2 || !cards[0].Enabled || len(cards[0].Instances) != 2 || len(cards[0].GPUInstances) != 2 ||
len(cards[0].ComputeInstances) != 2 || cards[1].Enabled {
t.Errorf("cards: %+v", cards)
}
}
+10 -4
View File
@@ -166,19 +166,25 @@ func splitTabs(l string) []string {
// Apply puts the topology on the GPUs it describes. What a GPU already knows
// — NVLink peers from NVML, which is the more exact source — is kept.
//
// The matrix is of whole cards. A MIG instance takes its card's NUMA node and
// adapters, and none of its NVLink: instances cannot reach each other or any
// other card over it, and peers listed for one would draw a tensor-parallel
// group onto it.
func (t Topology) Apply(gpus []GPU) {
for i := range gpus {
g := &gpus[i]
if len(g.NVLinkPeers) == 0 {
for peer := range t.NVLink[g.Index] {
card := g.CardIndex()
if len(g.NVLinkPeers) == 0 && !g.IsMIG() {
for peer := range t.NVLink[card] {
g.NVLinkPeers = append(g.NVLinkPeers, peer)
}
sort.Ints(g.NVLinkPeers)
}
if n, ok := t.NUMA[g.Index]; ok {
if n, ok := t.NUMA[card]; ok {
g.NUMANode = n
}
if paths := t.NICPath[g.Index]; len(paths) > 0 {
if paths := t.NICPath[card]; len(paths) > 0 {
g.NICPaths = paths
}
}
+34 -2
View File
@@ -62,6 +62,12 @@ type ShardSpec struct {
// GPUIndices are the local GPUs this shard uses.
GPUIndices []int
// GPUUUIDs name the same devices by UUID, in the same order, and are set
// by the agent only when one of them is a MIG instance. An instance has
// no index CUDA knows: CUDA_VISIBLE_DEVICES=MIG-<uuid> is the only way to
// give a process one, and the indices placement uses for it (1000 and
// up) mean nothing to CUDA.
GPUUUIDs []string `json:"gpu_uuids,omitempty"`
// HCAs are the adapters of the chosen fabric, and only that fabric. A shard
// must never be given HCAs from two fabrics.
HCAs []string
@@ -168,6 +174,13 @@ type CPUFallback interface {
Devices(ctx context.Context) ([]string, error)
}
// HelpCommand is an engine whose program is not asked for its help by
// running the command's path with --help, because the path is an interpreter
// starting it through a launcher.
type HelpCommand interface {
HelpCommand(c Command) (path string, args []string)
}
// GPUInfo is the subset of GPU facts validation needs.
type GPUInfo struct {
Index int
@@ -187,6 +200,7 @@ func NewRegistry(cfg Config) *Registry {
r.Add(&VLLM{cfg: cfg.VLLM})
r.Add(&SGLang{cfg: cfg.SGLang})
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
r.Add(&TRTLLM{cfg: cfg.TRTLLM})
r.Add(&Sim{cfg: cfg.Sim})
return r
}
@@ -232,7 +246,10 @@ type Config struct {
VLLM VLLMConfig `koanf:"vllm"`
SGLang SGLangConfig `koanf:"sglang"`
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
Sim SimConfig `koanf:"sim"`
// TRTLLM is TensorRT-LLM, which the agent does not install: a node
// offers it only when its interpreter is named here.
TRTLLM TRTLLMConfig `koanf:"trtllm"`
Sim SimConfig `koanf:"sim"`
}
// --- shared helpers ---
@@ -257,7 +274,7 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
// one instead of timing out.
func baseEnv(s ShardSpec) map[string]string {
env := map[string]string{
"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
"CUDA_VISIBLE_DEVICES": visibleDevices(s),
// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
// counts fastest first unless told otherwise, and on a node with two
// kinds of GPU the shard would get a different card than placement
@@ -329,6 +346,21 @@ func materializeEnv(generated map[string]string) []string {
return out
}
// visibleDevices is CUDA_VISIBLE_DEVICES for a shard: the indices placement
// gave it, or their UUIDs when the agent named them, which it does for a MIG
// instance.
func visibleDevices(s ShardSpec) string {
if len(s.GPUUUIDs) > 0 {
return strings.Join(s.GPUUUIDs, ",")
}
return joinInts(s.GPUIndices)
}
// OnMIG reports whether the shard was given a MIG instance.
func (s ShardSpec) OnMIG() bool {
return slices.ContainsFunc(s.GPUUUIDs, func(u string) bool { return strings.HasPrefix(u, "MIG-") })
}
func joinInts(v []int) string {
parts := make([]string, len(v))
for i, n := range v {
+2 -2
View File
@@ -457,14 +457,14 @@ func TestValidateRequiresModelPath(t *testing.T) {
func TestRegistry(t *testing.T) {
r := NewRegistry(Config{})
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "trtllm", "vllm"}) {
t.Errorf("registry contents: %v", got)
}
if _, err := r.Get("vllm"); err != nil {
t.Errorf("vllm missing: %v", err)
}
err := func() error { _, err := r.Get("nope"); return err }()
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, trtllm, vllm") {
t.Errorf("an unknown engine should list what is available: %v", err)
}
}
+5 -1
View File
@@ -178,7 +178,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
// as the driver lists them, which on NVIDIA follows the same bus order
// as nvidia-smi; were it not to, the shard would not hold memory on its
// own GPUs, and starting it checks exactly that.
if len(s.GPUIndices) > 0 {
//
// Not on a MIG instance, which Vulkan does not see at all: only CUDA
// opens one, by the UUID CUDA_VISIBLE_DEVICES carries, and an index here
// would name some other card.
if len(s.GPUIndices) > 0 && !s.OnMIG() {
env["GGML_VK_VISIBLE_DEVICES"] = joinInts(s.GPUIndices)
}
return []Command{{
+49
View File
@@ -0,0 +1,49 @@
package engines
import "testing"
const migUUID = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f"
func TestAShardOnAMIGInstanceIsGivenItByUUID(t *testing.T) {
// Placement numbers card 0's instance 1 as 1001, which CUDA has never
// heard of: CUDA_VISIBLE_DEVICES=1001 is no device at all, and =1 would
// be the wrong card.
s := singleNodeSpec()
s.GPUIndices, s.GPUUUIDs = []int{1001}, []string{migUUID}
s.Profile.TP = 1
env := envMap(buildOne(t, &VLLM{}, s).Env)
if env["CUDA_VISIBLE_DEVICES"] != migUUID {
t.Errorf("vLLM: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"])
}
l := llamaSpec([]int{1001})
l.GPUUUIDs = s.GPUUUIDs
cmds, err := (&LlamaCpp{}).Commands(l)
if err != nil {
t.Fatal(err)
}
env = envMap(cmds[0].Env)
if env["CUDA_VISIBLE_DEVICES"] != migUUID {
t.Errorf("llama.cpp: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"])
}
// Vulkan does not see MIG instances, and an index for it would name
// some other card.
if v, set := env["GGML_VK_VISIBLE_DEVICES"]; set {
t.Errorf("GGML_VK_VISIBLE_DEVICES=%q on a MIG instance", v)
}
if v, _ := flagValue(cmds[0].Args, "--n-gpu-layers"); v != "999" {
t.Errorf("the model was not offloaded to the instance: --n-gpu-layers %q", v)
}
}
func TestAShardOnWholeCardsIsStillGivenThemByIndex(t *testing.T) {
env := envMap(buildOne(t, &VLLM{}, singleNodeSpec()).Env)
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
}
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if env := envMap(cmds[0].Env); env["GGML_VK_VISIBLE_DEVICES"] != "2" {
t.Errorf("GGML_VK_VISIBLE_DEVICES=%q", env["GGML_VK_VISIBLE_DEVICES"])
}
}
+218
View File
@@ -0,0 +1,218 @@
package engines
import (
"context"
"fmt"
"strconv"
)
// TRTLLMConfig locates a TensorRT-LLM the operator installed.
//
// llmgw does not install TensorRT-LLM. It is a pip package pinned to one
// CUDA, one PyTorch and an MPI the machine must provide, and NVIDIA's own
// advice is its container; an installer that guessed at all of that would
// fail on most machines it met, a long way into a large download. So it is
// brought by the operator — a virtual environment, or NVIDIA's container —
// and named here.
type TRTLLMConfig struct {
// Python is the interpreter of the environment TensorRT-LLM is installed
// in: <venv>/bin/python for a virtual environment, python3 in NVIDIA's
// container. Naming it is what makes this node offer the engine; empty
// offers nothing.
//
// The interpreter rather than trtllm-serve, because llmgw starts the
// server through a few lines of its own that make it demand the
// gateway's key, which trtllm-serve has no option for.
Python string `koanf:"python"`
}
// TRTLLM builds TensorRT-LLM launch commands: its OpenAI-compatible server,
// trtllm-serve, on one node.
//
// Written against TensorRT-LLM's documented command line and server, and not
// yet run on hardware by anyone working on llmgw: no datacenter GPU was at
// hand. `llmgw ctl engine-check trtllm` asks an installed one whether it
// takes the flags built here.
type TRTLLM struct{ cfg TRTLLMConfig }
// Name identifies the engine in the catalog.
func (*TRTLLM) Name() string { return "trtllm" }
// HealthPath is trtllm-serve's liveness endpoint: 200 once the executor is
// up, 503 while it is not. The supervisor's /v1/models step is what says the
// model is being served.
func (*TRTLLM) HealthPath() string { return "/health" }
// MetricsPath is trtllm-serve's Prometheus endpoint. Its /metrics is a
// different thing — a JSON list of iteration statistics, drained by each
// read — and a scrape there would both fail to parse and steal the numbers
// from anyone else reading them.
func (*TRTLLM) MetricsPath() string { return "/prometheus/metrics" }
// ServesHTTP reports whether this rank answers the API; a TensorRT-LLM
// replica is one node, and so one rank.
func (*TRTLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
// Validate checks the profile against the GPUs it would run on.
//
// One node: trtllm-serve spans machines only when started under MPI or
// Slurm from one of them (trtllm-llmapi-launch), and the agents each start
// their own node's part, which is not that. Ampere or newer: the oldest
// generation current TensorRT-LLM builds kernels for.
func (*TRTLLM) Validate(p Profile, gpus []GPUInfo) error {
if p.Nodes > 1 {
return fmt.Errorf("profile %q: trtllm runs a replica on one node; "+
"use vllm or sglang for a %d-node replica", p.Name, p.Nodes)
}
if err := validateCommon(p, gpus); err != nil {
return err
}
return requireComputeCapability(p, gpus, 8.0, "TensorRT-LLM")
}
// Commands returns the processes for one shard, which is one process. Tensor
// and pipeline parallel ranks on the node are trtllm-serve's own MPI workers,
// started by it.
func (t *TRTLLM) Commands(s ShardSpec) ([]Command, error) {
if s.NNodes > 1 {
return nil, fmt.Errorf("trtllm: a replica spanning %d nodes is not something llmgw starts; "+
"TensorRT-LLM runs one node per replica here", s.NNodes)
}
p := s.Profile
args := []string{"-c", trtllmLauncher,
// The subcommand by name: trtllm-serve takes a bare model as "serve",
// but saying it costs nothing and does not depend on that default.
"serve", p.ModelPath,
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
"--tp_size", strconv.Itoa(p.TP),
}
if p.PP > 1 {
args = append(args, "--pp_size", strconv.Itoa(p.PP))
}
if p.ServedModelName != "" {
// Without it the model is named after its directory, and the agent's
// readiness check, which looks for this name in /v1/models, never
// finds it.
args = append(args, "--served_model_name", p.ServedModelName)
}
// What makes the queue and the cache measurable: the options file the
// launcher writes turns on the Prometheus endpoint and the iteration
// statistics its gauges are set from. A placeholder here, so the flag is
// one engine-check sees and checks; the launcher replaces it.
args = append(args, "--extra_llm_api_options", trtllmOptionsPlaceholder)
args = append(args, p.Args...)
env := baseEnv(s)
if s.APIKey != "" {
env[trtllmKeyEnv] = s.APIKey
}
// TensorRT-LLM reports usage to NVIDIA unless told not to. A gateway on
// somebody's own network does not phone home on their behalf, and a
// model's profile cannot make it: these are set after the profile's env.
env["TRTLLM_NO_USAGE_STATS"] = "1"
env["DO_NOT_TRACK"] = "1"
return []Command{{Name: "trtllm-serve", Path: t.python(), Args: args, Env: materializeEnv(env)}}, nil
}
// python is the interpreter to start trtllm-serve with.
func (t *TRTLLM) python() string {
if t.cfg.Python != "" {
return t.cfg.Python
}
return "python3"
}
// Configured reports whether the configuration names TensorRT-LLM's
// interpreter, which is the only way a node offers it.
func (t *TRTLLM) Configured() bool { return t.cfg.Python != "" }
// Probe reports the installed TensorRT-LLM's version.
//
// From the package's metadata rather than by importing it: the import loads
// CUDA, MPI and PyTorch and takes longer than a probe may, while what
// placement needs is whether the package is there. Whether it runs is
// engine-check's question, and the replica's.
func (t *TRTLLM) Probe(ctx context.Context) (string, error) {
return versionOfModule(ctx, t.python(), "-c",
"import importlib.metadata as m, importlib.util as u, sys\n"+
"if u.find_spec('tensorrt_llm') is None: sys.exit(1)\n"+
"sys.stdout.write(m.version('tensorrt_llm'))")
}
// HelpCommand is how to ask trtllm-serve for its help: through its module,
// since the command itself is llmgw's launcher.
func (t *TRTLLM) HelpCommand(Command) (string, []string) {
return t.python(), []string{"-m", "tensorrt_llm.commands.serve", "serve", "--help"}
}
const (
trtllmKeyEnv = "LLMGW_ENGINE_KEY"
trtllmOptionsPlaceholder = "@llmgw-options"
)
// trtllmLauncher starts trtllm-serve inside the process, after two things
// trtllm-serve has no flag for.
//
// The key. trtllm-serve checks none, and an engine listens on the node's
// management address, where without one anybody who can reach the port talks
// to the model past the gateway's authentication, quotas and accounting. So
// every FastAPI application the process makes — trtllm-serve's server is one
// — is given a middleware demanding it, before trtllm-serve imports anything.
// /health is left open: it says only up or down, and a supervisor of
// TensorRT-LLM's own may ask it without a key. The key comes in the
// environment and is taken out of it, so the MPI workers and anything else
// started from here never hold it.
//
// The options. The Prometheus endpoint exists only with return_perf_metrics,
// and its queue gauges move only with enable_iter_perf_stats; both are
// settings of TensorRT-LLM's options file, not flags. The file is written to
// an unnamed temporary file and passed by descriptor: nothing is left on disk
// however the server ends.
//
// trtllm-serve can start further HTTP front ends as copies of its own command
// line (--num_serve_frontends); those would not run this, so the flag is
// refused in a profile, and a copy started anyway would be given a program
// path that cannot exist — /dev/null is no directory — and fail rather than
// serve without the key. Click names itself by the path's last element, so
// trtllm-serve's messages still say trtllm-serve.
const trtllmLauncher = `import hmac, os, sys, tempfile
key = os.environ.pop("` + trtllmKeyEnv + `", "")
if key:
import fastapi
want = ("Bearer " + key).encode()
denied = b'{"error":{"message":"a valid API key is required","type":"invalid_request_error","code":"invalid_api_key"}}'
class RequireKey:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] in ("http", "websocket") and scope.get("path") != "/health":
got = b""
for name, value in scope.get("headers") or ():
if name == b"authorization":
got = value
if not hmac.compare_digest(got, want):
if scope["type"] == "websocket":
await send({"type": "websocket.close", "code": 1008})
return
await send({"type": "http.response.start", "status": 401,
"headers": [(b"content-type", b"application/json")]})
await send({"type": "http.response.body", "body": denied})
return
await self.app(scope, receive, send)
plain_init = fastapi.FastAPI.__init__
def init_with_key(self, *a, **kw):
plain_init(self, *a, **kw)
self.add_middleware(RequireKey)
fastapi.FastAPI.__init__ = init_with_key
args = sys.argv[1:]
if "` + trtllmOptionsPlaceholder + `" in args:
options = tempfile.TemporaryFile()
options.write(b"return_perf_metrics: true\nenable_iter_perf_stats: true\n")
options.flush()
options.seek(0)
args[args.index("` + trtllmOptionsPlaceholder + `")] = "/dev/fd/%d" % options.fileno()
sys.argv = ["/dev/null/trtllm-serve"] + args
from tensorrt_llm.commands.serve import main
main()
`
+316
View File
@@ -0,0 +1,316 @@
package engines
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
)
func trtllmSpec() ShardSpec {
s := singleNodeSpec()
s.Profile.Engine = "trtllm"
s.Profile.ServedModelName = "llama-3.1-70b"
return s
}
func TestTRTLLMRunsTrtllmServeWithTheFlagsItDocuments(t *testing.T) {
s := trtllmSpec()
s.BindAddr = "10.0.0.147"
s.Profile.Args = []string{"--max_batch_size", "64"}
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: "/opt/trtllm/bin/python"}}, s)
if cmd.Path != "/opt/trtllm/bin/python" {
t.Errorf("Path=%q, expected the configured interpreter", cmd.Path)
}
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], "from tensorrt_llm.commands.serve import main") {
t.Fatalf("not started through the launcher: %v", cmd.Args[:2])
}
rest := cmd.Args[2:]
if rest[0] != "serve" || rest[1] != "/models/llama" {
t.Errorf("expected serve and the model first, got %v", rest[:2])
}
for flag, want := range map[string]string{
"--host": "10.0.0.147", "--port": "20000", "--tp_size": "4",
"--served_model_name": "llama-3.1-70b",
"--extra_llm_api_options": trtllmOptionsPlaceholder,
} {
if got := argValue(rest, flag); got != want {
t.Errorf("%s=%q, want %q (args %v)", flag, got, want, rest)
}
}
// One pipeline stage is the default, and saying so would only be one
// more flag for a version to rename.
if slices.Contains(rest, "--pp_size") {
t.Errorf("--pp_size given for a single stage: %v", rest)
}
// The profile's own, last, so a tuning flag is the operator's to set.
if got := rest[len(rest)-2:]; !slices.Equal(got, []string{"--max_batch_size", "64"}) {
t.Errorf("profile args are not last: %v", rest)
}
s.Profile.TP, s.Profile.PP = 2, 2
if got := argValue(buildOne(t, &TRTLLM{}, s).Args, "--pp_size"); got != "2" {
t.Errorf("--pp_size=%q for two stages", got)
}
if got := buildOne(t, &TRTLLM{}, s).Path; got != "python3" {
t.Errorf("unconfigured, the interpreter is %q", got)
}
}
func TestTRTLLMsKeyIsNotOnItsCommandLine(t *testing.T) {
s := trtllmSpec()
s.APIKey = "k1"
cmd := buildOne(t, &TRTLLM{}, s)
for _, a := range cmd.Args {
if strings.Contains(a, "k1") && a != cmd.Args[1] {
t.Errorf("the key is in the arguments: %v", cmd.Args)
}
}
if envMap(cmd.Env)[trtllmKeyEnv] != "k1" {
t.Error("the key is not in the environment")
}
}
func TestTRTLLMDoesNotReportUsageToNVIDIA(t *testing.T) {
s := trtllmSpec()
s.Profile.Env = map[string]string{"TRTLLM_NO_USAGE_STATS": "0", "TRTLLM_ENABLE_PDL": "1"}
env := envMap(buildOne(t, &TRTLLM{}, s).Env)
if env["TRTLLM_NO_USAGE_STATS"] != "1" || env["DO_NOT_TRACK"] != "1" {
t.Errorf("usage reporting is not off: %q %q", env["TRTLLM_NO_USAGE_STATS"], env["DO_NOT_TRACK"])
}
// Tuning still reaches it.
if env["TRTLLM_ENABLE_PDL"] != "1" {
t.Error("a profile's TensorRT-LLM tuning was dropped")
}
}
func TestTRTLLMIsReadFromItsPrometheusEndpoint(t *testing.T) {
// /metrics on trtllm-serve is JSON, and each read drains what it returns.
if got := (&TRTLLM{}).MetricsPath(); got != "/prometheus/metrics" {
t.Errorf("MetricsPath=%q", got)
}
if !(&TRTLLM{}).ServesHTTP(trtllmSpec()) {
t.Error("the one rank of a replica serves its API")
}
}
func TestTRTLLMRunsAReplicaOnOneNode(t *testing.T) {
p := trtllmSpec().Profile
p.Nodes, p.GPUsPerNode, p.TP = 2, 4, 8
err := (&TRTLLM{}).Validate(p, nil)
if err == nil || !strings.Contains(err.Error(), "one node") {
t.Errorf("a two-node profile was accepted: %v", err)
}
s := multiNodeSpec(0)
s.Profile.Engine = "trtllm"
if _, err := (&TRTLLM{}).Commands(s); err == nil {
t.Error("a command was built for a shard of a two-node replica")
}
}
func TestTRTLLMNeedsAmpereOrNewer(t *testing.T) {
p := trtllmSpec().Profile
p.TP = 2
v100 := []GPUInfo{{Index: 0, Name: "Tesla V100", ComputeCapability: "7.0"}, {Index: 1, Name: "Tesla V100", ComputeCapability: "7.0"}}
if err := (&TRTLLM{}).Validate(p, v100); err == nil {
t.Error("a V100 was accepted")
}
h100 := []GPUInfo{{Index: 0, Name: "H100", ComputeCapability: "9.0"}, {Index: 1, Name: "H100", ComputeCapability: "9.0"}}
if err := (&TRTLLM{}).Validate(p, h100); err != nil {
t.Errorf("two H100s were refused: %v", err)
}
}
func TestTRTLLMProfilesMayNotOpenWhatTheAgentCloses(t *testing.T) {
for _, args := range [][]string{
{"--trust_remote_code"},
{"--num_serve_frontends", "4"},
{"--grpc"},
{"--config", "/etc/shadow"},
{"--extra_llm_api_options=/tmp/x.yaml"},
{"--served_model_name", "other"},
{"--middleware", "evil.mw"},
{"--tp_size", "8"},
} {
if err := UnsafeArgs("trtllm", args); err == nil {
t.Errorf("%v was allowed", args)
}
}
if err := UnsafeArgs("trtllm", []string{"--max_batch_size", "64", "--set", "kv_cache_config.enable_block_reuse=false",
"--free_gpu_memory_fraction", "0.85"}); err != nil {
t.Errorf("tuning was refused: %v", err)
}
}
func TestTRTLLMIsOfferedOnlyWhenItsPackageIsThere(t *testing.T) {
there := script(t, "python3", `echo "1.3.0rc29"`)
r := NewRegistry(Config{})
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: there}}}
if got := r.ProbeConfigured(t.Context()); got["trtllm"] != "1.3.0rc29" {
t.Errorf("expected the version, got %v", got)
}
// An interpreter without the package, which every machine has.
absent := script(t, "python3", "exit 1")
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: absent}}}
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
t.Errorf("reported without its package: %v", got)
}
// Not named, not offered: python3 on PATH is nobody's TensorRT-LLM.
r.engines = map[string]Engine{"trtllm": &TRTLLM{}}
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
t.Errorf("offered without being configured: %v", got)
}
}
// fakeTRTLLM is a Python path holding stand-ins for fastapi and
// tensorrt_llm: enough of each for the launcher to do what it does in front
// of the real server, and for the stand-in server to report what it saw.
func fakeTRTLLM(t *testing.T) string {
t.Helper()
dir := t.TempDir()
files := map[string]string{
"fastapi/__init__.py": `
class FastAPI:
def __init__(self, *a, **kw):
self.user_middleware = []
def add_middleware(self, cls, **kw):
self.user_middleware.append(cls)
`,
"tensorrt_llm/__init__.py": "",
"tensorrt_llm/commands/__init__.py": "",
"tensorrt_llm/commands/serve.py": `
import asyncio, json, os, sys
import fastapi
def main():
args = sys.argv[1:]
with open(args[args.index("--extra_llm_api_options") + 1]) as f:
options = f.read()
app = fastapi.FastAPI()
async def server(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
handler = server
for mw in reversed(app.user_middleware):
handler = mw(handler)
async def ask(path, auth=None):
headers = [(b"authorization", auth.encode())] if auth else []
sent = []
async def send(msg):
sent.append(msg)
async def receive():
return {"type": "http.request"}
await handler({"type": "http", "path": path, "headers": headers}, receive, send)
return sent[0]["status"]
async def run():
return {
"health": await ask("/health"),
"anonymous": await ask("/v1/chat/completions"),
"wrong": await ask("/v1/models", "Bearer nope"),
"keyed": await ask("/prometheus/metrics", "Bearer k1"),
}
out = asyncio.run(run())
out.update(options=options, argv0=sys.argv[0], args=args,
key_left=os.environ.get("LLMGW_ENGINE_KEY"))
print(json.dumps(out))
`,
}
for name, body := range files {
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
type launched struct {
Health, Anonymous, Wrong, Keyed int
Options, Argv0 string
Args []string
KeyLeft *string `json:"key_left"`
}
func launch(t *testing.T, s ShardSpec) launched {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("no python3 here to run the launcher with")
}
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: python}}, s)
c := exec.Command(cmd.Path, cmd.Args...) //nolint:gosec // the command under test
c.Env = append(cmd.Env, "PYTHONPATH="+fakeTRTLLM(t))
out, err := c.Output()
if err != nil {
var stderr string
if ee, ok := err.(*exec.ExitError); ok {
stderr = string(ee.Stderr)
}
t.Fatalf("the launcher failed: %v\n%s", err, stderr)
}
var got launched
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("%v: %s", err, out)
}
return got
}
func TestTheTRTLLMLauncherDemandsTheKey(t *testing.T) {
s := trtllmSpec()
s.APIKey = "k1"
got := launch(t, s)
if got.Anonymous != 401 || got.Wrong != 401 {
t.Errorf("served without the key: anonymous %d, wrong key %d", got.Anonymous, got.Wrong)
}
if got.Keyed != 200 {
t.Errorf("refused the key: %d", got.Keyed)
}
// Up or down is all /health says, and it is asked without one.
if got.Health != 200 {
t.Errorf("/health answered %d", got.Health)
}
// Taken out of the environment, so nothing the server starts holds it.
if got.KeyLeft != nil {
t.Errorf("the key was left in the environment: %q", *got.KeyLeft)
}
// A copy of this command line, as trtllm-serve's extra front ends are,
// names no program that exists and so cannot serve without the key.
if _, err := os.Stat(got.Argv0); err == nil || filepath.Base(got.Argv0) != "trtllm-serve" {
t.Errorf("argv[0] is %q", got.Argv0)
}
}
func TestTheTRTLLMLauncherTurnsOnTheMetricsTheRouterReads(t *testing.T) {
got := launch(t, trtllmSpec())
for _, want := range []string{"return_perf_metrics: true", "enable_iter_perf_stats: true"} {
if !strings.Contains(got.Options, want) {
t.Errorf("the options file lacks %q: %q", want, got.Options)
}
}
path := argValue(got.Args, "--extra_llm_api_options")
if !strings.HasPrefix(path, "/dev/fd/") {
t.Errorf("the options were not passed by descriptor: %q", path)
}
if slices.Contains(got.Args, trtllmOptionsPlaceholder) {
t.Error("the placeholder reached trtllm-serve")
}
// Without a key there is nothing to demand.
if got.Anonymous != 200 {
t.Errorf("with no key set, a request was refused: %d", got.Anonymous)
}
}
+27
View File
@@ -24,6 +24,7 @@ var envFamilies = []string{
"CUDA_", "NCCL_", "NVIDIA_", "NVTE_", "GLOO_", "UCX_", "TORCH_", "PYTORCH_",
"VLLM_", "SGLANG_", "GGML_", "LLAMA_", "OMP_", "MKL_", "OPENBLAS_", "KMP_",
"HF_", "TRANSFORMERS_", "TOKENIZERS_", "TRITON_", "FLASHINFER_",
"TRTLLM_", "TLLM_",
}
// envExact are single variables that are tuning too.
@@ -118,6 +119,32 @@ var flagsNever = map[string]map[string]string{
"--model-path": "is the model's source, set by the agent", "--served-model-name": "is set by the agent",
"--lora-paths": "loads files", "--download-dir": "writes files",
},
// trtllm-serve spells its flags with underscores, so the shared list's
// hyphenated names do not reach them.
"trtllm": {
"--trust_remote_code": "runs Python the model repository ships",
"--served_model_name": "is set by the agent",
"--config": "reads a file; the agent writes the options file itself",
"--extra_llm_api_options": "reads a file; the agent writes the options file itself",
"--tokenizer": "reads another repository or path",
"--custom_tokenizer": "imports Python", "--post_processor_hook": "imports Python",
"--custom_module_dirs": "imports Python", "--middleware": "imports Python",
"--chat_template": "reads a file",
"--metadata_server_config_file": "reads a file",
"--visual_gen_args": "reads a file",
"--report_addr": "writes a file",
"--otlp_traces_endpoint": "sends request traces elsewhere",
"--disagg_cluster_uri": "joins the server to another cluster",
"--server_role": "makes the server part of another cluster",
"--allow_request_chat_template": "lets a request supply the chat template",
"--grpc": "serves without the key the agent sets",
"--num_serve_frontends": "starts front ends that do not demand the key the agent sets",
"--telemetry": "sends usage to NVIDIA",
"--tensor_parallel_size": "is set by the agent from tp",
"--tp_size": "is set by the agent from tp",
"--pipeline_parallel_size": "is set by the agent from pp",
"--pp_size": "is set by the agent from pp",
},
}
// unsafeFlag is why an engine may not be given a flag, or "".
+40
View File
@@ -0,0 +1,40 @@
package agent
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
func TestAShardPlacedOnAMIGInstanceIsLaunchedWithItsUUID(t *testing.T) {
// The control plane sends device 1001: card 0's instance 1, by the number
// placement gives it. Only this node knows the instance's UUID, which is
// the one thing CUDA will open it by.
a := &Agent{}
a.inv.GPUs = []discovery.GPU{
{Index: 1000, UUID: "MIG-aaa", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 0, Profile: "3g.40gb"}},
{Index: 1001, UUID: "MIG-bbb", MIG: &discovery.MIGDevice{ParentIndex: 0, Device: 1, Profile: "3g.40gb"}},
{Index: 1, UUID: "GPU-whole"},
}
spec := engines.ShardSpec{GPUIndices: []int{1001}}
a.localizeSpec(&spec)
if got := engines.NCCLEnv(spec)["CUDA_VISIBLE_DEVICES"]; got != "MIG-bbb" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q, want MIG-bbb", got)
}
// A whole card is given by index, as it always was.
whole := engines.ShardSpec{GPUIndices: []int{1}}
a.localizeSpec(&whole)
if whole.GPUUUIDs != nil || engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"] != "1" {
t.Errorf("a whole card was given as %v / %q", whole.GPUUUIDs, engines.NCCLEnv(whole)["CUDA_VISIBLE_DEVICES"])
}
// A device this node does not have is not guessed at.
gone := engines.ShardSpec{GPUIndices: []int{1002}}
a.localizeSpec(&gone)
if gone.GPUUUIDs != nil {
t.Errorf("an unknown device was named %v", gone.GPUUUIDs)
}
}
+15 -1
View File
@@ -20,7 +20,11 @@ import (
"strings"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// Want is one engine at one version.
@@ -251,7 +255,17 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
//
// Traced as "install engine", a trace of its own: an install is not part of
// any one start, and runs once per version rather than once per replica.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) (err error) {
ctx, span := obs.Tracer("gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision").Start(ctx, "install engine",
trace.WithAttributes(
attribute.String("llmgw.engine", w.Engine),
attribute.String("llmgw.engine_version", w.Version),
attribute.String("llmgw.engine_build", w.Variant),
))
defer func() { obs.End(span, err) }()
if b.Python != "" {
return in.installPython(ctx, r, w, b, dir, st)
}
+75 -6
View File
@@ -18,9 +18,19 @@ import (
"syscall"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// tracerName names the spans the supervisor opens. There is no restart span
// here because the supervisor never restarts an engine on its own: a restart
// is the control plane's decision, traced there, and reaches a node as a stop
// and a start.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
// Config configures the supervisor.
type Config struct {
// WorkDir holds pid files and per-shard scratch space.
@@ -195,7 +205,25 @@ var ErrAlreadyRunning = errors.New("shard is already running")
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
// rather than a second process, because the control plane retries commands
// (plan §5.6).
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
//
// Traced as "launch engine": from the first process to the engine's being
// started, which for a multi-node worker includes waiting for its leader's
// rendezvous point. Loading the model is the readiness wait's, not this.
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (_ *Shard, err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "launch engine", trace.WithAttributes(
attribute.String("llmgw.shard", id),
attribute.String("llmgw.engine", spec.Profile.Engine),
attribute.Int("llmgw.processes", len(cmds)),
))
defer func() {
if errors.Is(err, ErrAlreadyRunning) {
// A repeated command, which is not this start failing.
span.SetAttributes(attribute.Bool("llmgw.already_running", true))
span.End()
return
}
obs.End(span, err)
}()
if len(cmds) == 0 {
return nil, fmt.Errorf("shard %s: no commands to run", id)
}
@@ -301,6 +329,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
sh.pid = c.Process.Pid
sh.state = Starting
sh.mu.Unlock()
trace.SpanFromContext(ctx).SetAttributes(attribute.Int("llmgw.pid", c.Process.Pid))
if err := s.writeState(sh); err != nil {
s.log.Warn("could not write pid file; adoption after a restart will not work",
@@ -337,7 +366,13 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
}
// waitForPort blocks until addr accepts a connection.
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
//
// Traced: a worker's start that took ten minutes because its leader was
// still loading reads as exactly that.
func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for rendezvous",
trace.WithAttributes(attribute.String("llmgw.rendezvous", addr)))
defer func() { obs.End(span, err) }()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -416,7 +451,26 @@ func (s *Supervisor) wait(sh *Shard) {
// The timeout comes from the profile because it is model-dependent: loading a
// 70B model across two nodes legitimately takes tens of minutes, while a small
// model failing to load for ten minutes means something is wrong.
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
//
// Traced as "wait for engine": the model loading, which is usually most of a
// start. The span says how many times the engine was asked and what it last
// answered, so one that never came up says why.
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
attribute.String("llmgw.shard", sh.ID),
attribute.String("llmgw.readiness", "health check"),
attribute.String("llmgw.timeout", timeout.String()),
))
polls := 0
var answer error
defer func() {
span.SetAttributes(attribute.Int("llmgw.polls", polls))
if answer != nil && err != nil {
span.SetAttributes(attribute.String("llmgw.last_answer", answer.Error()))
}
obs.End(span, err)
}()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -432,7 +486,8 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
last := strings.Join(sh.Logs(10), "\n")
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
case <-t.C:
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
polls++
if answer = s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); answer != nil {
continue
}
sh.setState(Ready, nil)
@@ -449,7 +504,13 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
// leader answering, which only happens once every worker has joined. Polling a
// worker's port instead would time out and fail a replica that is coming up
// perfectly well.
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) (err error) {
_, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
attribute.String("llmgw.shard", sh.ID),
attribute.String("llmgw.readiness", "stays alive"),
attribute.String("llmgw.timeout", s.cfg.AliveGrace.String()),
))
defer func() { obs.End(span, err) }()
sh.setState(Loading, nil)
select {
@@ -483,7 +544,14 @@ func (s *Supervisor) Drain(id string) error {
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
// StopTimeout.
func (s *Supervisor) Stop(ctx context.Context, id string) error {
//
// Traced as "stop engine", saying whether it had to be killed: an engine
// that ignores SIGTERM for a minute is a minute its replacement waits for
// the GPU.
func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop engine",
trace.WithAttributes(attribute.String("llmgw.shard", id)))
defer func() { obs.End(span, err) }()
sh, err := s.Get(id)
if err != nil {
return err
@@ -527,6 +595,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
// force kills the group outright. exec.Cmd's cancel does this for shards we
// started; an adopted shard has none, so signal its group directly.
force := func() {
span.SetAttributes(attribute.Bool("llmgw.killed", true))
if sh.cancel != nil {
sh.cancel()
return
+135
View File
@@ -0,0 +1,135 @@
package agent
import (
"testing"
"time"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAStartOnTheNodeIsInTheControlPlanesTrace(t *testing.T) {
rec := obstest.Record(t)
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = servingEngine(t)
})
// The control plane's side, as the reconciler sends it: the command
// dispatched under a span of its own, over the real stream.
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start",
trace.WithSpanKind(trace.SpanKindProducer))
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-a")); err != nil {
t.Fatal(err)
}
sent.End()
h.rec.await(t, 30*time.Second, func(s status) bool {
return s.shardID == "shard-a" && s.state == supervisor.Ready
})
// The node's part ends just after it reports the shard ready.
tid := sent.SpanContext().TraceID()
node := rec.Await(t, tid, "start shard", 5*time.Second)
if node.Parent().SpanID() != sent.SpanContext().SpanID() {
t.Fatal("the node's start is not beneath the control plane's command")
}
if node.SpanKind() != trace.SpanKindConsumer {
t.Errorf("the node's start is a %v span", node.SpanKind())
}
if a := obstest.Attrs(node); a["llmgw.node"] != "n1" || a["llmgw.replica"] != "rep-1" ||
a["llmgw.model"] != "test-model" || a["llmgw.engine"] != "vllm" {
t.Errorf("the node's start does not say what it started: %v", a)
}
if node.Status().Code == codes.Error {
t.Errorf("a start that served is %v", node.Status())
}
for _, name := range []string{"fetch weights", "launch engine", "wait for engine"} {
if !obstest.ChildOf(rec.One(t, tid, name), node) {
t.Errorf("%s is not beneath the node's start", name)
}
}
if a := obstest.Attrs(rec.One(t, tid, "fetch weights")); a["llmgw.weights.outcome"] != "local" {
t.Errorf("the weights do not say how they were found: %v", a)
}
if a := obstest.Attrs(rec.One(t, tid, "wait for engine")); a["llmgw.polls"] == "" || a["llmgw.polls"] == "0" {
t.Errorf("the readiness wait does not say how often it asked: %v", a)
}
// A stop joins the trace it was sent in, too.
sctx, stop := obs.Tracer("test").Start(t.Context(), "stop replica")
if err := h.svc.StopShard(sctx, "n1", "shard-a"); err != nil {
t.Fatal(err)
}
stop.End()
h.rec.await(t, 15*time.Second, func(s status) bool {
return s.shardID == "shard-a" && s.state == supervisor.Stopped
})
stid := stop.SpanContext().TraceID()
nodeStop := rec.Await(t, stid, "stop shard", 5*time.Second)
if !obstest.ChildOf(nodeStop, rec.One(t, stid, "stop replica")) {
t.Error("the node's stop is not beneath the control plane's")
}
if !obstest.ChildOf(rec.One(t, stid, "stop engine"), nodeStop) {
t.Error("stopping the engine is not beneath the node's stop")
}
}
func TestAnAdoptionIsTracedAndItsStopsAreInIt(t *testing.T) {
rec := obstest.Record(t)
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
})
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-orphaned")); err != nil {
t.Fatal(err)
}
h.rec.await(t, 15*time.Second, func(s status) bool {
return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
})
// The control plane, restarted, claims nothing the node is running.
h.rec.markUnclaimed("shard-orphaned")
h.ln.drop()
h.rec.await(t, 20*time.Second, func(s status) bool {
return s.shardID == "shard-orphaned" && s.state == supervisor.Stopped
})
adopt := rec.Named("adopt")
if len(adopt) != 1 {
t.Fatalf("%d adoptions traced", len(adopt))
}
if a := obstest.Attrs(adopt[0]); a["llmgw.node"] != "n1" || a["llmgw.shards"] != "1" || a["llmgw.unclaimed"] != "1" {
t.Errorf("the adoption does not say what it found: %v", a)
}
stop := rec.Await(t, adopt[0].SpanContext().TraceID(), "stop shard", 5*time.Second)
if !obstest.ChildOf(stop, adopt[0]) {
t.Error("stopping the unclaimed shard is not beneath the adoption")
}
}
func TestAStartThatFailsOnTheNodeSaysWhyInTheTrace(t *testing.T) {
rec := obstest.Record(t)
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo "CUDA out of memory" >&2; exit 1`)
})
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start")
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-b")); err != nil {
t.Fatal(err)
}
sent.End()
h.rec.await(t, 30*time.Second, func(s status) bool {
return s.shardID == "shard-b" && s.state == supervisor.Failed
})
tid := sent.SpanContext().TraceID()
node := rec.Await(t, tid, "start shard", 5*time.Second)
if node.Status().Code != codes.Error {
t.Errorf("a start that failed is %v", node.Status())
}
if wait := rec.One(t, tid, "wait for engine"); wait.Status().Code != codes.Error {
t.Errorf("the readiness wait of an engine that exited is %v", wait.Status())
}
}
+59 -2
View File
@@ -26,6 +26,11 @@ import (
"strings"
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// Progress reports how a fetch is going, for anything that wants to say so.
@@ -80,13 +85,48 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
// local disk, a Lustre mount or an image exported over the fabric, and a
// downloader that second-guessed that would copy a hundred gigabytes that were
// already present.
func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
//
// Traced as "fetch weights", with what it came to — the file was local,
// already cached, downloaded, or downloaded by another shard this one waited
// for — since that is most of the difference between a replica that starts
// in a minute and one that takes an hour.
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
defer func() {
span.SetAttributes(attribute.String("llmgw.weights.path", path))
obs.End(span, err)
}()
return f.ensure(ctx, source)
}
// tracerName names the spans this package opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
// outcome says on the current span how a source was made present.
func outcome(ctx context.Context, how string) {
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.weights.outcome", how))
}
// redact is a source fit for a trace: a URL without its credentials or
// query, which is where a token would be.
func redact(source string) string {
u, err := url.Parse(source)
if err != nil || u.Scheme == "" || u.Host == "" {
return source
}
u.User, u.RawQuery, u.Fragment = nil, "", ""
return u.String()
}
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
if source == "" {
return "", fmt.Errorf("no source given")
}
// A version rather than a file. Everything below assumes one URL is one
// file, and a set has to be handled before any of it.
if IsSet(source) {
outcome(ctx, "set")
return f.ensureSet(ctx, source)
}
if !isRemote(source) {
@@ -95,9 +135,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
// was never meant to be one. vLLM and SGLang resolve those
// themselves, and doing it for them would be both a duplicate
// downloader and a change in documented behaviour.
outcome(ctx, "left to the engine")
return source, nil
}
if _, err := os.Stat(source); err == nil {
outcome(ctx, "local")
return source, nil
}
return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
@@ -108,6 +150,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
return "", err
}
if _, err := os.Stat(target); err == nil {
outcome(ctx, "cached")
return target, nil
}
@@ -117,6 +160,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
}
if call, running := f.inflight[source]; running {
f.mu.Unlock()
outcome(ctx, "fetched by another shard")
select {
case <-call.done:
return call.path, call.err
@@ -128,6 +172,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
f.inflight[source] = call
f.mu.Unlock()
outcome(ctx, "downloaded")
call.path, call.err = f.fetch(ctx, source, target)
close(call.done)
@@ -147,7 +192,15 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err
// The two differ for a set: the shards waiting on it asked for the version, not
// for the fourth file in it, and progress reported under the file's own URL
// reaches nobody. What they see is one number climbing across the whole set.
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) {
//
// Each file is a "download" span, with how many bytes it took and where a
// resumed one started from.
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
defer func() { obs.End(span, err) }()
url, err := resolveURL(source)
if err != nil {
return "", err
@@ -208,6 +261,10 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
total := existing + resp.ContentLength
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
span.SetAttributes(
attribute.Int64("llmgw.weights.resumed_from", existing),
attribute.Int64("llmgw.weights.bytes", written),
)
closeErr := out.Close()
if err != nil {
// The part file is kept on purpose, so the next attempt resumes.
+43
View File
@@ -0,0 +1,43 @@
package weights
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
rec := obstest.Record(t)
srv := serveBytes(t, []byte("forty gigabytes, give or take"), nil)
f := &Fetcher{CacheDir: t.TempDir()}
src := strings.Replace(srv.URL, "http://", "http://user:secret@", 1) + "/m/model.gguf?token=hunter2"
if _, err := f.Ensure(t.Context(), src); err != nil {
t.Fatal(err)
}
fetch := rec.One(t, [16]byte{}, "fetch weights")
a := obstest.Attrs(fetch)
if a["llmgw.weights.outcome"] != "downloaded" {
t.Errorf("the first fetch says %v", a)
}
if strings.Contains(a["llmgw.weights.source"], "secret") || strings.Contains(a["llmgw.weights.source"], "hunter2") {
t.Errorf("the trace holds the source's credentials: %s", a["llmgw.weights.source"])
}
dl := rec.One(t, fetch.SpanContext().TraceID(), "download")
if !obstest.ChildOf(dl, fetch) || obstest.Attrs(dl)["llmgw.weights.bytes"] != "29" {
t.Errorf("the download does not say how much it moved: %v", obstest.Attrs(dl))
}
// The second start on this node finds it here.
if _, err := f.Ensure(t.Context(), src); err != nil {
t.Fatal(err)
}
again := rec.Named("fetch weights")
if len(again) != 2 || obstest.Attrs(again[1])["llmgw.weights.outcome"] != "cached" {
t.Errorf("the second fetch says %v", obstest.Attrs(again[len(again)-1]))
}
if len(rec.Named("download")) != 1 {
t.Error("a cached file was downloaded again")
}
}
+7 -1
View File
@@ -120,8 +120,14 @@ func API(sink Sink, actor func(*http.Request) string, log *slog.Logger, next htt
}
var body []byte
if r.Body != nil {
// The first 64 KiB are what is recorded; the handler is given the
// whole body, or a larger one — a quality suite sent whole — would
// reach it cut off.
body, _ = io.ReadAll(io.LimitReader(r.Body, 64<<10))
r.Body = io.NopCloser(bytes.NewReader(body))
r.Body = struct {
io.Reader
io.Closer
}{io.MultiReader(bytes.NewReader(body), r.Body), r.Body}
}
sw := &statusWriter{ResponseWriter: w}
next.ServeHTTP(sw, r)
+14
View File
@@ -2,6 +2,7 @@ package audit
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
@@ -58,3 +59,16 @@ func TestEveryChangeToTheAdminAPIIsRecorded(t *testing.T) {
t.Errorf("%+v", e)
}
}
func TestABodyLargerThanWhatIsRecordedReachesTheHandlerWhole(t *testing.T) {
big := `{"content":"` + strings.Repeat("x", 200<<10) + `"}`
var got int
h := API(&memSink{}, func(*http.Request) string { return "ilker" }, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
got = len(b)
}))
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/api/v1/models/m/eval", strings.NewReader(big)))
if got != len(big) {
t.Errorf("the handler was given %d of %d bytes", got, len(big))
}
}
+260
View File
@@ -0,0 +1,260 @@
package eval
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Caller sends a chat completion to a model as the gateway does for itself:
// the router's Replay, past admission, accounting and the guardrails — a
// suite measures the weights, and a tenant's quota or a guardrail's refusal
// would measure something else. It answers the status, the body, and how
// long it took.
type Caller func(ctx context.Context, path, model string, body []byte) (int, []byte, time.Duration, error)
// Options are how a suite is run.
type Options struct {
// Model is the catalog model asked.
Model string
// Concurrency is how many cases are asked at once; 4 unless given.
Concurrency int
// Judge grades the cases scored by a rubric, instead of the suite's.
Judge string
// MinScore is the score to pass at, instead of the suite's threshold.
MinScore float64
// CaseTimeout bounds one case, its judging included; five minutes
// unless given, which a model loading on demand needs.
CaseTimeout time.Duration
// Progress is told as each case finishes.
Progress func(done, total int)
}
// PassAt is the score a case passes at: a judge's 0.5 is a coin toss, and
// counts as passed so a lenient rubric is not a hidden zero.
const PassAt = 0.5
// keptAnswer is how much of an answer is kept with its result.
const keptAnswer = 2000
// Evaluate asks a model every case of a suite and scores the answers. A
// case that got no answer scores 0 with its error; a suite that cannot be
// run at all — a rubric and no judge to read it — is an error before
// anything is asked.
func Evaluate(ctx context.Context, call Caller, s Suite, o Options) (store.EvalRun, error) {
if o.Model == "" {
return store.EvalRun{}, errors.New("no model to ask")
}
judge := o.Judge
if judge == "" {
judge = s.Judge
}
for _, c := range s.Cases {
if c.Judge != "" && c.JudgeModel == "" && judge == "" {
return store.EvalRun{}, fmt.Errorf("case %s is graded by a judge and the suite names none: "+
"give one with --judge MODEL, or judge: in the suite", c.ID)
}
}
threshold := s.Threshold
if o.MinScore > 0 {
threshold = o.MinScore
}
if threshold <= 0 {
threshold = DefaultThreshold
}
workers := o.Concurrency
if workers <= 0 {
workers = 4
}
timeout := o.CaseTimeout
if timeout <= 0 {
timeout = 5 * time.Minute
}
run := store.EvalRun{Model: o.Model, Suite: s.Name, SuiteHash: s.Hash, At: time.Now().UTC(),
Threshold: threshold, Cases: len(s.Cases), Results: make([]store.EvalCase, len(s.Cases))}
if s.NeedsJudge() {
run.Judge = judge
}
todo := make(chan int)
var wg sync.WaitGroup
var mu sync.Mutex
done := 0
for range min(workers, len(s.Cases)) {
wg.Go(func() {
for i := range todo {
cctx, cancel := context.WithTimeout(ctx, timeout)
res := askOne(cctx, call, s, &s.Cases[i], o.Model, judge)
cancel()
mu.Lock()
run.Results[i] = res
done++
if o.Progress != nil {
o.Progress(done, len(s.Cases))
}
mu.Unlock()
}
})
}
for i := range s.Cases {
select {
case todo <- i:
case <-ctx.Done():
}
if ctx.Err() != nil {
break
}
}
close(todo)
wg.Wait()
run.TookMs = time.Since(run.At).Milliseconds()
var sum, weights float64
firstErr := ""
for i, r := range run.Results {
if r.ID == "" {
// Not asked: the run was stopped first.
r = store.EvalCase{ID: s.Cases[i].ID, Kind: s.Cases[i].Kind(), Weight: s.Cases[i].Weight,
Error: "not asked: the run was stopped"}
run.Results[i] = r
}
sum += r.Weight * r.Score
weights += r.Weight
if !r.Passed {
run.Below++
}
if r.Error != "" {
run.Errors++
if firstErr == "" {
firstErr = r.Error
}
}
}
if weights > 0 {
run.Score = sum / weights
}
run.Passed = run.Score >= threshold
if run.Errors == run.Cases {
run.Passed = false
run.Error = "no case was answered: " + firstErr
}
return run, nil
}
// askedOf is the conversation a case sends.
func askedOf(c *Case, suiteSystem string) []Message {
var out []Message
system := c.System
if system == "" {
system = suiteSystem
}
if system != "" && (len(c.Messages) == 0 || c.Messages[0].Role != "system") {
out = append(out, Message{Role: "system", Content: system})
}
if len(c.Messages) > 0 {
return append(out, c.Messages...)
}
return append(out, Message{Role: "user", Content: c.Prompt})
}
// askOne asks one case and scores its answer.
func askOne(ctx context.Context, call Caller, s Suite, c *Case, model, judge string) store.EvalCase {
res := store.EvalCase{ID: c.ID, Kind: c.Kind(), Weight: c.Weight}
start := time.Now()
defer func() { res.TookMs = time.Since(start).Milliseconds() }()
maxTokens := c.MaxTokens
if maxTokens <= 0 {
maxTokens = s.MaxTokens
}
if maxTokens <= 0 {
maxTokens = defaultMaxTokens
}
temp := 0.0
if c.Temperature != nil {
temp = *c.Temperature
}
answer, finish, err := chat(ctx, call, model, askedOf(c, s.System), maxTokens, temp)
if err != nil {
res.Error = err.Error()
return res
}
answer = cleanAnswer(answer)
res.Answer = answer
if len(res.Answer) > keptAnswer {
res.Answer = res.Answer[:keptAnswer]
}
var v verdict
if c.Judge != "" {
jm := c.JudgeModel
if jm == "" {
jm = judge
}
reply, _, err := chat(ctx, call, jm, judgeRequest(c, answer), 256, 0)
if err != nil {
res.Error = "the judge " + jm + ": " + err.Error()
return res
}
if v, err = readJudgement(reply); err != nil {
res.Error = err.Error()
return res
}
} else {
v = scoreRule(c, answer)
}
res.Score, res.Detail = v.score, v.detail
res.Passed = res.Score >= PassAt
if finish == "length" && !res.Passed {
res.Detail += fmt.Sprintf("; the answer was cut off at %d tokens", maxTokens)
}
if answer == "" && !res.Passed {
res.Detail += "; it answered nothing"
}
return res
}
// chat asks a model one conversation and says what it answered and why it
// stopped.
func chat(ctx context.Context, call Caller, model string, msgs []Message, maxTokens int, temp float64) (string, string, error) {
body, err := json.Marshal(map[string]any{"model": model, "messages": msgs, "max_tokens": maxTokens,
"temperature": temp, "stream": false})
if err != nil {
return "", "", err
}
status, out, _, err := call(ctx, "/v1/chat/completions", model, body)
if err != nil {
return "", "", err
}
if status != http.StatusOK {
var e struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
msg := strings.TrimSpace(string(out))
if json.Unmarshal(out, &e) == nil && e.Error.Message != "" {
msg = e.Error.Message
}
return "", "", fmt.Errorf("%s answered %d: %s", model, status, clip(msg, 200))
}
var resp struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
}
if err := json.Unmarshal(out, &resp); err != nil || len(resp.Choices) == 0 {
return "", "", fmt.Errorf("%s's answer is not a chat completion: %s", model, clip(string(out), 120))
}
return resp.Choices[0].Message.Content, resp.Choices[0].FinishReason, nil
}
+292
View File
@@ -0,0 +1,292 @@
package eval
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
const arith = `
name: arith
threshold: 0.75
judge: grader
cases:
- id: add
prompt: "What is 17 + 25? Answer with the number only."
number: 42
- id: mul
prompt: "What is 6 * 7?"
number: 42
weight: 2
- id: capital
prompt: "What is the capital of France?"
contains: Paris
- id: polite
prompt: "Say hello."
judge: "A greeting of any kind passes."
`
// fakeModel answers by the prompt, and as a judge by what it was asked to
// grade; it counts what it was asked.
type fakeModel struct {
mu sync.Mutex
answers map[string]string
asked []string
inFlight, most atomic.Int32
}
func (f *fakeModel) call(_ context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
n := f.inFlight.Add(1)
defer f.inFlight.Add(-1)
for {
m := f.most.Load()
if n <= m || f.most.CompareAndSwap(m, n) {
break
}
}
time.Sleep(5 * time.Millisecond)
var req struct {
Model string `json:"model"`
Messages []Message `json:"messages"`
}
if err := json.Unmarshal(body, &req); err != nil || path != "/v1/chat/completions" || req.Model != model {
return 400, []byte(`{"error":{"message":"bad request"}}`), 0, nil
}
f.mu.Lock()
f.asked = append(f.asked, model)
f.mu.Unlock()
last := req.Messages[len(req.Messages)-1].Content
answer := "I do not know"
switch {
case model == "grader":
answer = `{"score": 0, "reason": "no greeting"}`
if strings.Contains(last, "Answer:\nHello") {
answer = `{"score": 1, "reason": "a greeting"}`
}
case model == "missing":
return 404, []byte(`{"error":{"message":"the model \"missing\" does not exist"}}`), 0, nil
default:
for k, v := range f.answers {
if strings.Contains(last, k) {
answer = v
}
}
}
out, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{
"message": map[string]any{"content": answer}, "finish_reason": "stop"}}})
return 200, out, 0, nil
}
func TestASuiteIsAskedAndScoredByWeight(t *testing.T) {
s, err := Parse("arith.yaml", []byte(arith))
if err != nil {
t.Fatal(err)
}
f := &fakeModel{answers: map[string]string{"17 + 25": "<think>carry</think>42", "6 * 7": "41",
"France": "Paris.", "hello": "Hello there!"}}
var progress atomic.Int32
run, err := Evaluate(context.Background(), f.call, s, Options{Model: "qwen", Concurrency: 2,
Progress: func(done, total int) { progress.Store(int32(done)) }})
if err != nil {
t.Fatal(err)
}
// add 1, mul 0 at weight 2, capital 1, polite 1: 3 of 5.
if run.Score != 0.6 || run.Passed || run.Threshold != 0.75 || run.Below != 1 || run.Cases != 4 ||
run.Suite != "arith" || run.Judge != "grader" || len(run.SuiteHash) != 64 {
t.Fatalf("%+v", run)
}
byID := map[string]store.EvalCase{}
for _, r := range run.Results {
byID[r.ID] = r
}
if r := byID["add"]; !r.Passed || r.Answer != "42" || r.Kind != "number" {
t.Errorf("add: %+v", r)
}
if r := byID["mul"]; r.Passed || !strings.Contains(r.Detail, "read 41, wanted 42") || r.Weight != 2 {
t.Errorf("mul: %+v", r)
}
if r := byID["polite"]; !r.Passed || !strings.Contains(r.Detail, "the judge: a greeting") {
t.Errorf("polite: %+v", r)
}
if progress.Load() != 4 || f.most.Load() > 2 {
t.Errorf("progress %d, %d at once with a concurrency of 2", progress.Load(), f.most.Load())
}
// A lower bar passes the same answers.
run, _ = Evaluate(context.Background(), f.call, s, Options{Model: "qwen", MinScore: 0.5})
if !run.Passed || run.Threshold != 0.5 {
t.Errorf("at 0.5: %+v", run)
}
}
func TestAModelThatAnswersNothingFailsWithWhy(t *testing.T) {
s, _ := Parse("arith.yaml", []byte(arith))
run, err := Evaluate(context.Background(), (&fakeModel{}).call, s, Options{Model: "missing"})
if err != nil {
t.Fatal(err)
}
if run.Passed || run.Errors != 4 || !strings.Contains(run.Error, `"missing" does not exist`) {
t.Fatalf("%+v", run)
}
}
func TestARubricWithNoJudgeIsRefusedBeforeAnythingIsAsked(t *testing.T) {
s, _ := Parse("arith.yaml", []byte(strings.Replace(arith, "judge: grader\n", "", 1)))
f := &fakeModel{}
if _, err := Evaluate(context.Background(), f.call, s, Options{Model: "qwen"}); err == nil ||
!strings.Contains(err.Error(), "--judge") {
t.Fatalf("%v", err)
}
if len(f.asked) != 0 {
t.Errorf("asked %v first", f.asked)
}
}
func TestASuiteIsReadFromJSONLinesAsFromYAML(t *testing.T) {
jsonl := `{"name": "fmt", "threshold": 0.5}
{"id": "a", "prompt": "Give JSON", "json_schema": {"type": "object", "required": ["ok"]}}
{"prompt": "Say yes", "regex": "^yes", "ignore_case": true, "weight": 3}
`
s, err := Parse("format.jsonl", []byte(jsonl))
if err != nil {
t.Fatal(err)
}
if s.Name != "fmt" || s.Threshold != 0.5 || len(s.Cases) != 2 || s.Cases[1].ID != "case-2" ||
s.Cases[1].Weight != 3 || s.Cases[0].schema == nil || s.Cases[1].re == nil {
t.Fatalf("%+v", s)
}
// Named by its file when it names nothing, and 0.8 to pass.
s, err = Parse("dir/plain.jsonl", []byte(`{"prompt": "q", "exact": "a"}`))
if err != nil || s.Name != "plain" || s.Threshold != DefaultThreshold {
t.Fatalf("%+v %v", s, err)
}
}
func TestASuiteThatCannotBeScoredIsRefusedSayingWhy(t *testing.T) {
for _, tc := range []struct{ suite, why string }{
{"cases: []", "no cases"},
{"cases:\n - prompt: q", "no way to score it"},
{"cases:\n - prompt: q\n exact: a\n regex: b", "exact and regex"},
{"cases:\n - exact: a", "nothing to ask"},
{"cases:\n - prompt: q\n regex: '('", "the regex"},
{"cases:\n - id: x\n prompt: q\n exact: a\n - id: x\n prompt: q\n exact: a", "two cases are called x"},
{"threshold: 2\ncases:\n - prompt: q\n exact: a", "not 0 to 1"},
{"cases:\n - prompt: q\n exactly: a", "exactly"},
{"cases:\n - messages: [{role: robot, content: hi}]\n exact: a", "role"},
} {
if _, err := Parse("s.yaml", []byte(tc.suite)); err == nil || !strings.Contains(err.Error(), tc.why) {
t.Errorf("%q: %v, wanted it to say %q", tc.suite, err, tc.why)
}
}
}
type memStore struct {
mu sync.Mutex
runs []store.EvalRun
}
func (m *memStore) PutEvalRun(_ context.Context, r store.EvalRun) error {
m.mu.Lock()
defer m.mu.Unlock()
m.runs = append(m.runs, r)
return nil
}
func (m *memStore) EvalRuns(_ context.Context, model string, _ int) ([]store.EvalRun, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []store.EvalRun
for i := len(m.runs) - 1; i >= 0; i-- {
if r := m.runs[i]; model == "" || r.Model == model || r.Target == model {
r.Results = nil
out = append(out, r)
}
}
return out, nil
}
func (m *memStore) EvalRunByID(_ context.Context, id string) (store.EvalRun, error) {
m.mu.Lock()
defer m.mu.Unlock()
for _, r := range m.runs {
if r.ID == id {
return r, nil
}
}
return store.EvalRun{}, store.ErrNotFound
}
func TestTheControlPlaneRunsSuitesByNameAndFilesACanarysRunUnderItsModel(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "arith.yaml"), []byte(arith), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "broken.yaml"), []byte("cases: [}"), 0o600); err != nil {
t.Fatal(err)
}
builtin := fstest.MapFS{
"arith.yaml": {Data: []byte("cases:\n - prompt: shadowed\n exact: x\n")},
"json-format.yaml": {Data: []byte("description: JSON\ncases:\n - prompt: q\n json_schema: {type: object}\n")},
}
f := &fakeModel{answers: map[string]string{"17 + 25": "42", "6 * 7": "42", "France": "Paris", "hello": "Hello"}}
st := &memStore{}
svc := NewService(Deps{Call: f.call, Store: st, Dir: dir, Builtin: builtin, Log: slog.New(slog.DiscardHandler),
Origin: func(m string) (string, string) { return "o/r", "rev-of-" + m },
CanaryOf: func(n string) string {
if parent, ok := strings.CutSuffix(n, "-canary"); ok {
return parent
}
return ""
},
})
suites := svc.Suites()
if len(suites) != 3 || suites[0].Name != "arith" || suites[0].From != filepath.Join(dir, "arith.yaml") ||
suites[0].Cases != 4 || !suites[0].Judge || suites[1].Error == "" || suites[2].From != "built in" {
t.Fatalf("%+v", suites)
}
if _, err := svc.Load(Request{Suite: "../etc/passwd"}); err == nil {
t.Error("a path outside the eval directory was read")
}
live, err := svc.Start("qwen-canary", Request{Suite: "arith"}, "tester")
if err != nil {
t.Fatal(err)
}
if live.State != Running || live.Model != "qwen" || live.Target != "qwen-canary" || live.Cases != 4 {
t.Fatalf("%+v", live)
}
deadline := time.Now().Add(5 * time.Second)
for live.State == Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
live, _ = svc.Get(context.Background(), live.ID)
}
if live.State != Done || live.Score != 1 || !live.Passed || live.Revision != "rev-of-qwen-canary" || live.By != "tester" {
t.Fatalf("%+v", live)
}
if kept, err := st.EvalRunByID(context.Background(), live.ID); err != nil || len(kept.Results) != 4 {
t.Fatalf("kept %+v %v", kept, err)
}
// A gate waits for its run, and files it with the job.
run, err := svc.Run(context.Background(), "qwen", Request{Suite: "json-format"}, "tester", "update-1")
if err != nil || run.Job != "update-1" || run.Passed || run.Target != "" {
t.Fatalf("%+v %v", run, err)
}
hist, err := svc.History(context.Background(), "qwen", 10)
if err != nil || len(hist) != 2 || hist[0].ID != run.ID || hist[1].ID != live.ID {
t.Fatalf("%+v %v", hist, err)
}
if _, err := svc.Get(context.Background(), "eval-nope"); !errors.Is(err, store.ErrNotFound) {
t.Errorf("%v", err)
}
}
+338
View File
@@ -0,0 +1,338 @@
package eval
import (
"encoding/json"
"fmt"
"math"
"reflect"
"regexp"
"slices"
"sort"
"strings"
"unicode/utf8"
)
// A JSON Schema validator for what a suite asks of an answer's shape: types,
// properties, required ones, items, enums, bounds and patterns. Not the whole
// standard — no $ref, no oneOf — and a schema using what is not here is
// refused when the suite is read rather than passed by an answer it never
// checked.
type schema struct {
types []string
properties map[string]*schema
required []string
additional *schema // nil: anything; noneAllowed: nothing
items *schema
enum []any
constant any
hasConst bool
minimum, maximum *float64
exclMin, exclMax *float64
minLength, maxLength *int
minItems, maxItems *int
pattern *regexp.Regexp
anyOf, allOf []*schema
never bool
}
// noneAllowed is additionalProperties: false.
var noneAllowed = &schema{never: true}
// annotations are keywords that describe rather than constrain.
var annotations = map[string]bool{"$schema": true, "$id": true, "title": true, "description": true,
"default": true, "examples": true, "format": true, "$comment": true}
func compileSchema(v any) (*schema, error) {
// Through JSON, so a schema written in YAML and one in JSON are the same
// thing by the time they are read: numbers as float64, maps as map[string]any.
b, err := json.Marshal(v)
if err != nil {
return nil, err
}
var raw any
if err := json.Unmarshal(b, &raw); err != nil {
return nil, err
}
return compile(raw, "")
}
func compile(raw any, at string) (*schema, error) {
if b, ok := raw.(bool); ok {
if b {
return &schema{}, nil
}
return noneAllowed, nil
}
m, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s is not a schema object", where(at))
}
s := &schema{}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := m[k]
var err error
switch k {
case "type":
switch t := v.(type) {
case string:
s.types = []string{t}
case []any:
for _, x := range t {
if str, ok := x.(string); ok {
s.types = append(s.types, str)
}
}
}
for _, t := range s.types {
if !slices.Contains([]string{"object", "array", "string", "number", "integer", "boolean", "null"}, t) {
return nil, fmt.Errorf("%s: %q is not a JSON type", where(at), t)
}
}
case "properties":
props, ok := v.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: properties is not an object", where(at))
}
s.properties = map[string]*schema{}
for name, p := range props {
if s.properties[name], err = compile(p, at+"."+name); err != nil {
return nil, err
}
}
case "required":
list, _ := v.([]any)
for _, x := range list {
if str, ok := x.(string); ok {
s.required = append(s.required, str)
}
}
case "additionalProperties":
if s.additional, err = compile(v, at+".*"); err != nil {
return nil, err
}
case "items":
if s.items, err = compile(v, at+"[]"); err != nil {
return nil, err
}
case "enum":
s.enum, _ = v.([]any)
case "const":
s.constant, s.hasConst = v, true
case "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum":
f, ok := v.(float64)
if !ok {
return nil, fmt.Errorf("%s: %s is not a number", where(at), k)
}
*map[string]**float64{"minimum": &s.minimum, "maximum": &s.maximum,
"exclusiveMinimum": &s.exclMin, "exclusiveMaximum": &s.exclMax}[k] = &f
case "minLength", "maxLength", "minItems", "maxItems":
f, ok := v.(float64)
if !ok || f < 0 || f != math.Trunc(f) {
return nil, fmt.Errorf("%s: %s is not a whole number", where(at), k)
}
n := int(f)
*map[string]**int{"minLength": &s.minLength, "maxLength": &s.maxLength,
"minItems": &s.minItems, "maxItems": &s.maxItems}[k] = &n
case "pattern":
str, _ := v.(string)
if s.pattern, err = regexp.Compile(str); err != nil {
return nil, fmt.Errorf("%s: pattern: %w", where(at), err)
}
case "anyOf", "allOf":
list, ok := v.([]any)
if !ok || len(list) == 0 {
return nil, fmt.Errorf("%s: %s is not a list of schemas", where(at), k)
}
for i, x := range list {
sub, err := compile(x, fmt.Sprintf("%s(%s %d)", at, k, i+1))
if err != nil {
return nil, err
}
if k == "anyOf" {
s.anyOf = append(s.anyOf, sub)
} else {
s.allOf = append(s.allOf, sub)
}
}
default:
if !annotations[k] {
return nil, fmt.Errorf("%s: %s is not supported here (type, properties, required, "+
"additionalProperties, items, enum, const, minimum, maximum, exclusiveMinimum, exclusiveMaximum, "+
"minLength, maxLength, minItems, maxItems, pattern, anyOf and allOf are)", where(at), k)
}
}
}
return s, nil
}
func where(at string) string {
if at == "" {
return "the schema"
}
return "the schema at " + strings.TrimPrefix(at, ".")
}
// validate says what is wrong with v, or "".
func (s *schema) validate(v any, at string) string {
if s.never {
return fmt.Sprintf("%s is not allowed", place(at))
}
if len(s.types) > 0 && !slices.ContainsFunc(s.types, func(t string) bool { return isType(v, t) }) {
return fmt.Sprintf("%s is %s, not %s", place(at), typeOf(v), strings.Join(s.types, " or "))
}
if s.hasConst && !equal(v, s.constant) {
return fmt.Sprintf("%s is %s, not %s", place(at), short(v), short(s.constant))
}
if s.enum != nil && !slices.ContainsFunc(s.enum, func(e any) bool { return equal(v, e) }) {
return fmt.Sprintf("%s is %s, not one of %s", place(at), short(v), short(s.enum))
}
switch x := v.(type) {
case float64:
switch {
case s.minimum != nil && x < *s.minimum:
return fmt.Sprintf("%s is %v, below %v", place(at), x, *s.minimum)
case s.maximum != nil && x > *s.maximum:
return fmt.Sprintf("%s is %v, above %v", place(at), x, *s.maximum)
case s.exclMin != nil && x <= *s.exclMin:
return fmt.Sprintf("%s is %v, not above %v", place(at), x, *s.exclMin)
case s.exclMax != nil && x >= *s.exclMax:
return fmt.Sprintf("%s is %v, not below %v", place(at), x, *s.exclMax)
}
case string:
n := utf8.RuneCountInString(x)
switch {
case s.minLength != nil && n < *s.minLength:
return fmt.Sprintf("%s is %d characters, fewer than %d", place(at), n, *s.minLength)
case s.maxLength != nil && n > *s.maxLength:
return fmt.Sprintf("%s is %d characters, more than %d", place(at), n, *s.maxLength)
case s.pattern != nil && !s.pattern.MatchString(x):
return fmt.Sprintf("%s, %s, does not match %s", place(at), short(x), s.pattern)
}
case []any:
switch {
case s.minItems != nil && len(x) < *s.minItems:
return fmt.Sprintf("%s has %d items, fewer than %d", place(at), len(x), *s.minItems)
case s.maxItems != nil && len(x) > *s.maxItems:
return fmt.Sprintf("%s has %d items, more than %d", place(at), len(x), *s.maxItems)
}
if s.items != nil {
for i, e := range x {
if why := s.items.validate(e, fmt.Sprintf("%s[%d]", at, i)); why != "" {
return why
}
}
}
case map[string]any:
for _, r := range s.required {
if _, ok := x[r]; !ok {
return fmt.Sprintf("%s has no %s", place(at), r)
}
}
keys := make([]string, 0, len(x))
for k := range x {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if p, ok := s.properties[k]; ok {
if why := p.validate(x[k], at+"."+k); why != "" {
return why
}
} else if s.additional != nil {
if why := s.additional.validate(x[k], at+"."+k); why != "" {
return why
}
}
}
}
for _, sub := range s.allOf {
if why := sub.validate(v, at); why != "" {
return why
}
}
if len(s.anyOf) > 0 {
var first string
for _, sub := range s.anyOf {
why := sub.validate(v, at)
if why == "" {
return ""
}
if first == "" {
first = why
}
}
return first + " (nor any other choice)"
}
return ""
}
func place(at string) string {
if at == "" {
return "the answer"
}
return strings.TrimPrefix(at, ".")
}
func isType(v any, t string) bool {
switch t {
case "object":
_, ok := v.(map[string]any)
return ok
case "array":
_, ok := v.([]any)
return ok
case "string":
_, ok := v.(string)
return ok
case "number":
_, ok := v.(float64)
return ok
case "integer":
f, ok := v.(float64)
return ok && f == math.Trunc(f)
case "boolean":
_, ok := v.(bool)
return ok
case "null":
return v == nil
}
return false
}
func typeOf(v any) string {
switch x := v.(type) {
case map[string]any:
return "an object"
case []any:
return "a list"
case string:
return "a string"
case float64:
if x == math.Trunc(x) {
return "an integer"
}
return "a number"
case bool:
return "a boolean"
case nil:
return "null"
}
return fmt.Sprintf("%T", v)
}
func equal(a, b any) bool { return reflect.DeepEqual(a, b) }
func short(v any) string {
b, _ := json.Marshal(v)
s := string(b)
if len(s) > 60 {
s = s[:57] + "..."
}
return s
}
+214
View File
@@ -0,0 +1,214 @@
package eval
import (
"encoding/json"
"fmt"
"math"
"regexp"
"strconv"
"strings"
)
// verdict is how one answer scored, 0 to 1, and why.
type verdict struct {
score float64
detail string
}
func pass(detail string) verdict { return verdict{1, detail} }
func fail(detail string) verdict { return verdict{0, detail} }
var thinkBlock = regexp.MustCompile(`(?s)<think>.*?</think>`)
// cleanAnswer is what a model said, without a reasoning model's thinking —
// what a reader is shown — and without the white space around it.
func cleanAnswer(s string) string {
s = thinkBlock.ReplaceAllString(s, "")
// A reply cut off while still thinking has an opening tag and no close.
if i := strings.Index(s, "<think>"); i >= 0 {
s = s[:i]
}
return strings.TrimSpace(s)
}
// scoreRule scores an answer by a case's rule; a case graded by a judge is
// scored elsewhere, since it takes another request.
func scoreRule(c *Case, answer string) verdict {
fold := func(s string) string {
if c.IgnoreCase {
return strings.ToLower(s)
}
return s
}
switch {
case c.Exact != nil:
if fold(answer) == fold(strings.TrimSpace(*c.Exact)) {
return pass("the answer is exactly that")
}
return fail(fmt.Sprintf("wanted exactly %q", strings.TrimSpace(*c.Exact)))
case len(c.Contains) > 0:
var missing []string
for _, want := range c.Contains {
if !strings.Contains(fold(answer), fold(want)) {
missing = append(missing, strconv.Quote(want))
}
}
if len(missing) == 0 {
return pass("has everything asked for")
}
return fail("has no " + strings.Join(missing, ", "))
case c.re != nil:
if c.re.MatchString(answer) {
return pass("matches")
}
return fail("does not match " + c.re.String())
case c.Number != nil:
got, ok := lastNumber(answer)
if !ok {
return fail(fmt.Sprintf("no number in the answer; wanted %v", *c.Number))
}
if math.Abs(got-*c.Number) <= c.Tolerance {
return pass(fmt.Sprintf("read %v", got))
}
want := fmt.Sprintf("%v", *c.Number)
if c.Tolerance > 0 {
want += fmt.Sprintf(" ± %v", c.Tolerance)
}
return fail(fmt.Sprintf("read %v, wanted %s", got, want))
case c.schema != nil:
v, err := readJSON(answer)
if err != nil {
return fail(err.Error())
}
if why := c.schema.validate(v, ""); why != "" {
return fail(why)
}
return pass("JSON of the shape asked for")
}
return fail("no rule to score it by")
}
// number is a number as a model writes one: a sign, thousands separated by
// commas, a decimal part, an exponent.
var number = regexp.MustCompile(`[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?:[eE][-+]?\d+)?`)
// lastNumber is the last number in a text: a model that shows its working
// ends with the result, and one that does not says only the result.
func lastNumber(s string) (float64, bool) {
all := number.FindAllStringIndex(s, -1)
if len(all) == 0 {
return 0, false
}
i, j := all[len(all)-1][0], all[len(all)-1][1]
// A hyphen after a digit or letter joins (3-5, 2026-10-04); it is not a
// minus sign.
if (s[i] == '-' || s[i] == '+') && i > 0 && isWordByte(s[i-1]) {
i++
}
f, err := strconv.ParseFloat(strings.ReplaceAll(s[i:j], ",", ""), 64)
return f, err == nil
}
func isWordByte(b byte) bool {
return b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
}
var fence = regexp.MustCompile("(?s)```(?:json|JSON)?\\s*(.*?)```")
// readJSON reads an answer as JSON: the whole of it, else what is inside a
// code fence, else from the first brace or bracket to the last — models
// asked for JSON wrap it in a sentence often enough that refusing that would
// measure manners rather than the shape.
func readJSON(answer string) (any, error) {
try := []string{answer}
if m := fence.FindStringSubmatch(answer); m != nil {
try = append(try, m[1])
}
for _, pair := range [][2]string{{"{", "}"}, {"[", "]"}} {
i, j := strings.Index(answer, pair[0]), strings.LastIndex(answer, pair[1])
if i >= 0 && j > i {
try = append(try, answer[i:j+1])
}
}
for _, t := range try {
var v any
if json.Unmarshal([]byte(strings.TrimSpace(t)), &v) == nil {
return v, nil
}
}
if answer == "" {
return nil, fmt.Errorf("the answer is empty, not JSON")
}
return nil, fmt.Errorf("the answer is not JSON: %s", clip(answer, 80))
}
func clip(s string, n int) string {
s = strings.Join(strings.Fields(s), " ")
if len(s) <= n {
return s
}
return s[:n-3] + "..."
}
// judgePrompt asks a model to grade an answer by a rubric.
const judgePrompt = "You grade an answer to a question against a rubric. Read the question, the answer " +
"and the rubric, then reply with only a JSON object: {\"score\": a number from 0 to 1, " +
"\"reason\": \"one short sentence\"}. 1 means the answer meets the rubric fully, 0 not at all."
// judgeRequest is what a judge is asked about one answer.
func judgeRequest(c *Case, answer string) []Message {
var q strings.Builder
for _, m := range askedOf(c, "") {
if m.Role == "system" {
continue
}
fmt.Fprintf(&q, "%s: %s\n", m.Role, m.Content)
}
return []Message{
{Role: "system", Content: judgePrompt},
{Role: "user", Content: fmt.Sprintf("Question:\n%s\nAnswer:\n%s\n\nRubric:\n%s", q.String(), answer, c.Judge)},
}
}
var (
passWord = regexp.MustCompile(`(?i)\b(pass|passed|yes|correct)\b`)
failWord = regexp.MustCompile(`(?i)\b(fail|failed|no|incorrect|wrong)\b`)
)
// readJudgement reads a judge's reply: a JSON object with a score (or a
// pass true or false), else a lone number, else the words pass or fail.
func readJudgement(reply string) (verdict, error) {
reply = cleanAnswer(reply)
if v, err := readJSON(reply); err == nil {
if m, ok := v.(map[string]any); ok {
reason, _ := m["reason"].(string)
if f, ok := m["score"].(float64); ok {
return verdict{clamp(f), "the judge: " + clip(reason, 120)}, nil
}
if s, ok := m["score"].(string); ok {
if f, err := strconv.ParseFloat(strings.TrimSpace(s), 64); err == nil {
return verdict{clamp(f), "the judge: " + clip(reason, 120)}, nil
}
}
if b, ok := m["pass"].(bool); ok {
if b {
return pass("the judge: " + clip(reason, 120)), nil
}
return fail("the judge: " + clip(reason, 120)), nil
}
}
}
if f, err := strconv.ParseFloat(strings.TrimSpace(reply), 64); err == nil {
return verdict{clamp(f), "the judge gave " + strings.TrimSpace(reply)}, nil
}
p, f := passWord.MatchString(reply), failWord.MatchString(reply)
switch {
case p && !f:
return pass("the judge: " + clip(reply, 120)), nil
case f && !p:
return fail("the judge: " + clip(reply, 120)), nil
}
return verdict{}, fmt.Errorf("the judge's answer could not be read as a score: %s", clip(reply, 120))
}
func clamp(f float64) float64 { return math.Max(0, math.Min(1, f)) }
+133
View File
@@ -0,0 +1,133 @@
package eval
import (
"strings"
"testing"
)
func caseOf(t *testing.T, yamlCase string) *Case {
t.Helper()
s, err := Parse("t.yaml", []byte("cases:\n - prompt: q\n"+indent(yamlCase)))
if err != nil {
t.Fatal(err)
}
return &s.Cases[0]
}
func indent(s string) string {
var b strings.Builder
for _, l := range strings.Split(strings.TrimSpace(s), "\n") {
b.WriteString(" " + l + "\n")
}
return b.String()
}
func TestEachRuleScoresTheAnswersItShouldAndRefusesTheRest(t *testing.T) {
for _, tc := range []struct {
rule string
answer string
want float64
}{
{`exact: "42"`, "42", 1},
{`exact: "42"`, " 42\n", 1},
{`exact: "42"`, "42.", 0},
{"exact: Paris\nignore_case: true", "PARIS", 1},
{`contains: Paris`, "The capital is Paris.", 1},
{`contains: [Paris, France]`, "Paris is in France", 1},
{`contains: [Paris, France]`, "Paris", 0},
{"contains: paris\nignore_case: true", "PARIS", 1},
{`regex: '^\s*\d{4}-\d{2}-\d{2}\s*$'`, "2026-10-04", 1},
{`regex: '^\d+$'`, "about 12", 0},
{"regex: '^yes'\nignore_case: true", "Yes, it is", 1},
{`number: 42`, "17 + 25 = 42", 1},
{`number: 42`, "The answer is 42.", 1},
{`number: 42`, "It is 41", 0},
{`number: 1234567`, "1,234,567", 1},
{"number: 3.14\ntolerance: 0.01", "pi is about 3.1416", 1},
{"number: 3.14\ntolerance: 0.001", "pi is about 3.1416", 0},
{`number: -5`, "3 - 8 = -5", 1},
// A date's last part is a number, not a negative one.
{`number: 4`, "from 2026-10-04", 1},
{`number: 1`, "no number here", 0},
} {
c := caseOf(t, tc.rule)
v := scoreRule(c, cleanAnswer(tc.answer))
if v.score != tc.want {
t.Errorf("%s against %q scored %v (%s), wanted %v", strings.ReplaceAll(tc.rule, "\n", "; "), tc.answer, v.score, v.detail, tc.want)
}
}
}
func TestAnAnswerIsReadAsJSONAndHeldToItsSchema(t *testing.T) {
c := caseOf(t, `
json_schema:
type: object
required: [name, age, tags]
additionalProperties: false
properties:
name: {type: string, minLength: 1}
age: {type: integer, minimum: 0, maximum: 150}
tags: {type: array, items: {type: string}, maxItems: 3}
kind: {enum: [cat, dog]}`)
for _, tc := range []struct {
answer string
pass bool
why string
}{
{`{"name": "Ada", "age": 36, "tags": ["math"]}`, true, ""},
{"Here it is:\n```json\n{\"name\": \"Ada\", \"age\": 36, \"tags\": []}\n```", true, ""},
{`Sure! {"name": "Ada", "age": 36, "tags": []} Hope that helps.`, true, ""},
{`{"name": "Ada", "age": 36.5, "tags": []}`, false, "age is a number, not integer"},
{`{"name": "Ada", "tags": []}`, false, "has no age"},
{`{"name": "Ada", "age": 36, "tags": [], "extra": 1}`, false, "extra is not allowed"},
{`{"name": "Ada", "age": 200, "tags": []}`, false, "above 150"},
{`{"name": "Ada", "age": 3, "tags": [1]}`, false, "tags[0] is an integer, not string"},
{`{"name": "Ada", "age": 3, "tags": [], "kind": "fish"}`, false, "not one of"},
{`{"name": "", "age": 3, "tags": []}`, false, "fewer than 1"},
{`not json at all`, false, "not JSON"},
} {
v := scoreRule(c, tc.answer)
if (v.score == 1) != tc.pass || (tc.why != "" && !strings.Contains(v.detail, tc.why)) {
t.Errorf("%q: %v %q, wanted pass=%v saying %q", tc.answer, v.score, v.detail, tc.pass, tc.why)
}
}
}
func TestASchemaUsingWhatIsNotCheckedIsRefusedWhenTheSuiteIsRead(t *testing.T) {
_, err := Parse("t.yaml", []byte("cases:\n - prompt: q\n json_schema: {oneOf: [{type: string}]}\n"))
if err == nil || !strings.Contains(err.Error(), "oneOf is not supported") {
t.Fatalf("a schema with oneOf was accepted: %v", err)
}
}
func TestAJudgesReplyIsReadAsAScore(t *testing.T) {
for _, tc := range []struct {
reply string
want float64
bad bool
}{
{`{"score": 0.8, "reason": "mostly right"}`, 0.8, false},
{"<think>hmm</think>\n```json\n{\"score\": 1, \"reason\": \"ok\"}\n```", 1, false},
{`{"score": "0.25"}`, 0.25, false},
{`{"pass": false, "reason": "wrong city"}`, 0, false},
{`{"score": 7}`, 1, false},
{`0.6`, 0.6, false},
{`PASS`, 1, false},
{`Fail: it named the wrong city.`, 0, false},
{`I cannot say.`, 0, true},
} {
v, err := readJudgement(tc.reply)
if (err != nil) != tc.bad || v.score != tc.want {
t.Errorf("%q: %v %v, wanted %v (unreadable: %v)", tc.reply, v.score, err, tc.want, tc.bad)
}
}
}
func TestAReasoningModelsThinkingIsNotPartOfItsAnswer(t *testing.T) {
if got := cleanAnswer("<think>\n17+25... carry the one\n</think>\n\n42"); got != "42" {
t.Errorf("%q", got)
}
if got := cleanAnswer("<think>still thinking when it was cut off"); got != "" {
t.Errorf("%q", got)
}
}
+434
View File
@@ -0,0 +1,434 @@
package eval
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The control plane's side: suites found on its disk or carried in the
// binary, runs started and followed, and kept in the store.
// Store keeps finished runs.
type Store interface {
PutEvalRun(ctx context.Context, r store.EvalRun) error
EvalRuns(ctx context.Context, model string, limit int) ([]store.EvalRun, error)
EvalRunByID(ctx context.Context, id string) (store.EvalRun, error)
}
// Deps are what the service runs suites with.
type Deps struct {
// Call asks a model: the router's Replay.
Call Caller
// Store keeps runs; without one they are kept here until a restart.
Store Store
// Dir is where suites are read from by name: control.eval_dir.
Dir string
// Builtin are the suites the binary carries, found when Dir has none
// of the name.
Builtin fs.FS
// Origin is where a catalog model's weights came from, when known.
Origin func(model string) (repo, revision string)
// CanaryOf is the model a name is the canary of, or "": a run on a
// canary is filed under the model it is a canary of.
CanaryOf func(name string) string
Log *slog.Logger
}
// Request is a suite to run against a model: one on the control plane, by
// its name, or one sent whole.
type Request struct {
// Suite is a suite's name in the eval directory, or carried in the
// binary (arithmetic, json-format).
Suite string `json:"suite,omitempty"`
// Content is a suite's text instead, and File the name of the file it
// was read from, which names the suite unless it names itself.
Content string `json:"content,omitempty"`
File string `json:"file,omitempty"`
// Concurrency, Judge and MinScore are as Options has them.
Concurrency int `json:"concurrency,omitempty"`
Judge string `json:"judge,omitempty"`
MinScore float64 `json:"min_score,omitempty"`
}
// Named says which suite a request is, for a step's detail.
func (r Request) Named() string {
if r.Suite != "" {
return r.Suite
}
return r.File
}
// The states a run can be in.
const (
Running = "running"
Done = "done"
Failed = "failed"
)
// Live is a run as it stands: one running, with how many of its cases are
// done, or one finished.
type Live struct {
store.EvalRun
State string `json:"state"`
Done int `json:"done"`
}
// SuiteInfo is a suite the control plane can run by name.
type SuiteInfo struct {
Name string `json:"name"`
// From is the file it is read from, or "built in".
From string `json:"from"`
Description string `json:"description,omitempty"`
Cases int `json:"cases"`
Threshold float64 `json:"threshold"`
Judge bool `json:"judge,omitempty"`
Hash string `json:"hash,omitempty"`
// Error is why the file cannot be run.
Error string `json:"error,omitempty"`
}
// Service runs suites and remembers them.
type Service struct {
deps Deps
mu sync.Mutex
live map[string]*Live
// order is the live runs oldest first, so the finished ones past
// keepLive are forgotten — the store has them.
order []string
}
const keepLive = 50
// NewService builds a service.
func NewService(d Deps) *Service {
if d.Log == nil {
d.Log = slog.Default()
}
return &Service{deps: d, live: map[string]*Live{}}
}
var suiteExts = []string{".yaml", ".yml", ".jsonl", ".json"}
// Load reads the suite a request names or carries.
func (s *Service) Load(r Request) (Suite, error) {
if r.Content != "" {
file := r.File
if file == "" {
file = "suite"
}
return Parse(file, []byte(r.Content))
}
name := strings.TrimSpace(r.Suite)
if name == "" {
return Suite{}, errors.New("no suite: name one on the control plane, or send one's text")
}
// A name, not a path: this reads the control plane's disk on behalf of
// whoever holds the admin API, and only inside the eval directory.
if strings.ContainsAny(name, `/\`) || name == "." || name == ".." {
return Suite{}, fmt.Errorf("%q is not a suite's name: the control plane reads suites from %s by name; "+
"send a file from elsewhere with its text", name, s.dirOrNone())
}
tries := []string{name}
if !slices.Contains(suiteExts, filepath.Ext(name)) {
tries = nil
for _, e := range suiteExts {
tries = append(tries, name+e)
}
}
if s.deps.Dir != "" {
for _, t := range tries {
b, err := os.ReadFile(filepath.Join(s.deps.Dir, t))
if err == nil {
return Parse(t, b)
}
if !errors.Is(err, fs.ErrNotExist) {
return Suite{}, err
}
}
}
if s.deps.Builtin != nil {
for _, t := range tries {
if b, err := fs.ReadFile(s.deps.Builtin, t); err == nil {
return Parse(t, b)
}
}
}
return Suite{}, fmt.Errorf("no suite called %s in %s or built in: `llmgw ctl eval --suites` lists them", name, s.dirOrNone())
}
func (s *Service) dirOrNone() string {
if s.deps.Dir == "" {
return "no eval directory"
}
return s.deps.Dir
}
// Suites are those the control plane can run by name: its directory's, and
// the binary's that the directory does not replace.
func (s *Service) Suites() []SuiteInfo {
var out []SuiteInfo
seen := map[string]bool{}
add := func(fsys fs.FS, from func(string) string) {
entries, err := fs.ReadDir(fsys, ".")
if err != nil {
return
}
for _, e := range entries {
ext := filepath.Ext(e.Name())
if e.IsDir() || !slices.Contains(suiteExts, ext) {
continue
}
name := strings.TrimSuffix(e.Name(), ext)
if seen[name] {
continue
}
seen[name] = true
info := SuiteInfo{Name: name, From: from(e.Name())}
b, err := fs.ReadFile(fsys, e.Name())
var su Suite
if err == nil {
su, err = Parse(e.Name(), b)
}
if err != nil {
info.Error = err.Error()
} else {
info.Description, info.Cases, info.Threshold = su.Description, len(su.Cases), su.Threshold
info.Judge, info.Hash = su.NeedsJudge(), su.Hash
}
out = append(out, info)
}
}
if s.deps.Dir != "" {
add(os.DirFS(s.deps.Dir), func(n string) string { return filepath.Join(s.deps.Dir, n) })
}
if s.deps.Builtin != nil {
add(s.deps.Builtin, func(string) string { return "built in" })
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
func newID() string {
b := make([]byte, 4)
_, _ = rand.Read(b)
return fmt.Sprintf("eval-%s-%s", time.Now().UTC().Format("20060102-150405"), hex.EncodeToString(b))
}
// prepare reads the suite and checks it can be run against model, before
// anything is asked.
func (s *Service) prepare(model string, r Request) (Suite, error) {
if s.deps.Call == nil {
return Suite{}, errors.New("this control plane does not route requests, so it cannot ask a model")
}
if strings.TrimSpace(model) == "" {
return Suite{}, errors.New("no model to ask")
}
su, err := s.Load(r)
if err != nil {
return Suite{}, err
}
if r.MinScore < 0 || r.MinScore > 1 {
return Suite{}, fmt.Errorf("the minimum score %v is not 0 to 1", r.MinScore)
}
if su.NeedsJudge() && r.Judge == "" && su.Judge == "" {
for _, c := range su.Cases {
if c.Judge != "" && c.JudgeModel == "" {
return Suite{}, fmt.Errorf("case %s of %s is graded by a judge and the suite names none: give one "+
"with --judge MODEL", c.ID, su.Name)
}
}
}
return su, nil
}
// Check says whether a suite can be run against a model, without asking it
// anything: for a gate, refused before its job starts rather than after the
// new weights are fetched.
func (s *Service) Check(model string, r Request) error {
_, err := s.prepare(model, r)
return err
}
// file fills in what a run is filed under: the model it is about, the one
// that answered, and where its weights came from.
func (s *Service) file(run *store.EvalRun, asked string) {
run.Model = asked
if s.deps.CanaryOf != nil {
if parent := s.deps.CanaryOf(asked); parent != "" {
run.Model, run.Target = parent, asked
}
}
if s.deps.Origin != nil {
run.Repo, run.Revision = s.deps.Origin(asked)
}
}
// Start runs a suite against a model in the background and returns at once;
// Get follows it.
func (s *Service) Start(model string, r Request, by string) (Live, error) {
su, err := s.prepare(model, r)
if err != nil {
return Live{}, err
}
l := &Live{State: Running}
l.ID, l.Suite, l.SuiteHash, l.Cases, l.By, l.At = newID(), su.Name, su.Hash, len(su.Cases), by, time.Now().UTC()
s.file(&l.EvalRun, model)
s.mu.Lock()
s.keep(l)
s.mu.Unlock()
go func() {
_, _ = s.run(context.Background(), l, model, su, r)
}()
return s.snapshot(l), nil
}
// Run runs a suite against a model and waits for it: for a gate, which
// cannot go on until it knows. The run is kept like any other; job is the
// update or promotion it gates.
func (s *Service) Run(ctx context.Context, model string, r Request, by, job string) (store.EvalRun, error) {
su, err := s.prepare(model, r)
if err != nil {
return store.EvalRun{}, err
}
l := &Live{State: Running}
l.ID, l.Suite, l.SuiteHash, l.Cases, l.By, l.Job, l.At = newID(), su.Name, su.Hash, len(su.Cases), by, job, time.Now().UTC()
s.file(&l.EvalRun, model)
s.mu.Lock()
s.keep(l)
s.mu.Unlock()
return s.run(ctx, l, model, su, r)
}
func (s *Service) run(ctx context.Context, l *Live, model string, su Suite, r Request) (store.EvalRun, error) {
run, err := Evaluate(ctx, s.deps.Call, su, Options{Model: model, Concurrency: r.Concurrency, Judge: r.Judge,
MinScore: r.MinScore, Progress: func(done, _ int) {
s.mu.Lock()
l.Done = done
s.mu.Unlock()
}})
s.mu.Lock()
if err != nil {
l.State, l.Error = Failed, err.Error()
out := l.EvalRun
s.mu.Unlock()
return out, err
}
run.ID, run.By, run.Job = l.ID, l.By, l.Job
run.Model, run.Target, run.Repo, run.Revision = l.Model, l.Target, l.Repo, l.Revision
s.mu.Unlock()
// Kept before it is said to be done, so a history read in between does
// not find it in neither place.
if s.deps.Store != nil {
// Its own context: a gate's caller giving up does not unkeep a run
// that finished.
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := s.deps.Store.PutEvalRun(sctx, run); err != nil {
s.deps.Log.Warn("could not keep an eval run", "run", run.ID, "model", run.Model, "err", err)
}
}
s.mu.Lock()
l.EvalRun, l.State, l.Done = run, Done, run.Cases
s.mu.Unlock()
s.deps.Log.Info("eval run finished", "run", run.ID, "model", run.Model, "target", run.Target,
"suite", run.Suite, "score", fmt.Sprintf("%.3f", run.Score), "threshold", run.Threshold, "passed", run.Passed)
return run, nil
}
// keep remembers a run, and forgets the oldest finished ones past keepLive.
// Called with mu held.
func (s *Service) keep(l *Live) {
s.live[l.ID] = l
s.order = append(s.order, l.ID)
for len(s.order) > keepLive {
oldest := s.live[s.order[0]]
if oldest != nil && oldest.State == Running {
break
}
delete(s.live, s.order[0])
s.order = s.order[1:]
}
}
func (s *Service) snapshot(l *Live) Live {
s.mu.Lock()
defer s.mu.Unlock()
out := *l
out.Results = slices.Clone(l.Results)
return out
}
// Get is a run by its id: one running or finished here, else one kept.
func (s *Service) Get(ctx context.Context, id string) (Live, error) {
s.mu.Lock()
l, ok := s.live[id]
s.mu.Unlock()
if ok {
return s.snapshot(l), nil
}
if s.deps.Store == nil {
return Live{}, store.ErrNotFound
}
r, err := s.deps.Store.EvalRunByID(ctx, id)
if err != nil {
return Live{}, err
}
return Live{EvalRun: r, State: Done, Done: r.Cases}, nil
}
// History is a model's runs, newest first, without their cases: those
// running here, then those kept.
func (s *Service) History(ctx context.Context, model string, limit int) ([]Live, error) {
if limit <= 0 {
limit = 20
}
var out []Live
seen := map[string]bool{}
s.mu.Lock()
for i := len(s.order) - 1; i >= 0; i-- {
l := s.live[s.order[i]]
if l == nil || (model != "" && l.Model != model && l.Target != model) {
continue
}
// Finished ones are listed from the store, when there is one, in
// its order.
if l.State != Running && s.deps.Store != nil {
continue
}
v := *l
v.Results = nil
out = append(out, v)
seen[l.ID] = true
}
s.mu.Unlock()
if s.deps.Store != nil {
runs, err := s.deps.Store.EvalRuns(ctx, model, limit)
if err != nil {
return out, err
}
for _, r := range runs {
if !seen[r.ID] {
out = append(out, Live{EvalRun: r, State: Done, Done: r.Cases})
}
}
}
if len(out) > limit {
out = out[:limit]
}
return out, nil
}
+332
View File
@@ -0,0 +1,332 @@
// Package eval asks a model a suite of questions and scores its answers, so
// new weights can be held to the quality of the old before they replace them.
//
// internal/bench measures how fast a replica answers; this measures whether
// the answers are any good. A repository that republishes its weights — a
// new quantisation, a fixed chat template — can make a model faster and
// worse at once, and nothing in the latency numbers says so. A suite is a
// handful of cases with known answers, scored by rules where a rule can
// decide (an exact string, a number, JSON of a shape) and by another model
// with a rubric where only judgement can.
package eval
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"regexp"
"strings"
"go.yaml.in/yaml/v3"
)
// Suite is a set of cases a model is asked, and the score it must reach.
type Suite struct {
// Name is what runs are filed under; the file's name without its
// extension unless the suite gives one.
Name string `json:"name,omitempty" yaml:"name"`
Description string `json:"description,omitempty" yaml:"description"`
// Threshold is the score, 0 to 1, the suite is passed at. Zero is 0.8.
Threshold float64 `json:"threshold,omitempty" yaml:"threshold"`
// Judge is the model asked to grade the cases scored by a rubric, unless
// a case names its own.
Judge string `json:"judge,omitempty" yaml:"judge"`
// System and MaxTokens are every case's unless it gives its own.
System string `json:"system,omitempty" yaml:"system"`
MaxTokens int `json:"max_tokens,omitempty" yaml:"max_tokens"`
Cases []Case `json:"cases" yaml:"cases"`
// Hash is the sha256 of the text the suite was read from: two runs
// under one name compare only when they asked the same questions.
Hash string `json:"-" yaml:"-"`
}
// Message is one turn of a conversation a case sends.
type Message struct {
Role string `json:"role" yaml:"role"`
Content string `json:"content" yaml:"content"`
}
// Case is one question and how its answer is scored: by exactly one of
// Exact, Contains, Regex, Number, JSONSchema or Judge.
type Case struct {
// ID names the case in results; case-N, by its place, unless given.
ID string `json:"id,omitempty" yaml:"id"`
// Prompt is the user's message; Messages a whole conversation instead.
Prompt string `json:"prompt,omitempty" yaml:"prompt"`
Messages []Message `json:"messages,omitempty" yaml:"messages"`
System string `json:"system,omitempty" yaml:"system"`
// MaxTokens bounds the answer; the suite's, else 512.
MaxTokens int `json:"max_tokens,omitempty" yaml:"max_tokens"`
// Temperature is 0 unless given: a suite is a measurement, and sampling
// would make two runs of the same weights disagree.
Temperature *float64 `json:"temperature,omitempty" yaml:"temperature"`
// Weight is how much the case counts against the others; 1 unless given.
Weight float64 `json:"weight,omitempty" yaml:"weight"`
// Exact is the whole answer, surrounding white space aside.
Exact *string `json:"exact,omitempty" yaml:"exact"`
// Contains are strings that must all be in the answer.
Contains StringList `json:"contains,omitempty" yaml:"contains"`
// Regex must match somewhere in the answer.
Regex string `json:"regex,omitempty" yaml:"regex"`
// IgnoreCase makes Exact, Contains and Regex blind to case.
IgnoreCase bool `json:"ignore_case,omitempty" yaml:"ignore_case"`
// Number is the value the last number in the answer must be, within
// Tolerance either side.
Number *float64 `json:"number,omitempty" yaml:"number"`
Tolerance float64 `json:"tolerance,omitempty" yaml:"tolerance"`
// JSONSchema is a schema the answer, read as JSON, must satisfy.
JSONSchema any `json:"json_schema,omitempty" yaml:"json_schema"`
// Judge is a rubric another model grades the answer by, 0 to 1;
// JudgeModel names that model, else the suite's judge.
Judge string `json:"judge,omitempty" yaml:"judge"`
JudgeModel string `json:"judge_model,omitempty" yaml:"judge_model"`
re *regexp.Regexp
schema *schema
}
// StringList is one string or a list of them.
type StringList []string
// UnmarshalYAML takes a string or a list.
func (l *StringList) UnmarshalYAML(n *yaml.Node) error {
if n.Kind == yaml.ScalarNode {
*l = StringList{n.Value}
return nil
}
var v []string
if err := n.Decode(&v); err != nil {
return err
}
*l = v
return nil
}
// UnmarshalJSON takes a string or a list.
func (l *StringList) UnmarshalJSON(b []byte) error {
var s string
if json.Unmarshal(b, &s) == nil {
*l = StringList{s}
return nil
}
var v []string
if err := json.Unmarshal(b, &v); err != nil {
return errors.New("contains is a string or a list of strings")
}
*l = v
return nil
}
// Kind is how a case is scored: exact, contains, regex, number,
// json_schema or judge.
func (c Case) Kind() string {
var kinds []string
if c.Exact != nil {
kinds = append(kinds, "exact")
}
if len(c.Contains) > 0 {
kinds = append(kinds, "contains")
}
if c.Regex != "" {
kinds = append(kinds, "regex")
}
if c.Number != nil {
kinds = append(kinds, "number")
}
if c.JSONSchema != nil {
kinds = append(kinds, "json_schema")
}
if c.Judge != "" {
kinds = append(kinds, "judge")
}
return strings.Join(kinds, "+")
}
// DefaultThreshold is the score a suite that names none is passed at.
const DefaultThreshold = 0.8
// defaultMaxTokens bounds an answer when neither the case nor the suite do.
const defaultMaxTokens = 512
// Parse reads a suite: YAML (a mapping with its cases under cases:), or
// JSON lines (one case on each, and optionally a line without a prompt or
// messages that holds the suite's own settings). The name is the file's,
// used when the suite gives none.
func Parse(file string, data []byte) (Suite, error) {
var s Suite
var err error
if isJSONLines(data) {
s, err = parseJSONLines(data)
} else {
dec := yaml.NewDecoder(bytes.NewReader(data))
dec.KnownFields(true)
err = dec.Decode(&s)
if errors.Is(err, io.EOF) {
err = errors.New("the file is empty")
}
}
if err != nil {
return Suite{}, fmt.Errorf("%s: %w", file, err)
}
if s.Name == "" {
base := path.Base(strings.ReplaceAll(file, "\\", "/"))
s.Name = strings.TrimSuffix(base, path.Ext(base))
}
sum := sha256.Sum256(data)
s.Hash = hex.EncodeToString(sum[:])
if err := s.check(); err != nil {
return Suite{}, fmt.Errorf("%s: %w", file, err)
}
return s, nil
}
// isJSONLines is whether every line that is not blank is a JSON object, and
// there is more than one, or the one there is is a case rather than a whole
// suite.
func isJSONLines(data []byte) bool {
n := 0
for _, line := range bytes.Split(data, []byte("\n")) {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
if line[0] != '{' || !json.Valid(line) {
return false
}
n++
}
if n == 1 {
var probe struct {
Cases json.RawMessage `json:"cases"`
}
_ = json.Unmarshal(bytes.TrimSpace(data), &probe)
return probe.Cases == nil
}
return n > 0
}
func parseJSONLines(data []byte) (Suite, error) {
var s Suite
for i, line := range bytes.Split(data, []byte("\n")) {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
var probe struct {
Prompt *string `json:"prompt"`
Messages json.RawMessage `json:"messages"`
}
_ = json.Unmarshal(line, &probe)
dec := json.NewDecoder(bytes.NewReader(line))
dec.DisallowUnknownFields()
if probe.Prompt == nil && probe.Messages == nil {
cases := s.Cases
if err := dec.Decode(&s); err != nil {
return Suite{}, fmt.Errorf("line %d: %w", i+1, err)
}
s.Cases = append(cases, s.Cases...)
continue
}
var c Case
if err := dec.Decode(&c); err != nil {
return Suite{}, fmt.Errorf("line %d: %w", i+1, err)
}
s.Cases = append(s.Cases, c)
}
return s, nil
}
// check fills in what a suite left out and refuses what cannot be run.
func (s *Suite) check() error {
if len(s.Cases) == 0 {
return errors.New("no cases: a suite has them under cases:, or one on each line of JSON")
}
if s.Threshold < 0 || s.Threshold > 1 {
return fmt.Errorf("threshold %v is not 0 to 1", s.Threshold)
}
if s.Threshold == 0 {
s.Threshold = DefaultThreshold
}
seen := map[string]bool{}
for i := range s.Cases {
c := &s.Cases[i]
if c.ID == "" {
c.ID = fmt.Sprintf("case-%d", i+1)
}
if seen[c.ID] {
return fmt.Errorf("two cases are called %s", c.ID)
}
seen[c.ID] = true
if err := c.check(); err != nil {
return fmt.Errorf("case %s: %w", c.ID, err)
}
}
return nil
}
func (c *Case) check() error {
if strings.TrimSpace(c.Prompt) == "" && len(c.Messages) == 0 {
return errors.New("no prompt and no messages: nothing to ask")
}
if c.Prompt != "" && len(c.Messages) > 0 {
return errors.New("a prompt and messages both: give one")
}
for _, m := range c.Messages {
switch m.Role {
case "system", "user", "assistant":
default:
return fmt.Errorf("a message's role is system, user or assistant, not %q", m.Role)
}
}
switch k := c.Kind(); {
case k == "":
return errors.New("no way to score it: give one of exact, contains, regex, number, json_schema or judge")
case strings.Contains(k, "+"):
return fmt.Errorf("scored by %s: give one", strings.ReplaceAll(k, "+", " and "))
}
if c.Weight < 0 {
return fmt.Errorf("weight %v is negative", c.Weight)
}
if c.Weight == 0 {
c.Weight = 1
}
if c.Tolerance < 0 {
return fmt.Errorf("tolerance %v is negative", c.Tolerance)
}
if c.Regex != "" {
expr := c.Regex
if c.IgnoreCase {
expr = "(?i)" + expr
}
re, err := regexp.Compile(expr)
if err != nil {
return fmt.Errorf("the regex: %w", err)
}
c.re = re
}
if c.JSONSchema != nil {
sc, err := compileSchema(c.JSONSchema)
if err != nil {
return fmt.Errorf("the JSON schema: %w", err)
}
c.schema = sc
}
return nil
}
// NeedsJudge is whether any case is graded by a model.
func (s Suite) NeedsJudge() bool {
for _, c := range s.Cases {
if c.Judge != "" {
return true
}
}
return false
}
+19 -3
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
@@ -64,6 +65,21 @@ func newAgentCmd() *cobra.Command {
}
cfg.Version = version.Version
// Before the agent, so its tracer is the configured one.
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
Service: "llmgw-agent", Version: version.Version, Instance: cfg.NodeID,
})
if err != nil {
return err
}
defer func() {
// Bounded: a collector that is gone must not hold up a shutdown.
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = shutdownTracing(sctx)
}()
httpClient, err := agent.Client(cfg.TLS, 30*time.Second)
if err != nil {
return err
@@ -159,13 +175,13 @@ func printInventory(inv discovery.Inventory) {
if !g.Health.OK() {
health = g.Health.Problems[0]
}
fmt.Fprintf(w, " %d\t%s\t%s\t%d MiB\t%s\t%s\n",
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
fmt.Fprintf(w, " %s\t%s\t%s\t%d MiB\t%s\t%s\n",
discovery.DeviceLabel(g.Index), g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
}
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
for _, g := range inv.GPUs {
if len(g.NVLinkPeers) > 0 {
fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers)
fmt.Fprintf(w, " GPU %s NVLink peers: %v\n", discovery.DeviceLabel(g.Index), g.NVLinkPeers)
}
}
}
+85
View File
@@ -0,0 +1,85 @@
package cli
import (
"fmt"
"time"
"github.com/spf13/cobra"
)
func newAnswerCacheCmd(open opener) *cobra.Command {
var (
project string
ttl time.Duration
anyAnswer bool
off bool
)
cmd := &cobra.Command{
Use: "cache",
Short: "Reuse a project's answers: the same request is answered again without the model",
Long: "With this on, a request of the project that is the same as one answered within\n" +
"--ttl, to the same model, gets that answer again, from the gateway's memory,\n" +
"rather than asking the model. It is answered with x-llmgw-cache: hit, recorded\n" +
"as a request with no tokens, and costs nothing against a budget.\n\n" +
"Only answers that would come out the same are reused: a chat or completion\n" +
"asked at temperature 0 or with a seed, and embeddings, rerank and score.\n" +
"--any reuses every answer, sampled or not. A client sending\n" +
"Cache-Control: no-cache or x-llmgw-cache: no is always asked afresh, and so is\n" +
"a request continuing a stored conversation or under a guardrails no_store\n" +
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
"Off for every project until this is run. Without --ttl or --off, says how\n" +
"it stands. The answers are held in memory, router.cache bounds how much.",
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
" llmgw keys cache --project Acme/ml/train --off",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
switch {
case off && (ttl != 0 || anyAnswer):
return fmt.Errorf("--off, or --ttl with --any: not both")
case ttl < 0:
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
case ttl > 0 && ttl < time.Second:
return fmt.Errorf("--ttl is at least a second")
case anyAnswer && ttl == 0:
return fmt.Errorf("--any goes with --ttl: say how long answers are reused")
}
s, err := open(cmd.Context())
if err != nil {
return err
}
defer s.Close()
p, err := resolveProject(cmd.Context(), s, project)
if err != nil {
return err
}
if off || ttl > 0 {
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
return err
}
}
c, err := s.AnswerCacheOf(cmd.Context(), p.ID)
if err != nil {
return err
}
switch {
case c.TTL == 0:
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses no answers.\n", p.Path)
case c.Any:
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses every answer for %s, sampled or not.\n", p.Path, c.TTL)
default:
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
"and embeddings, rerank and score.\n", p.Path, c.TTL)
}
if off || ttl > 0 {
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
}
return nil
},
}
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
return cmd
}
+32
View File
@@ -0,0 +1,32 @@
package cli
import (
"path/filepath"
"strings"
"testing"
)
func TestAProjectIsToldToReuseItsAnswersAndToStop(t *testing.T) {
db := filepath.Join(t.TempDir(), "s.db")
mustRun(t, db, "org", "add", "org", "Acme")
mustRun(t, db, "org", "add", "team", "ml", "--in", "Acme")
mustRun(t, db, "org", "add", "project", "train", "--in", "Acme/ml")
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train"); !strings.Contains(out, "reuses no answers") {
t.Errorf("before: %s", out)
}
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train", "--ttl", "1h"); !strings.Contains(out, "for 1h0m0s: those asked at temperature 0") {
t.Errorf("on: %s", out)
}
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--ttl", "24h", "--any"); !strings.Contains(out, "every answer for 24h0m0s") {
t.Errorf("any: %s", out)
}
for _, bad := range [][]string{{"--off", "--ttl", "1h"}, {"--any"}, {"--ttl", "10ms"}} {
if _, err := run(t, db, append([]string{"keys", "cache", "--project", "train"}, bad...)...); err == nil {
t.Errorf("%v was accepted", bad)
}
}
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--off"); !strings.Contains(out, "reuses no answers") {
t.Errorf("off: %s", out)
}
}
+1 -1
View File
@@ -106,7 +106,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
f := cmd.Flags()
f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway")
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp")
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp | trtllm (whose are at /prometheus/metrics)")
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
+25 -2
View File
@@ -14,6 +14,7 @@ import (
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
@@ -147,6 +148,7 @@ func newCtlCmd() *cobra.Command {
// cert works on files rather than the admin API, so --server does not
// apply to it; it lives here because this is where operators look.
newBenchCmd(client, &asJSON),
newEvalCmd(client, &asJSON),
newStatusCmd(client, &asJSON),
newModelCmd(client, &asJSON),
newScaleCmd(client),
@@ -212,7 +214,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
// at when a model will not place and the hardware looks
// fine.
join(n.Engines),
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
}
_ = w.Flush()
@@ -410,7 +412,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
for _, r := range reps {
var nodes []string
for _, s := range r.Shards {
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
// "[0/mig2]" for a MIG instance, not the 1002 it is
// numbered by.
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, discovery.DeviceLabels(s.GPUs)))
}
// The state says stale rather than a column of its own: most
// replicas are not, a column would be empty for all of them,
@@ -540,6 +544,25 @@ func emitJSON(v any) error {
return enc.Encode(v)
}
// classesColumn is a node's GPU classes, with how many of each when it has
// more than one or MIG instances among them: a card cut into two 3g.40gb
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
// the GPUS column's 3 alone would read as three of something.
func classesColumn(n adminapi.NodeView) string {
counted := len(n.ClassGPUs) > 1
for _, c := range n.ClassGPUs {
counted = counted || c.MIG
}
if !counted {
return join(n.Classes)
}
parts := make([]string, len(n.ClassGPUs))
for i, c := range n.ClassGPUs {
parts[i] = fmt.Sprintf("%s×%d", c.Class, c.Total)
}
return strings.Join(parts, ",")
}
func join(v []string) string {
if len(v) == 0 {
return "-"
+408
View File
@@ -0,0 +1,408 @@
package cli
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
)
// newEvalService is the control plane's runner of quality suites: through
// the router as its own requests, with the suites of control.eval_dir and
// those the binary carries, kept in the store when there is one.
func newEvalService(cfg config.Server, rt *router.Router, admin *control.Controller, log *slog.Logger) *eval.Service {
builtin, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
if err != nil {
builtin = nil
}
d := eval.Deps{Call: rt.Replay, Dir: cfg.Control.EvalDir, Builtin: builtin, Log: log,
Origin: func(name string) (string, string) {
if m, ok := admin.Catalog().Resolve(name); ok && m.Origin != nil {
return m.Origin.Repo, m.Origin.Revision
}
return "", ""
},
CanaryOf: func(name string) string { return admin.Catalog().CanaryOf(name) },
}
// Only a store that is there: a nil one in the interface would look
// like one.
if st := rt.Store(); st != nil {
d.Store = st
}
return eval.NewService(d)
}
// suiteRequest is a suite named on the command line: a file here, sent
// whole, or else the name of one on the control plane.
func suiteRequest(arg string) (eval.Request, error) {
if st, err := os.Stat(arg); err == nil && !st.IsDir() {
b, err := os.ReadFile(arg)
if err != nil {
return eval.Request{}, err
}
// Read here first, so a mistake in it is said before anything is sent.
if _, err := eval.Parse(filepath.Base(arg), b); err != nil {
return eval.Request{}, err
}
return eval.Request{Content: string(b), File: filepath.Base(arg)}, nil
}
if strings.ContainsAny(arg, `/\`) {
return eval.Request{}, fmt.Errorf("%s: no such file here", arg)
}
return eval.Request{Suite: arg}, nil
}
func newEvalCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var (
suite, judge, run string
history, suites, detach bool
concurrency, limit int
minScore float64
)
c := &cobra.Command{
Use: "eval MODEL --suite FILE|NAME",
Short: "Ask a model a quality suite and score its answers",
Long: "A suite is a set of cases — a prompt or a conversation, and how its answer\n" +
"is scored: exact, contains, regex, a number within a tolerance, JSON that\n" +
"satisfies a schema, or a rubric another model on the cluster grades — with\n" +
"a weight each and a score, 0 to 1, the suite is passed at (docs/install.md,\n" +
"\"Quality suites\").\n\n" +
"It runs on the control plane, which asks the model through the router as\n" +
"its own requests, as a replay does: past admission, quotas, accounting and\n" +
"the guardrails, so the score is the weights' and not a tenant's limits'.\n" +
"--suite is a file here, sent whole, or the name of a suite in the control\n" +
"plane's control.eval_dir (/etc/llmgw/eval) or built in: arithmetic,\n" +
"json-format. Every run is kept, with the revision of the weights that\n" +
"answered; --history lists them.\n\n" +
"It exits non-zero when the model scores below the suite's threshold, or\n" +
"--min-score.",
Example: " llmgw ctl eval qwen-7b --suite arithmetic\n" +
" llmgw ctl eval qwen-7b --suite ./support.yaml --judge llama-70b --concurrency 8\n" +
" llmgw ctl eval --history qwen-7b\n" +
" llmgw ctl eval --run eval-20261004-101500-1a2b3c4d\n" +
" llmgw ctl eval --suites",
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
out := cmd.OutOrStdout()
switch {
case suites:
var list []eval.SuiteInfo
if err := client().get(cmd.Context(), "/api/v1/eval/suites", &list); err != nil {
return err
}
if *asJSON {
return emitJSON(list)
}
w := tabTo(out)
fmt.Fprintln(w, "SUITE\tCASES\tPASS AT\tJUDGE\tFROM\tABOUT")
for _, s := range list {
if s.Error != "" {
fmt.Fprintf(w, "%s\t-\t-\t-\t%s\tcannot be run: %s\n", s.Name, s.From, truncate(s.Error, 70))
continue
}
fmt.Fprintf(w, "%s\t%d\t%.2f\t%s\t%s\t%s\n", s.Name, s.Cases, s.Threshold, yesNo(s.Judge), s.From,
truncate(s.Description, 60))
}
return w.Flush()
case run != "":
var l eval.Live
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(run), &l); err != nil {
return err
}
if l.State == eval.Running && !*asJSON {
return followEval(cmd, client, l, false)
}
return showEval(cmd, l, *asJSON)
case history:
q := url.Values{}
if len(args) == 1 {
q.Set("model", args[0])
}
if limit > 0 {
q.Set("limit", strconv.Itoa(limit))
}
var runs []eval.Live
if err := client().get(cmd.Context(), "/api/v1/eval/runs?"+q.Encode(), &runs); err != nil {
return err
}
if *asJSON {
return emitJSON(runs)
}
return printEvalHistory(out, runs)
}
if len(args) != 1 {
return errors.New("name the model to ask: llmgw ctl eval MODEL --suite FILE|NAME")
}
if suite == "" {
return errors.New("--suite is required: a file, or a suite's name on the control plane (`llmgw ctl eval --suites`)")
}
req, err := suiteRequest(suite)
if err != nil {
return err
}
req.Concurrency, req.Judge, req.MinScore = concurrency, judge, minScore
body, _ := json.Marshal(req)
var l eval.Live
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(args[0])+"/eval", body, &l); err != nil {
return err
}
if detach {
fmt.Fprintf(out, "started %s on the control plane\nfollow it: llmgw ctl eval --run %s\n", l.ID, l.ID)
return nil
}
return followEval(cmd, client, l, *asJSON)
},
}
f := c.Flags()
f.StringVar(&suite, "suite", "", "the suite: a file here, or a suite's name on the control plane")
f.IntVar(&concurrency, "concurrency", 4, "cases asked at once")
f.StringVar(&judge, "judge", "", "the model that grades rubric cases, instead of the suite's judge")
f.Float64Var(&minScore, "min-score", 0, "the score to pass at, 0 to 1, instead of the suite's threshold")
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane")
f.BoolVar(&history, "history", false, "list the runs kept, of MODEL or of every model")
f.IntVar(&limit, "limit", 20, "with --history, how many runs")
f.StringVar(&run, "run", "", "show one run, by its id, with each case")
f.BoolVar(&suites, "suites", false, "list the suites the control plane can run by name")
return c
}
func yesNo(b bool) string {
if b {
return "yes"
}
return "-"
}
// followEval waits for a run, saying how far it has got, then shows it.
func followEval(cmd *cobra.Command, client func() *ctlClient, l eval.Live, asJSON bool) error {
out := cmd.OutOrStdout()
if !asJSON {
fmt.Fprintf(out, "run %s: %s asked %s, %d cases\n", l.ID, evalTarget(l), l.Suite, l.Cases)
}
shown := -1
for l.State == eval.Running {
if !asJSON && l.Done != shown {
fmt.Fprintf(out, " %d of %d answered\n", l.Done, l.Cases)
shown = l.Done
}
select {
case <-cmd.Context().Done():
return cmd.Context().Err()
case <-time.After(time.Second):
}
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(l.ID), &l); err != nil {
return err
}
}
if !asJSON {
fmt.Fprintln(out)
}
return showEval(cmd, l, asJSON)
}
func evalTarget(l eval.Live) string {
if l.Target != "" {
return l.Target
}
return l.Model
}
// showEval prints a finished run's cases and score, and fails below it.
func showEval(cmd *cobra.Command, l eval.Live, asJSON bool) error {
out := cmd.OutOrStdout()
if asJSON {
if err := emitJSON(l); err != nil {
return err
}
} else {
if l.State == eval.Failed {
return fmt.Errorf("the run could not be scored: %s", l.Error)
}
w := tabTo(out)
fmt.Fprintln(w, "CASE\tKIND\tWEIGHT\tSCORE\tTIME\tANSWER\tWHY")
for _, r := range l.Results {
score := fmt.Sprintf("%.2f", r.Score)
if !r.Passed {
score += " ✗"
}
why := r.Detail
if r.Error != "" {
why = "error: " + r.Error
}
fmt.Fprintf(w, "%s\t%s\t%g\t%s\t%s\t%s\t%s\n", r.ID, r.Kind, r.Weight, score,
(time.Duration(r.TookMs) * time.Millisecond).Round(10*time.Millisecond), dash(truncate(oneLine(r.Answer), 30)),
truncate(why, 70))
}
if err := w.Flush(); err != nil {
return err
}
fmt.Fprintln(out)
fmt.Fprintln(out, evalSummary(l))
if l.Error != "" {
fmt.Fprintln(out, l.Error)
}
}
if !l.Passed {
return fmt.Errorf("%s scored %.2f on %s, below %.2f", evalTarget(l), l.Score, l.Suite, l.Threshold)
}
return nil
}
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
// evalSummary is a run in a sentence.
func evalSummary(l eval.Live) string {
verdict := "passed"
if !l.Passed {
verdict = "failed"
}
s := fmt.Sprintf("%s scored %.2f on %s: %d of %d cases passed, %.2f asked: %s", evalTarget(l), l.Score, l.Suite,
l.Cases-l.Below, l.Cases, l.Threshold, verdict)
if l.Revision != "" {
s += fmt.Sprintf(" (weights %s at %s)", dash(l.Repo), catalog.ShortRevision(l.Revision))
}
return s + "."
}
func printEvalHistory(out io.Writer, runs []eval.Live) error {
if len(runs) == 0 {
fmt.Fprintln(out, "No runs kept: `llmgw ctl eval MODEL --suite NAME` makes one.")
return nil
}
w := tabTo(out)
fmt.Fprintln(w, "WHEN\tRUN\tMODEL\tWEIGHTS\tSUITE\tSCORE\tPASS AT\tRESULT\tFOR")
for _, r := range runs {
model := r.Model
if r.Target != "" {
model += " (" + r.Target + ")"
}
result := "passed"
switch {
case r.State == eval.Running:
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
case r.State == eval.Failed:
result = "could not run"
case !r.Passed:
result = "failed"
}
score := fmt.Sprintf("%.2f", r.Score)
if r.State == eval.Running {
score = "-"
}
forWhat := r.Job
if forWhat == "" {
forWhat = r.By
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%.2f\t%s\t%s\n", ago(r.At, time.Now()), r.ID, model,
dash(catalog.ShortRevision(r.Revision)), r.Suite, score, r.Threshold, result, dash(forWhat))
}
return w.Flush()
}
// gateFlags are how an update or a promotion is held to a quality suite.
type gateFlags struct {
suite, judge string
minScore float64
skip bool
}
func (g *gateFlags) add(c *cobra.Command) {
f := c.Flags()
f.StringVar(&g.suite, "eval", "", "a quality suite the new weights must pass first: a file here, or a suite's name on the control plane")
f.Float64Var(&g.minScore, "min-score", 0, "with --eval, the score to pass at, 0 to 1; the suite's threshold otherwise")
f.StringVar(&g.judge, "judge", "", "with --eval, the model that grades rubric cases")
f.BoolVar(&g.skip, "skip-eval", false, "move without the model's own quality suite (`llmgw ctl model gate`)")
}
// body is the change, as the API takes it.
func (g gateFlags) body() ([]byte, error) {
var ch modelserve.Change
if g.suite != "" {
if g.skip {
return nil, errors.New("--eval and --skip-eval together: give one")
}
r, err := suiteRequest(g.suite)
if err != nil {
return nil, err
}
r.MinScore, r.Judge = g.minScore, g.judge
ch.Eval = &r
} else if g.minScore != 0 || g.judge != "" {
return nil, errors.New("--min-score and --judge go with --eval")
}
ch.SkipEval = g.skip
return json.Marshal(ch)
}
func newModelGateCmd(client func() *ctlClient) *cobra.Command {
var (
g adminapi.EvalGateRequest
off bool
)
c := &cobra.Command{
Use: "gate MODEL --eval SUITE [--min-score N]",
Short: "Hold a model's updates and promotions to a quality suite",
Long: "From then on `llmgw ctl model update` and `model promote` (and the\n" +
"console's) ask the new weights the suite before they replace the old,\n" +
"and stop below its score: an update runs them first as a canary given\n" +
"none of the model's requests, and takes it out again when it fails. The\n" +
"suite is one on the control plane, by name: in control.eval_dir, or\n" +
"built in. --skip-eval on an update moves without it, once; --off takes it\n" +
"away.",
Example: " llmgw ctl model gate qwen-7b --eval arithmetic --min-score 0.8\n" +
" llmgw ctl model gate qwen-7b --off",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/eval-gate"
var out struct {
Model string `json:"model"`
Eval *catalog.EvalGate `json:"eval"`
}
if off {
if err := client().delete(cmd.Context(), path, &out); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s's updates are no longer held to a quality suite\n", args[0])
return nil
}
if g.Suite == "" {
return errors.New("--eval names the suite, or --off takes the gate away")
}
body, _ := json.Marshal(g)
if err := client().put(cmd.Context(), path, body, &out); err != nil {
return err
}
at := "the suite's threshold"
if g.MinScore > 0 {
at = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
}
fmt.Fprintf(cmd.OutOrStdout(), "%s's new weights must score at least %s on %s before they replace the old\n",
args[0], at, g.Suite)
return nil
},
}
c.Flags().StringVar(&g.Suite, "eval", "", "the suite: its name on the control plane")
c.Flags().Float64Var(&g.MinScore, "min-score", 0, "the score to pass at, 0 to 1; the suite's threshold otherwise")
c.Flags().StringVar(&g.Judge, "judge", "", "the model that grades the suite's rubric cases")
c.Flags().BoolVar(&off, "off", false, "take the gate away")
return c
}
+123
View File
@@ -0,0 +1,123 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestTheSuitesTheBinaryCarriesCanBeRun(t *testing.T) {
sub, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
if err != nil {
t.Fatal(err)
}
got := eval.NewService(eval.Deps{Builtin: sub}).Suites()
var names []string
for _, s := range got {
if s.Error != "" || s.Cases < 5 || s.Judge {
t.Errorf("%s: %+v", s.Name, s)
}
names = append(names, s.Name)
}
if strings.Join(names, " ") != "arithmetic json-format" {
t.Errorf("built in: %v", names)
}
}
func TestCtlEvalSendsAFileAndFailsBelowTheScore(t *testing.T) {
var mu sync.Mutex
var sent []string
finished := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "mine", Score: 0.5,
Threshold: 0.8, Cases: 2, Below: 1, Revision: "abcdef1234567", Repo: "o/r", Results: []store.EvalCase{
{ID: "add", Kind: "number", Weight: 1, Score: 1, Passed: true, Answer: "42", Detail: "read 42"},
{ID: "mul", Kind: "number", Weight: 1, Score: 0, Answer: "43", Detail: "read 43, wanted 42"}}},
State: eval.Done, Done: 2}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
mu.Lock()
sent = append(sent, r.Method+" "+r.URL.Path+" "+string(b))
mu.Unlock()
switch r.URL.Path {
case "/api/v1/models/qwen/eval":
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Cases: 2},
State: eval.Running})
case "/api/v1/eval/runs/eval-1":
_ = json.NewEncoder(w).Encode(finished)
case "/api/v1/models/qwen/update":
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(modelserve.Job{ID: "update-1", Model: "qwen", State: modelserve.Done})
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
file := filepath.Join(t.TempDir(), "mine.yaml")
if err := os.WriteFile(file, []byte("cases:\n - prompt: 40 + 2?\n number: 42\n"), 0o600); err != nil {
t.Fatal(err)
}
ctl := func(args ...string) (string, error) {
cmd := newCtlCmd()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs(append([]string{"--server", srv.URL}, args...))
err := cmd.ExecuteContext(context.Background())
return out.String(), err
}
out, err := ctl("eval", "qwen", "--suite", file, "--concurrency", "2")
if err == nil || !strings.Contains(err.Error(), "qwen scored 0.50 on mine, below 0.80") {
t.Fatalf("a run below its score did not fail: %v\n%s", err, out)
}
for _, want := range []string{"read 43, wanted 42", "1 of 2 cases passed", "weights o/r at abcdef1"} {
if !strings.Contains(out, want) {
t.Errorf("the output lacks %q:\n%s", want, out)
}
}
mu.Lock()
var req eval.Request
_ = json.Unmarshal([]byte(strings.SplitN(sent[0], " ", 3)[2]), &req)
mu.Unlock()
if req.File != "mine.yaml" || !strings.Contains(req.Content, "40 + 2?") || req.Concurrency != 2 {
t.Errorf("sent %+v", req)
}
// A broken file is said here, before anything is sent.
if err := os.WriteFile(file, []byte("cases: []\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := ctl("eval", "qwen", "--suite", file); err == nil || !strings.Contains(err.Error(), "no cases") {
t.Errorf("%v", err)
}
// An update held to a suite by name carries it.
if _, err := ctl("model", "update", "qwen", "--eval", "arithmetic", "--min-score", "0.9"); err != nil {
t.Fatal(err)
}
mu.Lock()
last := sent[len(sent)-1]
mu.Unlock()
var ch modelserve.Change
if err := json.Unmarshal([]byte(strings.SplitN(last, " ", 3)[2]), &ch); err != nil || ch.Eval == nil ||
ch.Eval.Suite != "arithmetic" || ch.Eval.MinScore != 0.9 {
t.Errorf("sent %s", last)
}
if _, err := ctl("model", "update", "qwen", "--min-score", "0.9"); err == nil {
t.Error("--min-score without --eval was sent")
}
}
+3
View File
@@ -56,6 +56,9 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
fmt.Fprintf(out, "status %d\n", v.Status)
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
if v.CacheHit {
fmt.Fprintf(out, "cache answered from the project's answer cache: nothing generated, nothing charged\n")
}
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
if v.EndUser != "" {
fmt.Fprintf(out, "end user %s (the client's own name for them)\n", v.EndUser)
+3 -2
View File
@@ -110,9 +110,10 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
if len(f.GPUs) > 0 {
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
fmt.Fprintln(w, "\nGPU\tNAME\tCLASS\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
for _, g := range f.GPUs {
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
// A MIG instance as 0/mig2, under its card.
fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\n", discovery.DeviceLabel(g.Index), g.Name, g.Class(), g.NUMANode,
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
}
}
+15 -6
View File
@@ -57,6 +57,8 @@ func newEngineCheckCmd() *cobra.Command {
cfg.VLLM.Bin = bin
case "sglang":
cfg.SGLang.Python = bin
case "trtllm":
cfg.TRTLLM.Python = bin
case "sim":
cfg.Sim.Bin = bin
}
@@ -97,7 +99,7 @@ func newEngineCheckCmd() *cobra.Command {
continue
default:
var err error
if text, err = runHelp(cmd.Context(), c); err != nil {
if text, err = runHelp(cmd.Context(), eng, c); err != nil {
skipped++
fmt.Fprintf(errOut, " skipped: %v\n", err)
continue
@@ -167,9 +169,10 @@ func newEngineCheckCmd() *cobra.Command {
// representativeShard is the widest command llmgw would ever build, because a
// flag that only appears in the multi-node case is exactly the one nobody
// notices has been renamed.
// notices has been renamed. For an engine that runs a replica on one node,
// that is one node.
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
if nodes < 1 {
if nodes < 1 || engine == "trtllm" {
nodes = 1
}
return engines.ShardSpec{
@@ -203,9 +206,10 @@ var helpArgs = map[string][]string{
}
// runHelp asks one program to describe itself.
func runHelp(ctx context.Context, c engines.Command) (string, error) {
func runHelp(ctx context.Context, eng engines.Engine, c engines.Command) (string, error) {
// Everything up to the first flag, plus the help request. Passing the real
// arguments would start the engine.
path := c.Path
var args []string
for _, a := range c.Args {
if strings.HasPrefix(a, "-") {
@@ -218,11 +222,16 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
ask = []string{"--help"}
}
args = append(args, ask...)
// An engine started through a launcher of llmgw's is asked through its
// own entry point: the interpreter's --help describes the interpreter.
if h, ok := eng.(engines.HelpCommand); ok {
path, args = h.HelpCommand(c)
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ec := exec.CommandContext(ctx, c.Path, args...)
ec := exec.CommandContext(ctx, path, args...)
// vLLM refuses to do anything at all on a machine with no accelerator,
// including print its help, unless it is told what to assume. Asking an
// engine about its flags should not require the hardware it runs on.
@@ -231,7 +240,7 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
raw, err := ec.CombinedOutput()
if err != nil && len(raw) == 0 {
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
c.Path, err)
path, err)
}
return string(raw), nil
}
+77
View File
@@ -1,6 +1,11 @@
package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -69,3 +74,75 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
t.Error("a truncated help was taken for a complete one")
}
}
// fakeInterpreter stands in for TensorRT-LLM's Python: asked for
// trtllm-serve's help through its module, it prints the help given; asked
// anything else, it fails, as running the launcher here would.
func fakeInterpreter(t *testing.T, help string) string {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
t.Fatal(err)
}
p := filepath.Join(dir, "python")
body := "#!/bin/sh\n" +
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
"echo \"unexpected: $*\" >&2; exit 2\n"
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
t.Fatal(err)
}
return p
}
// trtllmHelp is the part of trtllm-serve serve --help (1.3) that holds the
// flags llmgw builds.
const trtllmHelp = `Usage: trtllm-serve serve [OPTIONS] MODEL
Options:
--host TEXT Hostname of the server.
--port INTEGER Port of the server.
--tensor_parallel_size, --tp_size INTEGER
Tensor parallelism size.
--pipeline_parallel_size, --pp_size INTEGER
Pipeline parallelism size.
--config, --extra_llm_api_options TEXT
Path to a YAML configuration file.
--served_model_name TEXT The model name used in the API.
--help Show this message and exit.
`
func runEngineCheck(t *testing.T, args ...string) (string, error) {
t.Helper()
c := newEngineCheckCmd()
var out bytes.Buffer
c.SetOut(&out)
c.SetErr(&out)
c.SetArgs(args)
err := c.ExecuteContext(context.Background())
return out.String(), err
}
func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, trtllmHelp))
if err != nil {
t.Fatalf("%v\n%s", err, out)
}
if !strings.Contains(out, "every flag llmgw builds is accepted") {
t.Errorf("unexpected report:\n%s", out)
}
// The interpreter's own help would describe Python, not trtllm-serve; and
// one node is what trtllm runs, so no multi-node flag is in the command.
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
t.Errorf("the help was not asked of trtllm-serve:\n%s", out)
}
}
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
// never be seen serving the name the agent waits for.
old := strings.ReplaceAll(trtllmHelp, " --served_model_name TEXT The model name used in the API.\n", "")
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, old))
if err == nil || !strings.Contains(out, "trtllm-serve: --served_model_name") {
t.Errorf("an old trtllm-serve passed: %v\n%s", err, out)
}
}
+1
View File
@@ -44,6 +44,7 @@ func newTenancyCmd() *cobra.Command {
newThreadsCmd(open),
newRequestLogCmd(open),
newResidencyCmd(open),
newAnswerCacheCmd(open),
newMemoryCmd(open),
newMoveStoreCmd(),
)
+40
View File
@@ -0,0 +1,40 @@
package cli
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
func TestAPlanSeesANodesMIGInstancesAsAClassOfTheirOwn(t *testing.T) {
// Card 0 cut into two 3g.40gb instances, both free, and card 1 whole and
// busy: the instances are a pool of forty-gigabyte devices, and the busy
// card no pool at all.
hw := hardwareFrom([]adminapi.NodeView{{ID: "a100", State: "ready", GPUTotal: 3, GPUFree: 2,
Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"},
ClassGPUs: []adminapi.ClassGPUs{
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 2, MIG: true},
{Class: "a100-80g", Total: 1, Free: 0},
}}})
if len(hw) != 1 {
t.Fatalf("pools: %+v", hw)
}
want := modelplan.Hardware{Class: "a100-80g-mig-3g.40gb", GPUMemGB: 40, GPUsPerNode: 2, Nodes: 1}
if hw[0] != want {
t.Errorf("pool %+v, want %+v", hw[0], want)
}
}
func TestCtlNodesCountsANodesDevicesByClassWhenItHasMIG(t *testing.T) {
mig := adminapi.NodeView{Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"}, ClassGPUs: []adminapi.ClassGPUs{
{Class: "a100-80g-mig-3g.40gb", Total: 2, MIG: true}, {Class: "a100-80g", Total: 1}}}
if got := classesColumn(mig); got != "a100-80g-mig-3g.40gb×2,a100-80g×1" {
t.Errorf("MIG node: %q", got)
}
// A node of one kind of whole card reads as it always did.
plain := adminapi.NodeView{Classes: []string{"h100-80g"}, ClassGPUs: []adminapi.ClassGPUs{{Class: "h100-80g", Total: 8}}}
if got := classesColumn(plain); got != "h100-80g" {
t.Errorf("plain node: %q", got)
}
}
+2
View File
@@ -44,6 +44,8 @@ func newMirrorCmd() *cobra.Command {
c.AddCommand(
newMirrorAddCmd(&dir),
newMirrorImportCmd(&dir),
newMirrorImportOllamaCmd(&dir),
newMirrorListCmd(&dir),
newMirrorUnusedCmd(&dir),
newMirrorRemoveCmd(&dir),
+216
View File
@@ -0,0 +1,216 @@
package cli
import (
"context"
"fmt"
"io"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
// placeholderBase is how the mirror's address is written when the operator
// has not said it, as `mirror add` writes it.
const placeholderBase = "http://<this-host>:8081"
func newMirrorImportCmd(dir *string) *cobra.Command {
var model, version, as, sum, base string
c := &cobra.Command{
Use: "import <file>... --model NAME",
Short: "Copy model files on this machine into the mirror",
Long: "For weights already here: a GGUF you converted or downloaded by hand,\n" +
"or the files of a safetensors model. The same discipline as add — copied\n" +
"under a temporary name and renamed when whole, the checksum kept beside\n" +
"it for nodes to check against, an existing version never replaced.\n\n" +
"A file named .gguf is checked to be one. Several files go into one\n" +
"version, as the files of one model; its source is then the version,\n" +
"ending in a slash. Run it on the mirror; for a model in Ollama, use\n" +
"import-ollama.",
Example: " llmgw mirror import ./qwen2.5-7b-instruct-q4_k_m.gguf --model qwen2.5-7b-instruct",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if model == "" {
return fmt.Errorf("--model is required: it is the mirror's directory for the model")
}
if as != "" && len(args) > 1 {
return fmt.Errorf("--as names one file; with %d files each keeps its own name", len(args))
}
if sum != "" && len(args) > 1 {
return fmt.Errorf("--sha256 is one file's checksum; with %d files, import them one at a time", len(args))
}
if version == "" {
version = time.Now().Format("2006-01-02")
}
out := cmd.OutOrStdout()
var added []mirror.Added
for _, src := range args {
abs, err := filepath.Abs(src)
if err != nil {
return err
}
file := as
if file == "" {
file = filepath.Base(src)
}
e := mirror.Entry{Model: model, Version: version, File: file}
fmt.Fprintf(out, "copying %s\n", src)
a, err := mirror.Import(cmd.Context(), *dir, abs, e, sum, "file://"+abs, progressTo(out))
if err != nil {
return err
}
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n", model, version, file, fileSize(a.Entry.Bytes), a.SHA256)
added = append(added, a)
}
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", importedSource(base, added))
return nil
},
}
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
c.Flags().StringVar(&version, "version", "", "the version, one directory level (default today's date)")
c.Flags().StringVar(&as, "as", "", "the filename to keep it as, for one file; its own name when omitted")
c.Flags().StringVar(&sum, "sha256", "", "the checksum its publisher gave, to refuse a file that does not match")
c.Flags().StringVar(&base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
return c
}
func newMirrorImportOllamaCmd(dir *string) *cobra.Command {
var o ollamaImport
c := &cobra.Command{
Use: "import-ollama [NAME[:TAG]]",
Short: "Put a model you already have in Ollama into the mirror, for llama.cpp",
Long: "Ollama keeps each model as a GGUF under its models directory, named by\n" +
"its sha256. This copies that file into the mirror — checked against the\n" +
"digest it is named by, and with a vision model's projector beside it —\n" +
"so llama.cpp can serve it on any node, from your own network.\n\n" +
"With no name it lists the models Ollama has. The mirror version is the\n" +
"tag and the start of the digest, since a tag such as latest moves in\n" +
"Ollama and a mirror version must not; --model and --version choose\n" +
"otherwise.\n\n" +
"Ollama's prompt template is not carried: llama.cpp renders the chat\n" +
"template inside the GGUF. Run it on the mirror, where Ollama's directory\n" +
"is readable — ~/.ollama/models, /usr/share/ollama/.ollama/models for\n" +
"Ollama's Linux service, or OLLAMA_MODELS.",
Example: " llmgw mirror import-ollama\n" +
" sudo llmgw mirror import-ollama llama3.2:3b --ollama-dir /usr/share/ollama/.ollama/models",
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
o.mirrorDir = *dir
if len(args) == 1 {
o.name = args[0]
}
return o.run(cmd.Context(), cmd.OutOrStdout())
},
}
c.Flags().StringVar(&o.ollamaDir, "ollama-dir", "", "Ollama's models directory (default: found where Ollama keeps it)")
c.Flags().StringVar(&o.model, "model", "", "the mirror's directory for it (default the Ollama model's name)")
c.Flags().StringVar(&o.version, "version", "", "the mirror version (default its tag and the start of its digest)")
c.Flags().StringVar(&o.base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
return c
}
// ollamaImport is one import-ollama, from the command line or the console.
type ollamaImport struct {
mirrorDir, ollamaDir, name, model, version, base string
// quiet leaves out the copy's progress, for the console, which shows
// the output once the import is over.
quiet bool
}
func (o ollamaImport) run(ctx context.Context, out io.Writer) error {
dir, err := mirror.OllamaDir(o.ollamaDir)
if err != nil {
return err
}
if o.name == "" {
names, err := mirror.ListOllama(dir)
if err != nil {
return err
}
if len(names) == 0 {
fmt.Fprintf(out, "Ollama in %s has no models.\n", dir)
return nil
}
fmt.Fprintf(out, "Ollama's models in %s:\n", dir)
for _, n := range names {
fmt.Fprintf(out, " %s\n", n)
}
fmt.Fprintln(out, "\nImport one with: llmgw mirror import-ollama NAME:TAG")
return nil
}
n, err := mirror.ParseOllamaName(o.name)
if err != nil {
return err
}
m, err := mirror.ReadOllama(dir, n)
if err != nil {
return err
}
fmt.Fprintf(out, "importing %s from %s\n", n, dir)
last := ""
added, err := mirror.ImportOllama(ctx, o.mirrorDir, dir, m, o.model, o.version, func(file string, done, total int64) {
if file != last {
fmt.Fprintf(out, "copying %s\n", file)
last = file
}
if !o.quiet {
progressTo(out)(done, total)
}
})
if err != nil {
return err
}
for _, a := range added {
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s, as Ollama named it\n",
a.Entry.Model, a.Entry.Version, a.Entry.File, fileSize(a.Entry.Bytes), a.SHA256)
}
for _, l := range m.Left {
fmt.Fprintf(out, "Not carried: %s.\n", l)
}
name := n.Model
if n.Tag != "latest" {
name += "-" + n.Tag
}
fmt.Fprintf(out, "\nA model definition for it, for llmgw ctl model add -f:\n\n"+
"name: %s\nsource: %s\nprofiles:\n - name: cpu\n engine: llamacpp\n",
name, importedSource(o.base, added))
if m.ContextLength > 0 {
// Ollama's context, which llama.cpp's own default would not match;
// llama.cpp divides it between its parallel slots.
fmt.Fprintf(out, " args: [\"--ctx-size\", %q] # Ollama ran it with num_ctx %d\n",
strconv.Itoa(m.ContextLength), m.ContextLength)
}
return nil
}
// importedSource is what a model's source says for what was just imported:
// the file, or the version's directory when it is several.
func importedSource(base string, added []mirror.Added) string {
if base == "" {
base = placeholderBase
}
if len(added) == 1 {
return added[0].Entry.URL(base)
}
e := added[0].Entry
return strings.TrimSuffix(mirror.Entry{Model: e.Model, Version: e.Version}.URL(base), "/") + "/"
}
// progressTo reports a copy's progress the way add reports a download's.
func progressTo(out io.Writer) func(done, total int64) {
return func(done, total int64) {
if total > 0 {
fmt.Fprintf(out, " %s of %s (%d%%)\n", fileSize(done), fileSize(total), done*100/total)
return
}
fmt.Fprintf(out, " %s so far\n", fileSize(done))
}
}
+144
View File
@@ -0,0 +1,144 @@
package cli
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
tea "github.com/charmbracelet/bubbletea"
)
// runMirror runs `llmgw mirror ...` as an operator would and returns what it
// printed.
func runMirror(t *testing.T, args ...string) (string, error) {
t.Helper()
c := Root()
var out bytes.Buffer
c.SetOut(&out)
c.SetErr(&out)
c.SetArgs(append([]string{"mirror"}, args...))
err := c.ExecuteContext(t.Context())
return out.String(), err
}
// ollamaWith lays out an Ollama models directory holding one model, as
// Ollama writes it, and returns the directory and the weights' sha256.
func ollamaWith(t *testing.T, name, tag string, weights []byte, params string) (string, string) {
t.Helper()
dir := filepath.Join(t.TempDir(), "models")
blob := func(content []byte) string {
s := sha256.Sum256(content)
h := hex.EncodeToString(s[:])
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "blobs", "sha256-"+h), content, 0o644); err != nil {
t.Fatal(err)
}
return h
}
w := blob(weights)
p := blob([]byte(params))
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json",` +
`"config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:` + blob([]byte("{}")) + `","size":2},` +
`"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:` + w + `","size":` + strconv.Itoa(len(weights)) + `},` +
`{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:` + p + `","size":` + strconv.Itoa(len(params)) + `}]}`
mdir := filepath.Join(dir, "manifests", "registry.ollama.ai", "library", name)
if err := os.MkdirAll(mdir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(mdir, tag), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
return dir, w
}
func TestImportOllamaListsThenImportsAModelForLlamaCpp(t *testing.T) {
weights := []byte("GGUF\x03\x00\x00\x00qwen weights")
ollama, sum := ollamaWith(t, "qwen2.5", "7b", weights, `{"num_ctx":16384}`)
root := t.TempDir()
out, err := runMirror(t, "--dir", root, "import-ollama", "--ollama-dir", ollama)
if err != nil || !strings.Contains(out, " qwen2.5:7b\n") {
t.Fatalf("listing: %v\n%s", err, out)
}
out, err = runMirror(t, "--dir", root, "import-ollama", "qwen2.5:7b", "--ollama-dir", ollama,
"--base", "http://models.internal:8081")
if err != nil {
t.Fatalf("%v\n%s", err, out)
}
file := filepath.Join(root, "qwen2.5", "7b-"+sum[:12], "qwen2.5-7b.gguf")
if b, err := os.ReadFile(file); err != nil || !bytes.Equal(b, weights) {
t.Fatalf("the GGUF is not in the mirror at %s: %v", file, err)
}
for _, want := range []string{
"sha256 " + sum,
"source: http://models.internal:8081/qwen2.5/7b-" + sum[:12] + "/qwen2.5-7b.gguf",
"engine: llamacpp",
`args: ["--ctx-size", "16384"]`,
} {
if !strings.Contains(out, want) {
t.Errorf("the output does not say %q:\n%s", want, out)
}
}
}
func TestTheConsoleImportsFromOllamaIntoTheMirrorBesideIt(t *testing.T) {
weights := []byte("GGUF\x03\x00\x00\x00llama weights")
ollama, sum := ollamaWith(t, "llama3.2", "3b", weights, `{}`)
root := t.TempDir()
m, _ := consoleWith(t, cMirror)
var tm tea.Model = press(m, "i")
f := tm.(consoleModel).formD
if f == nil || f.title != "Import from Ollama" {
t.Fatalf("I opened %+v", f)
}
f.fields[0].text.SetValue("llama3.2:3b")
f.fields[1].text.SetValue(ollama)
f.fields[2].text.SetValue(root)
for range len(f.fields) {
tm = press(tm, "tab")
}
tm = pressRun(tm, "enter")
if _, err := os.Stat(filepath.Join(root, "llama3.2", "3b-"+sum[:12], "llama3.2-3b.gguf")); err != nil {
t.Fatalf("not imported: %v", err)
}
text := tm.(consoleModel).dialogText
if !strings.Contains(text, "engine: llamacpp") || strings.Contains(text, "(100%)") {
t.Errorf("the console said:\n%s", text)
}
}
func TestImportCopiesALocalGGUFIntoTheMirror(t *testing.T) {
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
content := []byte("GGUF\x03\x00\x00\x00converted")
if err := os.WriteFile(src, content, 0o644); err != nil {
t.Fatal(err)
}
root := t.TempDir()
out, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10")
if err != nil {
t.Fatalf("%v\n%s", err, out)
}
if b, err := os.ReadFile(filepath.Join(root, "qwen", "2026-10", "q4_k_m.gguf")); err != nil || !bytes.Equal(b, content) {
t.Fatalf("not imported: %v", err)
}
if !strings.Contains(out, "source: http://<this-host>:8081/qwen/2026-10/q4_k_m.gguf") {
t.Errorf("no source given:\n%s", out)
}
// The layout is the point: a version is never replaced.
if _, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10"); err == nil {
t.Error("an existing version was replaced")
}
if _, err := runMirror(t, "--dir", root, "import", src); err == nil {
t.Error("imported without --model")
}
}
+2 -1
View File
@@ -37,7 +37,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
newModelRollbackCmd(client), newModelOriginCmd(client),
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON))
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON),
newModelGateCmd(client))
return c
}
+16 -3
View File
@@ -83,17 +83,29 @@ func newModelCanaryCmd(client func() *ctlClient) *cobra.Command {
}
func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
var detach bool
var (
detach bool
gate gateFlags
)
c := &cobra.Command{
Use: "promote MODEL",
Short: "Move a model to its canary's weights, and take the canary out",
Long: "The update the canary was a trial of: the catalog names the canary's\n" +
"files, the model's replicas are replaced one at a time, and the canary\n" +
"is removed. `llmgw ctl model rollback MODEL` goes back.",
"is removed. `llmgw ctl model rollback MODEL` goes back.\n\n" +
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
"gate`), the canary is asked it first, and below its score nothing\n" +
"moves: the canary stays, with its share, until it is taken out.",
Example: " llmgw ctl model promote qwen-7b\n" +
" llmgw ctl model promote qwen-7b --eval ./support.yaml --min-score 0.9 --judge llama-70b",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
body, err := gate.body()
if err != nil {
return err
}
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", []byte("{}"), &job); err != nil {
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", body, &job); err != nil {
return err
}
if detach {
@@ -103,6 +115,7 @@ func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
},
}
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
gate.add(c)
return c
}
+17 -2
View File
@@ -82,7 +82,10 @@ func updateCell(u modelserve.UpdateInfo) string {
}
func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
var detach bool
var (
detach bool
gate gateFlags
)
c := &cobra.Command{
Use: "update MODEL",
Short: "Move a model to the weights its repository publishes now",
@@ -90,12 +93,23 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
"checked against the registry's checksums and the nodes' llama.cpp; then\n" +
"the catalog names them and the replicas are replaced one at a time, each\n" +
"new one up before an old one stops, and one is asked to say hello.\n\n" +
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
"gate`), the new weights first run beside the old as MODEL-canary, given\n" +
"none of its requests, and are asked the suite; below its score they are\n" +
"taken out again and the model keeps its weights. That needs room for one\n" +
"more replica while it runs.\n\n" +
"The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" +
"back to them. A model from server.yaml is changed there instead.",
Example: " llmgw ctl model update qwen-7b\n" +
" llmgw ctl model update qwen-7b --eval arithmetic --min-score 0.8",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
body, err := gate.body()
if err != nil {
return err
}
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil {
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", body, &job); err != nil {
return err
}
if detach {
@@ -105,6 +119,7 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
},
}
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
gate.add(c)
return c
}
+35 -32
View File
@@ -155,28 +155,41 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
if len(n.Fabrics) > 0 {
fabric = n.Fabrics[0]
}
class := ""
if len(n.Classes) > 0 {
class = n.Classes[0]
// Each class of the node's devices is a pool of its own: a node with
// a card in MIG mode has its instances' classes, and any whole card's
// beside them. A node that does not count by class is its first
// class, as it always was.
kinds := n.ClassGPUs
if len(kinds) == 0 {
class := ""
if len(n.Classes) > 0 {
class = n.Classes[0]
}
kinds = []adminapi.ClassGPUs{{Class: class, Total: n.GPUTotal, Free: n.GPUFree}}
}
k := key{class, fabric}
if pools[k] == nil {
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
}
p := pools[k]
p.Nodes++
// The widest a single node could contribute, not the total: a tensor
// group takes the same number of cards from every node it spans.
if n.GPUFree > p.GPUsPerNode {
p.GPUsPerNode = n.GPUFree
}
if n.MemoryGB > p.HostMemGB {
p.HostMemGB = n.MemoryGB
}
// Card memory from the class label, which carries it by construction:
// "h100-80g" is eighty gigabytes and says so.
if class != "" && p.GPUMemGB == 0 {
p.GPUMemGB = memFromClass(class)
for _, kind := range kinds {
class := kind.Class
k := key{class, fabric}
if pools[k] == nil {
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
}
p := pools[k]
p.Nodes++
// The widest a single node could contribute, not the total: a
// tensor group takes the same number of cards from every node it
// spans.
if kind.Free > p.GPUsPerNode {
p.GPUsPerNode = kind.Free
}
if n.MemoryGB > p.HostMemGB {
p.HostMemGB = n.MemoryGB
}
// Card memory from the class label, which carries it by
// construction: "h100-80g" is eighty gigabytes and says so, and
// "a100-80g-mig-3g.40gb" is an instance of forty.
if class != "" && p.GPUMemGB == 0 {
p.GPUMemGB = memFromClass(class)
}
}
}
@@ -191,17 +204,7 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
}
// memFromClass reads the card size out of a class label such as "h100-80g".
func memFromClass(class string) int {
i := strings.LastIndex(class, "-")
if i < 0 || !strings.HasSuffix(class, "g") {
return 0
}
var gb int
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
return 0
}
return gb
}
func memFromClass(class string) int { return modelplan.MemFromClass(class) }
// catalogEntry renders a fit as the YAML that would serve it.
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
+12 -2
View File
@@ -9,6 +9,7 @@ import (
"maps"
"net"
"net/http"
"os"
"os/signal"
"slices"
"strings"
@@ -32,6 +33,7 @@ import (
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
@@ -102,10 +104,12 @@ func newServerCmd() *cobra.Command {
"or give --endpoint for a router-only run")
}
// Before the router, so its tracer is the configured one.
// Before the router, so its tracer is the configured one. The
// host tells two control planes' traces apart.
host, _ := os.Hostname()
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
Service: "llmgw-server", Version: version.Version,
Service: "llmgw-server", Version: version.Version, Instance: host,
})
if err != nil {
return err
@@ -286,6 +290,7 @@ func newServerCmd() *cobra.Command {
var (
adminAPI *adminapi.API
catalogSvc *modelsvc.Service
evals *eval.Service
)
if admin != nil {
api := adminapi.New(admin)
@@ -303,6 +308,9 @@ func newServerCmd() *cobra.Command {
api.SetReleases(cfg.Releases)
api.SetNotifier(notifier)
api.SetReplayer(rt.Replay)
// Quality suites, asked of a model the way a replay is.
evals = newEvalService(cfg, rt, admin, log)
api.SetEval(evals)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
}
@@ -419,6 +427,8 @@ func newServerCmd() *cobra.Command {
Pool: adminAPI.PoolNodes, Notify: notifier,
// Canaries: how the router saw each side serve.
CanaryStats: rt.CanaryStats, ResetCanary: rt.ResetCanary,
// Updates and promotions held to a quality suite.
Eval: evals,
})
go serve.WatchCanaries(ctx, 30*time.Second)
adminAPI.SetServe(serve)
+4 -3
View File
@@ -423,6 +423,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
}},
}
items = append(items, m.updateItems(md)...)
items = append(items, m.evalItems(md)...)
if md.Parked {
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
@@ -706,7 +707,7 @@ func (m consoleModel) canaryItems(md adminapi.ModelView) []menuItem {
{"Promote the canary…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Promote " + cn.Model,
text: md.Name + " moves to the canary's weights: its replicas are replaced one at a time,\n" +
"and the canary is removed. Going back is \"Go back to\" on this menu.",
"and the canary is removed. Going back is \"Go back to\" on this menu." + gateNote(md),
yes: func(m *consoleModel) tea.Cmd {
return m.call("Promote "+cn.Model, http.MethodPost, path+"/promote", map[string]string{}, startedSays)
}}
@@ -780,8 +781,8 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
m.confirmD = &confirmDialog{title: "Update " + md.Name,
text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
"replicas are replaced one at a time. The old files stay, to go back to.",
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")),
"replicas are replaced one at a time. The old files stay, to go back to.%s",
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", "), gateNote(md)),
yes: func(m *consoleModel) tea.Cmd {
return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update",
map[string]string{}, startedSays)
+60 -8
View File
@@ -17,6 +17,7 @@ import (
"github.com/charmbracelet/lipgloss"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -154,6 +155,11 @@ type consoleModel struct {
formD *formDialog
confirmD *confirmDialog
busy string
// busyNote says how far the request under way has got, when it can;
// evalWatch is the quality run being followed, which Esc stops
// following while it carries on.
busyNote string
evalWatch string
quotas []adminapi.QuotaView
quotasErr error
@@ -337,8 +343,12 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.people = peopleData(msg)
case consoleQuotasMsg:
m.quotas, m.quotasErr = msg.rows, msg.err
case evalSuitesMsg:
return m.onEvalSuites(msg)
case evalPollMsg:
return m.onEvalPoll(msg)
case actionDoneMsg:
m.busy = ""
m.busy, m.busyNote, m.evalWatch = "", "", ""
m.dialog, m.dialogTitle = "error", msg.title
if msg.err != nil {
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
@@ -367,6 +377,9 @@ func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if m.busy != "" {
if m.evalWatch != "" && k.Type == tea.KeyEsc {
m.busy, m.busyNote, m.evalWatch = "", "", ""
}
return m, nil
}
if m.dialog != "" {
@@ -536,6 +549,10 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case cOrgs:
m.createOrgForm()
}
case "i":
if m.menu == cMirror {
m.mirrorImportOllamaForm()
}
case "n":
if m.menu == cMonitor {
return m, m.runCLI("Test the webhooks", "ctl", "--server="+m.client.base, "notify", "test")
@@ -973,7 +990,11 @@ func (m consoleModel) View() string {
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
f.DialogTitle, f.Dialog = t, d
} else if m.busy != "" {
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
note := "asking the control plane…"
if m.busyNote != "" {
note = m.busyNote
}
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" "+note
} else if t, d := m.actionDialog(); d != "" {
f.DialogTitle, f.Dialog = t, d
}
@@ -1080,7 +1101,7 @@ func (m consoleModel) panel() (title, body, keys string) {
for _, r := range m.data.replicas {
var nodes []string
for _, s := range r.Shards {
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
}
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
@@ -1224,14 +1245,31 @@ func (m consoleModel) gpuRows() [][]string {
for _, n := range m.data.nodes {
for _, g := range n.GPUs {
busy, mem, temp, power := "—", "—", "—", "—"
// A MIG instance under its card: its own memory, the card's heat
// and power, and no utilisation, which no driver measures per
// instance — said, rather than shown as the card's or as zero.
mig := g.MIGProfile != ""
if mig {
busy = sDim.Render("n/a on MIG")
if g.MemoryTotalMB > 0 {
mem = fmt.Sprintf("%.1f/%.0f GB", float64(g.MemoryUsedMB)/1024, float64(g.MemoryTotalMB)/1024)
}
}
card := g.Card
if g.Label == "" {
// A control plane older than MIG, which sends no card.
card = g.Index
}
if n.Load != nil {
for _, s := range n.Load.GPUs {
if s.Index != g.Index {
if s.Index != card {
continue
}
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
if s.MemoryTotalMB > 0 {
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
if !mig {
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
if s.MemoryTotalMB > 0 {
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
}
}
if s.TempC > 0 {
temp = fmt.Sprintf("%.0f°C", s.TempC)
@@ -1245,7 +1283,12 @@ func (m consoleModel) gpuRows() [][]string {
if holder == "" {
holder = sDim.Render("free")
}
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
label := g.Label
if label == "" {
// A control plane older than labels.
label = strconv.Itoa(g.Index)
}
rows = append(rows, []string{n.ID, label, g.Name, busy, mem, temp, power, holder})
}
}
return rows
@@ -1266,6 +1309,15 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
}
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
kv("replicas", strconv.Itoa(n.Replicas))
// By class, which on a node with a card in MIG mode is the instances'
// profiles beside any whole card: what a profile's gpu_class can name.
for _, c := range n.ClassGPUs {
what := fmt.Sprintf("%d, %d free", c.Total, c.Free)
if c.MIG {
what += " · MIG instances, one to a replica"
}
kv(c.Class, what)
}
if len(n.Provision) > 0 {
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
for _, p := range n.Provision {
+6 -50
View File
@@ -2,7 +2,6 @@ package cli
import (
"fmt"
"io/fs"
"sort"
"strings"
@@ -24,60 +23,17 @@ type docTopic struct {
src func() string
}
// docOrder is the documents in the order someone new reads them.
var docOrder = []string{"console", "quick-install", "install", "node-onboarding", "onboarding",
"guardrails", "choosing-models", "fabrics", "facility-aware-placement", "data", "reference", "development"}
// docTopics is the manual: the documents in llmgw.Manual's order, which the
// gateway's docs portal shares, with a page for the commands after the
// configuration reference — the one page only the command tree can write.
func docTopics() []docTopic {
read := func(path string) func() string {
return func() string {
b, err := fs.ReadFile(llmgw.Docs, path)
if err != nil {
return "# " + path + "\n\nNot in this build."
}
return string(b)
}
}
titleOf := func(path string) string {
b, _ := fs.ReadFile(llmgw.Docs, path)
for _, l := range strings.Split(string(b), "\n") {
if strings.HasPrefix(l, "# ") {
return strings.TrimPrefix(l, "# ")
}
}
return path
}
var out []docTopic
seen := map[string]bool{}
add := func(key, path string) {
if seen[key] {
return
}
if _, err := fs.Stat(llmgw.Docs, path); err != nil {
return
}
seen[key] = true
out = append(out, docTopic{key: key, title: titleOf(path), src: read(path)})
}
for _, k := range docOrder {
add(k, "docs/"+k+".md")
if k == "reference" {
for _, d := range llmgw.Manual() {
out = append(out, docTopic{key: d.Key, title: d.Title, src: d.Source})
if d.Key == "reference" {
out = append(out, docTopic{key: "commands", title: "Commands", src: commandsManual})
seen["commands"] = true
}
}
// Any document not placed above, so a new one is never missing.
entries, _ := fs.ReadDir(llmgw.Docs, "docs")
var rest []string
for _, e := range entries {
rest = append(rest, strings.TrimSuffix(e.Name(), ".md"))
}
sort.Strings(rest)
for _, k := range rest {
add(k, "docs/"+k+".md")
}
add("readme", "README.md")
add("changes", "CHANGELOG.md")
return out
}
+34 -2
View File
@@ -1,6 +1,7 @@
package cli
import (
"bytes"
"context"
"errors"
"fmt"
@@ -178,10 +179,41 @@ func (m *consoleModel) mirrorFetchForm() {
}})
}
// mirrorImportOllamaForm puts a model from an Ollama on this machine into a
// mirror on this machine: the control plane's own, when the mirror runs
// beside it. A mirror elsewhere cannot read this machine's files, and its
// API is given none; there it is `llmgw mirror import-ollama`, on the mirror.
func (m *consoleModel) mirrorImportOllamaForm() {
m.openForm(&formDialog{title: "Import from Ollama",
note: "From an Ollama on this machine into the mirror's directory on this machine.\n" +
"A mirror elsewhere: run llmgw mirror import-ollama there.",
fields: []formField{
textF("model", "", "as Ollama names it, like llama3.2:3b"),
textF("ollama dir", "", "optional: found where Ollama keeps it"),
textF("mirror dir", "/srv/models", "the mirror's directory on this machine"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if v[0] == "" {
return nil, fmt.Errorf("a model, as Ollama names it")
}
if st, err := os.Stat(v[2]); err != nil || !st.IsDir() {
return nil, fmt.Errorf("%s is not a directory on this machine", v[2])
}
title := "Import " + v[0]
m.busy = title
o := ollamaImport{name: v[0], ollamaDir: v[1], mirrorDir: v[2], base: m.mirror.res.Base, quiet: true}
return func() tea.Msg {
var out bytes.Buffer
err := o.run(context.Background(), &out)
return actionDoneMsg{title: title, text: strings.TrimSpace(out.String()), err: err}
}, nil
}})
}
func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ row · Enter actions · A download one · ← back · Q leave"
keys = "↑↓ row · Enter actions · A download one · I import from Ollama · ← back · Q leave"
}
if m.mirror.err != nil {
return "Mirror", "\n " + sBad.Render(m.mirror.err.Error()), keys
@@ -216,7 +248,7 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
}
if len(rows) == 0 {
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
return "Mirror", head + " " + sDim.Render("Empty. A downloads one; I imports one from Ollama."), keys
}
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
}
+39
View File
@@ -72,6 +72,8 @@ type peopleData struct {
notes map[string]int // project id → hours kept, agent notes
// requests is how each project keeps its requests and answers.
requests map[string]store.RequestLog
// cache is how each project reuses answers.
cache map[string]store.AnswerCache
// guardPolicies are server.yaml's guardrail policies, guardDefault those
// applied everywhere, and guardAt which are attached where.
guardPolicies []string
@@ -125,6 +127,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
d.guardAt, _ = s.GuardAttachments(ctx)
}
d.threads, d.notes, d.requests = map[string]int{}, map[string]int{}, map[string]store.RequestLog{}
d.cache = map[string]store.AnswerCache{}
for _, r := range d.orgs {
if r.level != "project" {
continue
@@ -138,6 +141,9 @@ func (m consoleModel) fetchPeople() tea.Cmd {
if l, err := s.RequestLogOf(ctx, r.id); err == nil {
d.requests[r.id] = l
}
if c, err := s.AnswerCacheOf(ctx, r.id); err == nil {
d.cache[r.id] = c
}
}
return consolePeopleMsg(d)
}
@@ -552,6 +558,39 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
return m.runCLI("Keep requests", append(args, "--retain-hours="+v[0], "--sample="+v[1])...), nil
}})
return nil
}},
menuItem{"Cache answers…", func(m *consoleModel) tea.Cmd {
c := m.people.cache[r.id]
ttl, which := "off", "reproducible"
if c.TTL > 0 {
ttl = c.TTL.String()
}
if c.Any {
which = "any"
}
m.openForm(&formDialog{title: "Cache answers · " + r.path,
note: "Off by default. On, the same request to the same model within the time below is\n" +
"answered again from the gateway's memory, without the model, and costs nothing.\n" +
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.",
fields: []formField{
textF("reuse for", ttl, "like 1h; off reuses none"),
choiceF("answers", []string{"reproducible", "any"}, which),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
args := []string{"keys", "cache", "--project=" + r.id}
if v[0] == "off" || v[0] == "0" {
return m.runCLI("Cache answers", append(args, "--off")...), nil
}
if d, err := time.ParseDuration(v[0]); err != nil || d < time.Second {
return nil, fmt.Errorf("a duration like 1h, or off")
}
args = append(args, "--ttl="+v[0])
if v[1] == "any" {
args = append(args, "--any")
}
return m.runCLI("Cache answers", args...), nil
}})
return nil
}})
}
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
+291
View File
@@ -0,0 +1,291 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// Quality suites from the model menu: ask a model one of the suites the
// control plane has (its eval directory's, and those built in) and watch
// its score come in; list its past runs; hold its updates to one — the same
// as `llmgw ctl eval` and `llmgw ctl model gate`.
// evalSuitesMsg is the suites the control plane can run, fetched to open
// the form that runs one (or, with gate, the one that sets a model's).
type evalSuitesMsg struct {
model adminapi.ModelView
suites []eval.SuiteInfo
gate bool
err error
}
// evalPollMsg is a run still going, to be asked about again.
type evalPollMsg struct {
title string
live eval.Live
err error
}
// gateNote says, in an update's or a promotion's question, that the new
// weights are asked the model's suite first.
func gateNote(md adminapi.ModelView) string {
g := md.Eval
if g == nil {
return ""
}
at := "its threshold"
if g.MinScore > 0 {
at = fmt.Sprintf("%.2f", g.MinScore)
}
return fmt.Sprintf("\n\nFirst the new weights are asked the quality suite %s, and must score %s;\n"+
"below it nothing moves.", g.Suite, at)
}
// evalItems are the model menu's quality entries.
func (m consoleModel) evalItems(md adminapi.ModelView) []menuItem {
gate := "Hold its updates to a quality suite…"
if g := md.Eval; g != nil {
gate = "Its updates' quality suite (" + g.Suite + ")…"
}
return []menuItem{
{"Ask it a quality suite…", func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, false) }},
{"Its quality runs", func(m *consoleModel) tea.Cmd {
return m.call("Quality runs of "+md.Name, http.MethodGet,
"/api/v1/eval/runs?limit=15&model="+url.QueryEscape(md.Name), nil, func(raw json.RawMessage) string {
var runs []eval.Live
if err := json.Unmarshal(raw, &runs); err != nil {
return string(raw)
}
return evalRunsText(runs)
})
}},
{gate, func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, true) }},
}
}
func (m *consoleModel) fetchSuites(md adminapi.ModelView, gate bool) tea.Cmd {
m.busy = "Quality suites"
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var list []eval.SuiteInfo
err := c.get(ctx, "/api/v1/eval/suites", &list)
return evalSuitesMsg{model: md, suites: list, gate: gate, err: err}
}
}
func (m consoleModel) onEvalSuites(msg evalSuitesMsg) (tea.Model, tea.Cmd) {
m.busy = ""
md := msg.model
var names []string
for _, s := range msg.suites {
if s.Error == "" {
names = append(names, s.Name)
}
}
if msg.err == nil && len(names) == 0 {
msg.err = errors.New("the control plane has no suite it can run: put one in its control.eval_dir " +
"(/etc/llmgw/eval); docs/install.md, \"Quality suites\", says how one is written")
}
if msg.err != nil {
m.dialog, m.dialogTitle = "error", "Quality suites"
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
return m, nil
}
about := func(name string) string {
for _, s := range msg.suites {
if s.Name == name {
return fmt.Sprintf("%s: %d cases, passed at %.2f, %s", s.Name, s.Cases, s.Threshold, s.From)
}
}
return ""
}
path := "/api/v1/models/" + url.PathEscape(md.Name)
if msg.gate {
current, minScore := "none", ""
if g := md.Eval; g != nil {
current = g.Suite
if g.MinScore > 0 {
minScore = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
}
if !slices.Contains(names, current) {
names = append(names, current)
}
}
m.openForm(&formDialog{title: "Quality gate of " + md.Name,
note: "An update or a promotion asks the new weights this suite first, and stops below its score;\n" +
"an update tries them as a canary given none of " + md.Name + "'s requests, and takes it out again.",
fields: []formField{
choiceF("suite", append([]string{"none"}, names...), current),
textF("min score", minScore, "0 to 1; empty is the suite's own threshold"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if v[0] == "none" {
return m.call("Quality gate of "+md.Name, http.MethodDelete, path+"/eval-gate", nil,
func(json.RawMessage) string { return md.Name + "'s updates are no longer held to a suite." }), nil
}
score, err := minScoreOf(v[1])
if err != nil {
return nil, err
}
return m.call("Quality gate of "+md.Name, http.MethodPut, path+"/eval-gate",
adminapi.EvalGateRequest{Suite: v[0], MinScore: score}, func(json.RawMessage) string {
return fmt.Sprintf("%s's new weights must pass %s before they replace the old.", md.Name, v[0])
}), nil
}})
return m, nil
}
m.openForm(&formDialog{title: "Ask " + md.Name + " a quality suite",
note: "Asked on the control plane, through the router as its own requests: past quotas and guardrails.\n" +
about(names[0]),
fields: []formField{
choiceF("suite", names, names[0]),
textF("at once", "4", "cases asked at the same time"),
textF("min score", "", "0 to 1; empty is the suite's own threshold"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
n, err := strconv.Atoi(v[1])
if err != nil || n < 1 {
return nil, fmt.Errorf("at once is a whole number, 1 or more")
}
score, err := minScoreOf(v[2])
if err != nil {
return nil, err
}
return m.startEval(md.Name, eval.Request{Suite: v[0], Concurrency: n, MinScore: score}), nil
}})
return m, nil
}
func minScoreOf(s string) (float64, error) {
if s == "" {
return 0, nil
}
f, err := strconv.ParseFloat(s, 64)
if err != nil || f < 0 || f > 1 {
return 0, fmt.Errorf("the min score is a number from 0 to 1")
}
return f, nil
}
// startEval starts a run and follows it.
func (m *consoleModel) startEval(model string, req eval.Request) tea.Cmd {
title := model + ": " + req.Suite
m.busy, m.busyNote = title, "starting…"
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var l eval.Live
err := c.do(ctx, http.MethodPost, "/api/v1/models/"+url.PathEscape(model)+"/eval", req, &l)
if err != nil {
return actionDoneMsg{title: title, err: err}
}
return evalPollMsg{title: title, live: l}
}
}
func (m consoleModel) onEvalPoll(msg evalPollMsg) (tea.Model, tea.Cmd) {
if m.evalWatch != "" && m.evalWatch != msg.live.ID {
return m, nil
}
if m.evalWatch == "" && m.busy != msg.title {
// Esc: no longer followed here; it carries on on the control plane.
return m, nil
}
m.evalWatch = msg.live.ID
m.busy = msg.title
m.busyNote = fmt.Sprintf("%d of %d answered · Esc stops watching (it carries on)", msg.live.Done, msg.live.Cases)
c, id, title := m.client, msg.live.ID, msg.title
return m, func() tea.Msg {
time.Sleep(time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var l eval.Live
if err := c.get(ctx, "/api/v1/eval/runs/"+url.PathEscape(id), &l); err != nil {
return actionDoneMsg{title: title, err: err}
}
if l.State == eval.Running {
return evalPollMsg{title: title, live: l}
}
var runs []eval.Live
_ = c.get(ctx, "/api/v1/eval/runs?limit=6&model="+url.QueryEscape(l.Model), &runs)
text := evalRunText(l)
if len(runs) > 1 {
text += "\n\nIts last runs:\n" + evalRunsText(runs)
}
if l.State == eval.Failed || !l.Passed {
return actionDoneMsg{title: title, err: errors.New(text)}
}
return actionDoneMsg{title: title, text: text}
}
}
// evalRunText is a finished run: its score, then each case.
func evalRunText(l eval.Live) string {
if l.State == eval.Failed {
return "The run could not be scored: " + l.Error
}
var b strings.Builder
b.WriteString(evalSummary(l) + "\n")
if l.Error != "" {
b.WriteString(l.Error + "\n")
}
b.WriteString("\n")
shown := min(len(l.Results), 14)
for _, r := range l.Results[:shown] {
mark := sOK.Render("✓")
if !r.Passed {
mark = sBad.Render("✗")
}
why := r.Detail
if r.Error != "" {
why = r.Error
}
fmt.Fprintf(&b, "%s %-18s %4.2f %s\n", mark, truncate(r.ID, 18), r.Score, truncate(why, 60))
}
if len(l.Results) > shown {
fmt.Fprintf(&b, " and %d more: llmgw ctl eval --run %s\n", len(l.Results)-shown, l.ID)
}
return strings.TrimRight(b.String(), "\n")
}
// evalRunsText is a model's runs, one a line.
func evalRunsText(runs []eval.Live) string {
if len(runs) == 0 {
return "No runs kept yet: \"Ask it a quality suite\" on this menu makes one."
}
var b strings.Builder
for _, r := range runs {
result := sOK.Render("passed")
switch {
case r.State == eval.Running:
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
case r.State == eval.Failed:
result = sBad.Render("could not run")
case !r.Passed:
result = sBad.Render("failed")
}
who := r.Model
if r.Target != "" {
who = r.Target
}
fmt.Fprintf(&b, "%-9s %-14s %-16s %4.2f of %4.2f %s\n", ageShort(r.At), truncate(r.Suite, 14),
truncate(who, 16), r.Score, r.Threshold, result)
}
return strings.TrimRight(b.String(), "\n")
}
+116
View File
@@ -0,0 +1,116 @@
package cli
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestAModelIsAskedASuiteFromItsMenuAndItsScoreShown(t *testing.T) {
var mu sync.Mutex
var asked eval.Request
polls := 0
done := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Score: 0.75,
Threshold: 0.7, Passed: true, Cases: 2, Below: 1, Results: []store.EvalCase{
{ID: "add", Kind: "number", Score: 1, Passed: true, Detail: "read 42"},
{ID: "mul", Kind: "number", Score: 0, Detail: "read 43, wanted 42"}}}, State: eval.Done, Done: 2}
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
switch {
case r.URL.Path == "/api/v1/eval/suites":
_ = json.NewEncoder(w).Encode([]eval.SuiteInfo{
{Name: "arithmetic", From: "built in", Cases: 2, Threshold: 0.7},
{Name: "broken", From: "/etc/llmgw/eval/broken.yaml", Error: "no cases"},
{Name: "json-format", From: "built in", Cases: 7, Threshold: 0.7}})
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/qwen/eval":
b, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(b, &asked)
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
Suite: "arithmetic", Cases: 2}, State: eval.Running})
case r.URL.Path == "/api/v1/eval/runs/eval-1":
polls++
if polls == 1 {
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
Suite: "arithmetic", Cases: 2}, State: eval.Running, Done: 1})
return
}
_ = json.NewEncoder(w).Encode(done)
case r.URL.Path == "/api/v1/eval/runs":
older := done
older.ID, older.Score, older.Passed = "eval-0", 0.5, false
_ = json.NewEncoder(w).Encode([]eval.Live{done, older})
default:
http.NotFound(w, r)
}
}))
t.Cleanup(admin.Close)
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, cModels, true
md := adminapi.ModelView{Name: "qwen"}
// run plays a command and what it leads to, the way the program would.
run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
for cmd != nil {
msg := cmd()
tm, cmd = tm.Update(msg)
if _, ok := msg.(actionDoneMsg); ok {
break
}
}
return tm
}
var tm tea.Model = m
items := m.evalItems(md)
if len(items) != 3 || !strings.Contains(items[0].label, "quality suite") {
t.Fatalf("%+v", items)
}
tm = run(tm, items[0].act(&m))
f := tm.(consoleModel).formD
if f == nil || f.fields[0].options == nil || len(f.fields[0].options) != 2 {
t.Fatalf("no form offering the suites that can run: %+v", f)
}
tm = press(tm, "tab")
f.fields[1].text.SetValue("2")
tm = press(tm, "tab", "tab")
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
if busy := tm.(consoleModel).busy; busy != "qwen: arithmetic" {
t.Fatalf("busy %q", busy)
}
// Started, then followed: one poll still running, then its score.
tm, next := tm.Update(cmd())
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "0 of 2 answered") {
t.Errorf("progress %q", note)
}
tm, next = tm.Update(next())
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "1 of 2 answered") {
t.Errorf("progress %q", note)
}
tm, _ = tm.Update(next())
view := tm.View()
mu.Lock()
sent := asked
mu.Unlock()
if sent.Suite != "arithmetic" || sent.Concurrency != 2 {
t.Errorf("asked %+v", sent)
}
for _, want := range []string{"qwen scored 0.75 on arithmetic", "1 of 2 cases passed", "read 43, wanted 42", "Its last runs"} {
if !strings.Contains(view, want) {
t.Errorf("the result does not say %q:\n%s", want, view)
}
}
if c := tm.(consoleModel); c.busy != "" || c.evalWatch != "" {
t.Errorf("still busy: %q %q", c.busy, c.evalWatch)
}
}
+35 -18
View File
@@ -20,6 +20,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -94,10 +95,11 @@ type Server struct {
// Monitoring is where the dashboard reads its history from.
Monitoring Monitoring `koanf:"monitoring"`
// Tracing sends a trace of each inference request to an OpenTelemetry
// collector. Off unless an endpoint is set here or in
// Tracing sends traces to an OpenTelemetry collector: each inference
// request, and on a control plane each placement, replica start and
// model job. Off unless an endpoint is set here or in
// OTEL_EXPORTER_OTLP_ENDPOINT.
Tracing Tracing `koanf:"tracing"`
Tracing obs.Tracing `koanf:"tracing"`
// Mirror points the control plane at a model mirror it can administer.
// Empty means there is none, and /admin/mirror stays unmounted.
@@ -197,21 +199,6 @@ type Monitoring struct {
Timeout time.Duration `koanf:"timeout"`
}
// Tracing is where request traces go, over OTLP/HTTP.
//
// One span per inference request, with admission, scheduling and each attempt
// at an engine beneath it, and the trace carried on to the engine in a
// traceparent header. The request's X-Request-Id is on the span, so a trace
// and `llmgw ctl request` name the same request.
type Tracing struct {
// Endpoint is the collector, for example http://otel.internal:4318.
// Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.
Endpoint string `koanf:"endpoint"`
// SampleRatio is the share of requests traced, 0 to 1, when the caller
// did not decide it with a sampled traceparent of its own. Zero takes 1.
SampleRatio float64 `koanf:"sample_ratio"`
}
// Mirror is a model mirror this control plane administers over its API.
//
// The mirror is a separate machine with a large disk, not part of the gateway,
@@ -265,6 +252,13 @@ type Landing struct {
// a language with the switcher at the foot of every page and whose
// browser asks for none it speaks: "en" or "tr". Empty is English.
Language string `koanf:"language"`
// DocsSignIn serves the docs portal at /docs only to someone signed in.
// Off, the portal is public like this page: its client guide is for the
// person who has no account yet, and the operator documents describe
// llmgw in general — no address, model or key of this installation —
// which the project's repository publishes anyway. On, a visitor is sent
// to sign in, and without accounts the portal needs an API key.
DocsSignIn bool `koanf:"docs_sign_in"`
}
// Threads configures conversation and memory storage for a gateway without
@@ -396,6 +390,11 @@ type Control struct {
// Timezone is where models' scaling schedules are read, an IANA name
// like Europe/Istanbul; empty is the control plane's own.
Timezone string `koanf:"timezone"`
// EvalDir is where quality suites are read from by name, for `llmgw ctl
// eval MODEL --suite NAME`, a model's eval gate and the console. The
// suites the binary carries (arithmetic, json-format) are found when it
// has none of the name; a suite here of the same name replaces one.
EvalDir string `koanf:"eval_dir"`
// Enabled runs the control plane even when configuration declares no
// models.
@@ -480,6 +479,11 @@ type Router struct {
// another. Needs the store.
Batch Batch `koanf:"batch"`
// Cache bounds the answers kept for the projects that reuse them
// (llmgw keys cache). Which projects do, and for how long, is theirs
// to set; this is only how much memory it may take.
Cache Cache `koanf:"cache"`
// TrustedProxies are the reverse proxies in front of the client port,
// as addresses or CIDR prefixes. Their X-Forwarded-For, -Proto and -Host
// are believed; anyone else's are not, since they decide the address
@@ -613,6 +617,18 @@ type Batch struct {
RetainDays int `koanf:"retain_days"`
}
// Cache is how much of this process's memory reused answers may take. The
// answers are held here and nowhere else: each gateway process keeps its
// own, and a restart begins them again.
type Cache struct {
// MaxBytes bounds every project's answers together; 128 MB unless set.
// Past it, the answers used longest ago go first.
MaxBytes int64 `koanf:"max_bytes"`
// MaxEntryBytes bounds one answer; a longer one is not kept. 1 MB
// unless set.
MaxEntryBytes int64 `koanf:"max_entry_bytes"`
}
// Default returns the configuration used when nothing is specified.
func Default() Server {
return Server{
@@ -629,6 +645,7 @@ func Default() Server {
LostAfter: 60 * time.Second,
PortRange: []int{20000, 20999},
MgmtInterface: "eth0",
EvalDir: "/etc/llmgw/eval",
},
Router: Router{
ScrapeInterval: 2 * time.Second,
+119 -29
View File
@@ -28,6 +28,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
@@ -43,6 +44,8 @@ import (
type API struct {
// replay sends a kept request again; nil without a router here.
replay Replayer
// evals runs quality suites against models; nil without a router here.
evals *eval.Service
// notifier is the webhooks `ctl notify test` reaches; nil without any.
notifier *notify.Notifier
// releases is where release binaries are fetched from, for upgrades.
@@ -75,13 +78,26 @@ func (a *API) SetRevoker(f func(ctx context.Context, nodeID string, at time.Time
a.revoke = f
}
// Mux is where the endpoints are mounted. An *http.ServeMux is one; a test
// that records the patterns is another, and is how the route table is checked
// against what is actually served.
type Mux interface {
HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request))
}
// Register mounts the endpoints on a mux under /api/v1.
func (a *API) Register(mux *http.ServeMux) {
//
// Every pattern registered here, or by the register helpers it calls, must
// also be in Routes: that table is what the OpenAPI document is generated
// from, and a test fails when the two disagree.
func (a *API) Register(mux Mux) {
mux.HandleFunc("GET /api/v1/summary", a.summary)
// Adding a model is a write, and the only one this API has. It lives here
// rather than on the client-facing listener for the same reason the rest of
// it does: reaching this port already means holding the control plane.
mux.HandleFunc("GET /api/v1/openapi.json", serveOpenAPI)
// Changes to the catalog live here rather than on the client-facing
// listener for the same reason the rest of this does: reaching this port
// already means holding the control plane.
mux.HandleFunc("POST /api/v1/models", a.addModel)
mux.HandleFunc("GET /api/v1/models/{name}", a.getModel)
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
@@ -97,9 +113,7 @@ func (a *API) Register(mux *http.ServeMux) {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req struct {
Class string `json:"class"`
}
var req ClassRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
@@ -109,7 +123,7 @@ func (a *API) Register(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "class": m.EffectiveClass()})
writeJSON(w, http.StatusOK, ClassView{Model: m.Name, Class: m.EffectiveClass()})
})
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
@@ -157,6 +171,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
a.registerCanary(mux)
a.registerEval(mux)
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
mux.HandleFunc("GET /api/v1/plan", a.planModel)
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
@@ -228,7 +243,12 @@ type NodeView struct {
DiskFreeGB int `json:"disk_free_gb,omitempty"`
DiskTotalGB int `json:"disk_total_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// ClassGPUs counts the node's devices by class, in the order of Classes.
// A node with a card in MIG mode has several classes — each profile its
// instances are cut to, and any whole card beside them — and GPUTotal
// alone would add 3g.40gb instances to 80 GB cards.
ClassGPUs []ClassGPUs `json:"gpus_by_class,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -255,10 +275,34 @@ type NodeView struct {
IMEXChannels int `json:"imex_channels,omitempty"`
}
// ClassGPUs is how many of a node's devices are of one class, and how many
// of those are free or kept out of placement.
type ClassGPUs struct {
Class string `json:"class"`
Total int `json:"total"`
Free int `json:"free"`
Unhealthy int `json:"unhealthy,omitempty"`
// MIG marks a class of MIG instances, each of which takes a replica of
// its own and never joins a tensor-parallel group.
MIG bool `json:"mig,omitempty"`
}
// GPUView is one card as a node's page shows it.
type GPUView struct {
Index int `json:"index"`
Name string `json:"name"`
// Label is the device as people are shown it: "3" for a whole card,
// "0/mig2" for card 0's MIG instance 2.
Label string `json:"label"`
Class string `json:"class"`
// Card is the physical card's index: the device's own, or for a MIG
// instance its parent's, which is what the card's sensors are under.
Card int `json:"card"`
// MIGProfile is the instance's profile, like 3g.40gb; empty for a
// whole card.
MIGProfile string `json:"mig_profile,omitempty"`
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
MemoryUsedMB int `json:"memory_used_mb,omitempty"`
// ComputeCapability is what decides which engine builds run on the card:
// CUDA 13 builds need 7.5, so a V100 (7.0) runs only older ones.
ComputeCapability string `json:"compute_capability,omitempty"`
@@ -339,6 +383,8 @@ type ModelView struct {
// set on the candidate, naming the model it is a canary of.
Canary *catalog.Canary `json:"canary,omitempty"`
CanaryOf string `json:"canary_of,omitempty"`
// Eval is the quality suite its new weights must pass, when it has one.
Eval *catalog.EvalGate `json:"eval,omitempty"`
// Description is what the model is for, when somebody said.
Description string `json:"description,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -556,15 +602,25 @@ func (a *API) Nodes() []NodeView {
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
seen := map[string]int{}
free := inv.FreeGPUs(n.ID)
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
seen[g.Class()] = struct{}{}
k, dup := seen[g.Class()]
if !dup {
k = len(v.ClassGPUs)
seen[g.Class()] = k
v.Classes = append(v.Classes, g.Class())
v.ClassGPUs = append(v.ClassGPUs, ClassGPUs{Class: g.Class(), MIG: g.IsMIG()})
}
if !g.Health.OK() {
c := &v.ClassGPUs[k]
c.Total++
switch {
case !g.Health.OK():
c.Unhealthy++
v.Unhealthy = append(v.Unhealthy,
fmt.Sprintf("gpu %d: %s", g.Index, g.Health.Problems[0]))
fmt.Sprintf("gpu %s: %s", discovery.DeviceLabel(g.Index), g.Health.Problems[0]))
case slices.Contains(free, g.Index):
c.Free++
}
}
if !n.LastSeen.IsZero() {
@@ -574,7 +630,12 @@ func (a *API) Nodes() []NodeView {
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
for _, g := range n.GPUs {
gv := GPUView{Index: g.Index, Name: g.Name, ComputeCapability: g.ComputeCapability,
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers,
Label: discovery.DeviceLabel(g.Index), Class: g.Class(), Card: g.CardIndex(),
MemoryTotalMB: g.MemoryTotalMB, MemoryUsedMB: g.MemoryUsedMB}
if g.MIG != nil {
gv.MIGProfile = g.MIG.Profile
}
for hca, rel := range g.NICPaths {
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
@@ -660,7 +721,7 @@ func (a *API) Models() []ModelView {
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m), Adapters: m.Adapters,
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name),
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -953,7 +1014,25 @@ func (a *API) addModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"added": m.Name})
writeJSON(w, http.StatusOK, ModelAdded{Added: m.Name})
}
// getModel answers with one model's whole definition as it is in force: what
// was added, with the defaults the catalog filled in. The listing shows a
// model's state and only the names of its profiles; this is what a tool
// managing the definition — Terraform, say — compares against what it wants.
//
// By its name only, not an alias: an alias answering would let a tool believe
// a model it manages exists when it is another one.
func (a *API) getModel(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
for _, m := range a.ctl.Catalog().Models() {
if m.Name == name {
writeJSON(w, http.StatusOK, m)
return
}
}
writeError(w, http.StatusNotFound, "no model named "+name+" is in the catalog")
}
// ScaleRequest is how many replicas a model should run.
@@ -1039,7 +1118,7 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"removed": name})
writeJSON(w, http.StatusOK, Removed{Removed: name})
}
func (a *API) summary(w http.ResponseWriter, _ *http.Request) {
@@ -1137,15 +1216,13 @@ func (a *API) removeNode(w http.ResponseWriter, r *http.Request) {
}
inv.Remove(id)
a.ctl.Reconciler().Notify()
writeJSON(w, http.StatusOK, map[string]string{"id": id, "removed": "true"})
writeJSON(w, http.StatusOK, NodeRemoved{ID: id, Removed: "true"})
}
func (a *API) setNodeState(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
var body struct {
State string `json:"state"`
}
var body NodeStateRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
@@ -1173,7 +1250,7 @@ func (a *API) setNodeState(w http.ResponseWriter, r *http.Request) {
// is usually about to do something to it.
a.ctl.Reconciler().Notify()
writeJSON(w, http.StatusOK, map[string]string{"id": id, "state": st.String()})
writeJSON(w, http.StatusOK, NodeStateView{ID: id, State: st.String()})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
@@ -1183,7 +1260,7 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
}
func writeError(w http.ResponseWriter, code int, msg string) {
writeJSON(w, code, map[string]string{"error": msg})
writeJSON(w, code, ErrorView{Error: msg})
}
// profileChanged reports whether a replica is running a profile the catalog no
@@ -1384,7 +1461,7 @@ func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
return
}
was := a.ctl.Reconciler().Retry(name)
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
writeJSON(w, http.StatusOK, RetryView{WasParked: was})
}
// requestFinder looks a request up by the id it was given. Asked of the usage
@@ -1419,6 +1496,8 @@ type RequestView struct {
// What the client said about it: OpenAI's user field, and its tags.
EndUser string `json:"end_user,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
// CacheHit is an answer reused from the project's answer cache.
CacheHit bool `json:"cache_hit,omitempty"`
}
func (a *API) request(w http.ResponseWriter, r *http.Request) {
@@ -1442,7 +1521,7 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardMs: u.GuardMs,
EndUser: u.EndUser, Tags: u.Tags,
EndUser: u.EndUser, Tags: u.Tags, CacheHit: u.CacheHit,
})
}
@@ -1477,7 +1556,7 @@ func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadGateway, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
writeJSON(w, http.StatusOK, UpgradeView{Node: r.PathValue("id"), Version: req.Version})
}
// SetServe supplies what serves a model from the registry.
@@ -1559,7 +1638,11 @@ func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
@@ -1577,7 +1660,7 @@ func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
writeJSON(w, http.StatusOK, Result{Result: msg})
}
// OriginRequest names the repository a model's weights came from.
@@ -1617,6 +1700,13 @@ func (a *API) PoolNodes() []modelplan.PoolNode {
p.Class = n.Classes[0]
p.GPUMemGB = modelplan.MemFromClass(p.Class)
}
// By class, with each class's own count of devices: a node with a
// card in MIG mode is several, and its first class alone would
// count every instance and card as that one.
for _, c := range n.ClassGPUs {
p.Kinds = append(p.Kinds, modelplan.PoolKind{Class: c.Class,
GPUMemGB: modelplan.MemFromClass(c.Class), GPUs: c.Total, FreeGPUs: c.Free})
}
if len(n.Fabrics) > 0 {
p.Fabric = n.Fabrics[0]
}
+15 -7
View File
@@ -8,7 +8,13 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func (a *API) registerCanary(mux *http.ServeMux) {
// CanaryPercentRequest changes the share of a model's requests its canary
// takes.
type CanaryPercentRequest struct {
Percent int `json:"percent"`
}
func (a *API) registerCanary(mux Mux) {
need := func(w http.ResponseWriter) bool {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "a canary needs a mirror and a store, and this control plane lacks one")
@@ -45,9 +51,7 @@ func (a *API) registerCanary(mux *http.ServeMux) {
if !need(w) {
return
}
var req struct {
Percent int `json:"percent"`
}
var req CanaryPercentRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
@@ -57,7 +61,7 @@ func (a *API) registerCanary(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
writeJSON(w, http.StatusOK, Result{Result: msg})
})
mux.HandleFunc("DELETE /api/v1/models/{name}/canary", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
@@ -68,13 +72,17 @@ func (a *API) registerCanary(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
writeJSON(w, http.StatusOK, Result{Result: msg})
})
mux.HandleFunc("POST /api/v1/models/{name}/promote", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r))
ch, ok := readChange(w, r)
if !ok {
return
}
job, err := a.serve.StartPromote(r.PathValue("name"), actorOf(r), ch)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+186
View File
@@ -0,0 +1,186 @@
package adminapi
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Quality suites: run against a model on the control plane, through the
// router as the gateway's own requests (like a replay), followed while they
// run, and kept; and a model's standing suite that its updates must pass.
// SetEval gives the API the service that runs quality suites.
func (a *API) SetEval(s *eval.Service) { a.evals = s }
// maxSuiteBytes bounds a suite sent whole: a few hundred cases with their
// prompts, which is more than a gate should take to run.
const maxSuiteBytes = 4 << 20
// EvalGateRequest sets a model's standing quality suite; an empty suite
// takes it away.
type EvalGateRequest struct {
Suite string `json:"suite"`
MinScore float64 `json:"min_score,omitempty"`
Judge string `json:"judge,omitempty"`
}
type evalGater interface {
SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error)
}
// readChange reads how an update or a promotion is held to quality; an
// empty body is no change to how.
func readChange(w http.ResponseWriter, r *http.Request) (modelserve.Change, bool) {
var ch modelserve.Change
b, err := io.ReadAll(io.LimitReader(r.Body, maxSuiteBytes+1))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return ch, false
}
if len(b) > maxSuiteBytes {
writeError(w, http.StatusRequestEntityTooLarge, "the suite is over 4 MiB")
return ch, false
}
if len(strings.TrimSpace(string(b))) == 0 {
return ch, true
}
if err := json.Unmarshal(b, &ch); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return ch, false
}
return ch, true
}
func (a *API) registerEval(mux Mux) {
need := func(w http.ResponseWriter) bool {
if a.evals == nil {
writeError(w, http.StatusNotImplemented, "this control plane does not route requests, so it cannot ask a model a suite")
return false
}
return true
}
mux.HandleFunc("GET /api/v1/eval/suites", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
out := a.evals.Suites()
if out == nil {
out = []eval.SuiteInfo{}
}
writeJSON(w, http.StatusOK, out)
})
// Started and answered at once: a suite against a model on CPUs takes
// minutes, longer than a client waits on one request.
mux.HandleFunc("POST /api/v1/models/{name}/eval", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
var req eval.Request
if err := json.NewDecoder(io.LimitReader(r.Body, maxSuiteBytes)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
name := r.PathValue("name")
if a.ctl != nil {
// By its own name, so an alias's runs are filed with the rest.
m, ok := a.ctl.Catalog().Resolve(name)
if !ok {
writeError(w, http.StatusNotFound, "no model called "+name+": `llmgw ctl models` lists them")
return
}
name = m.Name
}
live, err := a.evals.Start(name, req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, live)
})
mux.HandleFunc("GET /api/v1/eval/runs", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
out, err := a.evals.History(r.Context(), r.URL.Query().Get("model"), limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if out == nil {
out = []eval.Live{}
}
writeJSON(w, http.StatusOK, out)
})
mux.HandleFunc("GET /api/v1/eval/runs/{id}", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
live, err := a.evals.Get(r.Context(), r.PathValue("id"))
if errors.Is(err, store.ErrNotFound) {
writeError(w, http.StatusNotFound, "no eval run "+r.PathValue("id"))
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, live)
})
setGate := func(w http.ResponseWriter, r *http.Request, g *catalog.EvalGate) {
gater, ok := a.models.(evalGater)
if !ok {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
return
}
name := r.PathValue("name")
if g != nil && a.evals != nil {
if err := a.evals.Check(name, eval.Request{Suite: g.Suite, Judge: g.Judge, MinScore: g.MinScore}); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
}
m, err := gater.SetEval(r.Context(), name, g, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, EvalGateView{Model: m.Name, Eval: m.Eval})
}
mux.HandleFunc("PUT /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
var req EvalGateRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
if strings.TrimSpace(req.Suite) == "" {
writeError(w, http.StatusBadRequest, "name a suite: one in the control plane's eval directory, or built in")
return
}
if req.MinScore < 0 || req.MinScore > 1 {
writeError(w, http.StatusBadRequest, "the minimum score is 0 to 1")
return
}
setGate(w, r, &catalog.EvalGate{Suite: strings.TrimSpace(req.Suite), MinScore: req.MinScore, Judge: req.Judge})
})
mux.HandleFunc("DELETE /api/v1/models/{name}/eval-gate", func(w http.ResponseWriter, r *http.Request) {
setGate(w, r, nil)
})
}
// EvalGateView is a model's standing quality gate, as it is now; Eval is
// absent when it has none.
type EvalGateView struct {
Model string `json:"model"`
Eval *catalog.EvalGate `json:"eval,omitempty"`
}
+84
View File
@@ -0,0 +1,84 @@
package adminapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"testing/fstest"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// gatingModels is the catalog's service, keeping the gate it is given.
type gatingModels struct {
*fakeModels
gate *catalog.EvalGate
set bool
}
func (g *gatingModels) SetEval(_ context.Context, name string, gate *catalog.EvalGate, _ string) (catalog.Model, error) {
g.gate, g.set = gate, true
return catalog.Model{Name: name, Eval: gate}, nil
}
func TestASuiteIsStartedByAModelsAliasAndAGateNeedsASuiteThatIsThere(t *testing.T) {
a, _ := setup(t, []catalog.Model{{Name: "qwen", Aliases: []string{"q"}, Source: "/m.gguf",
Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}})
answer := func(_ context.Context, _, _ string, _ []byte) (int, []byte, time.Duration, error) {
return 200, []byte(`{"choices":[{"message":{"content":"42"}}]}`), 0, nil
}
a.SetEval(eval.NewService(eval.Deps{Call: answer, Log: slog.New(slog.DiscardHandler),
Builtin: fstest.MapFS{"arith.yaml": {Data: []byte("cases:\n - prompt: 40 + 2?\n number: 42\n")}}}))
models := &gatingModels{fakeModels: &fakeModels{}}
a.SetModels(models)
mux := http.NewServeMux()
a.Register(mux)
do := func(method, path, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
return rec
}
rec := do(http.MethodPost, "/api/v1/models/q/eval", `{"suite": "arith"}`)
var live eval.Live
if rec.Code != http.StatusAccepted || json.Unmarshal(rec.Body.Bytes(), &live) != nil || live.Model != "qwen" {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/nobody/eval", `{"suite": "arith"}`); rec.Code != http.StatusNotFound {
t.Errorf("a model that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPost, "/api/v1/models/qwen/eval", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest ||
!strings.Contains(rec.Body.String(), "--suites") {
t.Errorf("a suite that is not there: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodGet, "/api/v1/eval/suites", ""); !strings.Contains(rec.Body.String(), `"name":"arith"`) {
t.Errorf("suites: %s", rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "nope"}`); rec.Code != http.StatusBadRequest || models.set {
t.Errorf("a gate on a suite that is not there was set: %d %s", rec.Code, rec.Body.String())
}
if rec := do(http.MethodPut, "/api/v1/models/qwen/eval-gate", `{"suite": "arith", "min_score": 0.9}`); rec.Code != http.StatusOK ||
models.gate == nil || models.gate.Suite != "arith" || models.gate.MinScore != 0.9 {
t.Errorf("%d %s %+v", rec.Code, rec.Body.String(), models.gate)
}
if rec := do(http.MethodDelete, "/api/v1/models/qwen/eval-gate", ""); rec.Code != http.StatusOK || models.gate != nil {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
deadline := time.Now().Add(5 * time.Second)
for live.State == eval.Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
_ = json.Unmarshal(do(http.MethodGet, "/api/v1/eval/runs/"+live.ID, "").Body.Bytes(), &live)
}
if live.State != eval.Done || live.Score != 1 || len(live.Results) != 1 {
t.Fatalf("%+v", live)
}
}
+1 -4
View File
@@ -34,10 +34,7 @@ type RailView struct {
// sends traffic over the fabric for seconds a rail, so it is a POST and only
// ever run when asked.
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
var body struct {
A, B, Fabric string
Seconds int
}
var body FabricTestRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+12 -7
View File
@@ -66,7 +66,17 @@ func (a *API) guardTarget(ctx context.Context, target string) (guard.Attachment,
return guard.Attachment{}, fmt.Errorf("%q: attach to org, team, project, key or user", kind)
}
func (a *API) registerGuard(mux *http.ServeMux) {
// GuardAttachmentRequest attaches a policy to a part of the tenancy tree, a
// key or a person, or with DELETE takes it off again.
type GuardAttachmentRequest struct {
// Target is org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT (or the ID
// after the colon), key:KEY-ID or user:USER-ID.
Target string `json:"target"`
// Policy is a policy's name in server.yaml's guard.policies.
Policy string `json:"policy"`
}
func (a *API) registerGuard(mux Mux) {
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
if a.guard == nil {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard: in server.yaml")
@@ -97,12 +107,7 @@ func (a *API) registerGuard(mux *http.ServeMux) {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard: in server.yaml")
return
}
var at struct {
// Target is org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT
// (or the ID after the colon), key:KEY-ID or user:USER-ID.
Target string `json:"target"`
Policy string `json:"policy"`
}
var at GuardAttachmentRequest
if err := json.NewDecoder(r.Body).Decode(&at); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
+65
View File
@@ -0,0 +1,65 @@
package adminapi
import (
"reflect"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// migNode has card 0 cut into two 3g.40gb instances and card 1 whole.
func migNode(id string) inventory.Node {
nd := inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", MemoryTotalMB: 512 * 1024}
for dev := range 2 {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: discovery.MIGIndex(0, dev), Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", UUID: "MIG-x",
MemoryTotalMB: 40192, MemoryUsedMB: 37,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"},
})
}
nd.GPUs = append(nd.GPUs, discovery.GPU{Index: 1, Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920})
return nd
}
func TestANodeWithACardInMIGModeCountsItsDevicesByClass(t *testing.T) {
a, ctl := setup(t, testModels(), migNode("a100"))
ctl.Inventory().Allocate("a100", inventory.Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{1000}})
n := a.Nodes()[0]
want := []ClassGPUs{
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 1, MIG: true},
{Class: "a100-80g", Total: 1, Free: 1},
}
if !reflect.DeepEqual(n.ClassGPUs, want) {
t.Errorf("by class: %+v\nwant %+v", n.ClassGPUs, want)
}
// Each device as people are shown it: the instance under its card, with
// its own memory, and the card its sensors are read under.
g := n.GPUs[1]
if g.Label != "0/mig1" || g.Card != 0 || g.MIGProfile != "3g.40gb" || g.MemoryTotalMB != 40192 ||
g.Class != "a100-80g-mig-3g.40gb" {
t.Errorf("instance view: %+v", g)
}
if w := n.GPUs[2]; w.Label != "1" || w.Card != 1 || w.MIGProfile != "" {
t.Errorf("whole card view: %+v", w)
}
// The fleet summary attributes each device to its own class, where it
// once put every card of a node under the node's only class.
s := a.Summary()
got := map[string][2]int{}
for _, c := range s.GPUs {
got[c.Class] = [2]int{c.Total, c.Used}
}
if !reflect.DeepEqual(got, map[string][2]int{"a100-80g-mig-3g.40gb": {2, 1}, "a100-80g": {1, 0}}) {
t.Errorf("summary by class: %v", got)
}
// And a plan sees both classes, each with its own count.
pool := a.PoolNodes()
if len(pool) != 1 || len(pool[0].Kinds) != 2 || pool[0].Kinds[0].GPUMemGB != 40 ||
pool[0].Kinds[0].FreeGPUs != 1 || pool[0].Kinds[1].GPUMemGB != 80 {
t.Errorf("pool: %+v", pool)
}
}
+24 -7
View File
@@ -50,7 +50,26 @@ type QuotaRequest struct {
Limit store.Resources `json:"limit"`
}
func (a *API) registerQuotas(mux *http.ServeMux) {
// QuotaSet is the scope a quota was set on, as org:ID, team:ID or
// project:ID whatever name it was asked by.
type QuotaSet struct {
Scope string `json:"scope"`
}
// OwnerRequest sets the part of the tenancy tree a model's capacity counts
// against: org:NAME, team:ORG/TEAM or project:ORG/TEAM/PROJECT, or their IDs.
// Empty makes the model shared.
type OwnerRequest struct {
Owner string `json:"owner"`
}
// OwnerView is a model's owner now, as scope:ID.
type OwnerView struct {
Model string `json:"model"`
Owner string `json:"owner"`
}
func (a *API) registerQuotas(mux Mux) {
mux.HandleFunc("GET /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
@@ -78,7 +97,7 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"scope": key})
writeJSON(w, http.StatusOK, QuotaSet{Scope: key})
})
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
@@ -90,16 +109,14 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"removed": key})
writeJSON(w, http.StatusOK, Removed{Removed: key})
})
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil || a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req struct {
Owner string `json:"owner"`
}
var req OwnerRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
@@ -118,7 +135,7 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "owner": m.Owner})
writeJSON(w, http.StatusOK, OwnerView{Model: m.Name, Owner: m.Owner})
})
}
+84
View File
@@ -0,0 +1,84 @@
package adminapi
// The small bodies the endpoints read and answer with. Named types rather
// than maps and anonymous structs, so the OpenAPI document generated from
// Routes describes exactly what a handler decodes and writes: a map literal
// in a handler is a shape no generator can see.
// ErrorView is every refusal: a 4xx or 5xx status with what went wrong.
type ErrorView struct {
Error string `json:"error"`
}
// Result is what a change that has no other answer says it did.
type Result struct {
Result string `json:"result"`
}
// ModelAdded names the model added to the catalog.
type ModelAdded struct {
Added string `json:"added"`
}
// Removed names what was taken away: a model, or a quota's scope.
type Removed struct {
Removed string `json:"removed"`
}
// ClassRequest sets a model's capacity class: critical, standard or
// opportunistic, or empty to go back to what its scaling implies.
type ClassRequest struct {
Class string `json:"class"`
}
// ClassView is a model's capacity class now.
type ClassView struct {
Model string `json:"model"`
Class string `json:"class"`
}
// RetryView says whether the model had been parked, given up on after
// failing to start, before it was retried.
type RetryView struct {
WasParked bool `json:"was_parked"`
}
// NodeStateRequest cordons, drains or readies a node: ready (or uncordon),
// cordoned (or cordon), draining (or drain).
type NodeStateRequest struct {
State string `json:"state"`
}
// NodeStateView is a node's state after the change.
type NodeStateView struct {
ID string `json:"id"`
State string `json:"state"`
}
// NodeRemoved names the node forgotten. Removed is the string "true".
type NodeRemoved struct {
ID string `json:"id"`
Removed string `json:"removed"`
}
// UpgradeView is the release a node was told to run.
type UpgradeView struct {
Node string `json:"node"`
Version string `json:"version"`
}
// FabricTestRequest is a bandwidth test between two nodes.
type FabricTestRequest struct {
A string `json:"a"`
B string `json:"b"`
// Fabric is the declared fabric to test over; empty is the one the two
// nodes share.
Fabric string `json:"fabric,omitempty"`
// Seconds each rail is measured for, from 1 to 30.
Seconds int `json:"seconds,omitempty"`
}
// ReplayRequest names the model a kept request is sent to again.
type ReplayRequest struct {
Model string `json:"model"`
}
+1 -3
View File
@@ -60,9 +60,7 @@ func (a *API) replayRequest(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotImplemented, "this control plane does not route requests")
return
}
var req struct {
Model string `json:"model"`
}
var req ReplayRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil || req.Model == "" {
writeError(w, http.StatusBadRequest, "say which model: {\"model\": \"...\"}")
return
+272
View File
@@ -0,0 +1,272 @@
package adminapi
import (
"net/http"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Route is one endpoint as the OpenAPI document describes it.
//
// A table written by hand beside Register rather than read out of the
// handlers, because what a handler decodes and answers is not visible from
// outside it. The table is checked against Register by a test, so an endpoint
// cannot be added without being described; the bodies are Go values whose
// types the generator reads, so a field added to one is in the document the
// next time it is generated.
type Route struct {
Method string
Path string
// Summary is one line; Description, when there is more to say, the rest.
Summary string
Description string
// Query is the query parameters the handler reads.
Query []Param
// Request is a value of the body's type; nil reads none.
Request any
// OptionalBody is set when the body may be left out.
OptionalBody bool
// Response is a value of the type a success answers with.
Response any
// Status is the success status; zero is 200.
Status int
// CSV is set when the answer is CSV unless JSON is asked for.
CSV bool
}
// Param is a query parameter.
type Param struct {
Name string
Description string
// Type is a JSON Schema type: string when empty, or integer, number,
// boolean.
Type string
Required bool
}
// PathParams describes the parameters in Routes' paths, by the segment
// before them and their name.
var PathParams = map[string]string{
"models/{name}": "The model's name in the catalog.",
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
"jobs/{id}": "The job's ID, as starting it answered.",
"runs/{id}": "The quality run's ID, as starting it answered.",
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
}
// Routes is every endpoint Register mounts.
var Routes = []Route{
// The document itself.
{Method: "GET", Path: "/api/v1/openapi.json", Summary: "This document",
Description: "The admin API in OpenAPI 3.1, generated from the route table and built into the binary.",
Response: map[string]any{}},
// The fleet.
{Method: "GET", Path: "/api/v1/summary", Summary: "The whole fleet in one answer",
Response: Summary{}},
{Method: "GET", Path: "/api/v1/nodes", Summary: "Every node, with its cards, load and fabrics",
Response: []NodeView{}},
{Method: "POST", Path: "/api/v1/nodes/{id}/state", Summary: "Cordon, drain or ready a node",
Request: NodeStateRequest{}, Response: NodeStateView{}},
{Method: "POST", Path: "/api/v1/nodes/{id}/upgrade", Summary: "Have a node run a release",
Request: UpgradeRequest{}, Response: UpgradeView{}},
{Method: "POST", Path: "/api/v1/nodes/{id}/engines", Summary: "Choose an engine's build for a node",
Description: "The node installs the build chosen. Build is auto, a build such as cu129, or none.",
Request: NodeEngineRequest{}, Response: NodeEngineRequest{}},
{Method: "DELETE", Path: "/api/v1/nodes/{id}", Summary: "Forget a node that has gone",
Description: "Only a node that is lost, or unreachable with nothing on it; its certificate is revoked.",
Response: NodeRemoved{}},
{Method: "GET", Path: "/api/v1/nodes/{id}/logs", Summary: "The last lines of a node's logs",
Query: []Param{
{Name: "lines", Type: "integer", Description: "How many lines; 300 when absent, at most 2000."},
{Name: "source", Description: "Which log: agent when absent, or an engine the node lists in sources."},
},
Response: LogsView{}},
{Method: "GET", Path: "/api/v1/fabrics", Summary: "The physical fabrics and the names nodes give them",
Response: fabrics.Report{}},
{Method: "GET", Path: "/api/v1/fabrics/{id}/topology", Summary: "One fabric's map, with the cluster's adapters live on it",
Response: TopologyView{}},
{Method: "GET", Path: "/api/v1/nvlink", Summary: "The multi-node NVLink domains and what keeps nodes out of them",
Response: nvlink.Report{}},
{Method: "POST", Path: "/api/v1/fabric-test", Summary: "Measure bandwidth between two nodes, rail by rail",
Request: FabricTestRequest{}, Response: FabricTestView{}},
{Method: "GET", Path: "/api/v1/weights/orphans", Summary: "Weights on nodes that no model uses",
Response: []OrphanView{}},
{Method: "POST", Path: "/api/v1/weights/collect", Summary: "Delete weights that no model uses",
Request: CollectRequest{}, OptionalBody: true, Response: []CollectResult{}},
// The catalog.
{Method: "GET", Path: "/api/v1/models", Summary: "Every model, with its replicas and state",
Response: []ModelView{}},
{Method: "POST", Path: "/api/v1/models", Summary: "Add a model to the catalog",
Description: "Kept in the store across restarts. Refused when the name is taken, a model in " +
"configuration has it, or the definition would make the catalog invalid.",
Request: catalog.Model{}, Response: ModelAdded{}},
{Method: "GET", Path: "/api/v1/models/{name}", Summary: "One model's definition, as it is in force",
Description: "With the defaults the catalog filled in. By its name only, not an alias.",
Response: catalog.Model{}},
{Method: "DELETE", Path: "/api/v1/models/{name}", Summary: "Remove a model added at runtime",
Description: "A model declared in configuration is refused: it would come back on the next restart.",
Response: Removed{}},
{Method: "POST", Path: "/api/v1/models/{name}/scale", Summary: "Set how many replicas a model runs",
Request: ScaleRequest{}, Response: ScaleResponse{}},
{Method: "POST", Path: "/api/v1/models/{name}/resources", Summary: "Set what a model's profiles reserve of a node",
Request: ResourcesRequest{}, Response: catalog.Model{}},
{Method: "POST", Path: "/api/v1/models/{name}/class", Summary: "Set a model's capacity class",
Request: ClassRequest{}, Response: ClassView{}},
{Method: "POST", Path: "/api/v1/models/{name}/owner", Summary: "Set the part of the tenancy tree a model's capacity counts against",
Request: OwnerRequest{}, Response: OwnerView{}},
{Method: "POST", Path: "/api/v1/models/{name}/adapters", Summary: "Replace a model's LoRA adapters",
Description: "Each new file is read first, to check it is an adapter for the model's architecture.",
Request: AdaptersRequest{}, Response: AdaptersRequest{}},
{Method: "POST", Path: "/api/v1/models/{name}/retry", Summary: "Try a parked model again",
Response: RetryView{}},
{Method: "GET", Path: "/api/v1/models/{name}/target", Summary: "How many replicas a throughput needs, and whether they fit",
Query: []Param{{Name: "tokens_per_sec", Type: "number", Required: true,
Description: "The output tokens a second wanted of the model."}},
Response: TargetView{}},
{Method: "GET", Path: "/api/v1/models/{name}/headroom", Summary: "How many more replicas of a model fit now, and where",
Query: []Param{{Name: "limit", Type: "integer",
Description: "The most to try placing; 16 when absent or past 64."}},
Response: HeadroomView{}},
{Method: "PUT", Path: "/api/v1/models/{name}/capacity", Summary: "Record what a replica was measured to serve",
Description: "Replaces an earlier measurement of the same model on the same profile.",
Request: bench.Capacity{}, Response: bench.Capacity{}},
{Method: "GET", Path: "/api/v1/capacity", Summary: "Every recorded measurement",
Response: []CapacityView{}},
{Method: "GET", Path: "/api/v1/replicas", Summary: "Every replica, with its shards",
Query: []Param{{Name: "model", Description: "Only this model's replicas."}},
Response: []ReplicaView{}},
{Method: "GET", Path: "/api/v1/explain-placement", Summary: "Why a model could not be placed",
Query: []Param{{Name: "model", Required: true, Description: "The model's name or an alias."}},
Response: ExplainView{}},
{Method: "POST", Path: "/api/v1/restart", Summary: "Restart replicas, every replica of some models, or all",
Request: RestartRequest{}, Response: RestartResult{}, Status: http.StatusAccepted},
// Serving from the registry, and newer weights.
{Method: "GET", Path: "/api/v1/plan", Summary: "How a registry repository would be served here",
Query: []Param{
{Name: "repo", Required: true, Description: "The repository, owner/name."},
{Name: "format", Description: "gguf or safetensors; gguf when absent."},
{Name: "context", Type: "integer", Description: "The context length to plan for, in tokens."},
},
Response: modelserve.Plan{}},
{Method: "POST", Path: "/api/v1/models/serve", Summary: "Fetch a registry repository and serve it",
Request: modelserve.Request{}, Response: modelserve.Job{}, Status: http.StatusAccepted},
{Method: "GET", Path: "/api/v1/jobs", Summary: "Every job since the control plane started",
Response: []modelserve.Job{}},
{Method: "GET", Path: "/api/v1/jobs/{id}", Summary: "One job",
Response: modelserve.Job{}},
{Method: "GET", Path: "/api/v1/updates", Summary: "Newer weights the registry has for models",
Response: []modelserve.UpdateInfo{}},
{Method: "POST", Path: "/api/v1/updates/check", Summary: "Ask the registry now, about one model or every one",
Request: CheckRequest{}, OptionalBody: true, Response: []modelserve.UpdateInfo{}},
{Method: "POST", Path: "/api/v1/models/{name}/update", Summary: "Move a model to its repository's newer weights",
Description: "With a quality suite, or the model's standing gate, the new weights are asked it first as a hidden canary.",
Request: modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted},
{Method: "POST", Path: "/api/v1/models/{name}/rollback", Summary: "Go back to the weights the last update replaced",
Response: Result{}},
{Method: "POST", Path: "/api/v1/models/{name}/origin", Summary: "Record the repository a model's weights came from",
Request: OriginRequest{}, Response: OriginView{}},
{Method: "GET", Path: "/api/v1/canaries", Summary: "Every canary, and how its two sides serve",
Response: []modelserve.CanaryView{}},
{Method: "POST", Path: "/api/v1/models/{name}/canary", Summary: "Start a canary of a model's newer weights",
Request: catalog.Canary{}, Response: modelserve.Job{}, Status: http.StatusAccepted},
{Method: "PUT", Path: "/api/v1/models/{name}/canary", Summary: "Change the share of requests a canary takes",
Request: CanaryPercentRequest{}, Response: Result{}},
{Method: "DELETE", Path: "/api/v1/models/{name}/canary", Summary: "Take a canary out: its share first, then the candidate",
Response: Result{}},
{Method: "POST", Path: "/api/v1/models/{name}/promote", Summary: "Make a canary's weights the model's",
Description: "With a quality suite, or the model's standing gate, the canary is asked it first and kept below the score.",
Request: modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted},
// Quality suites.
{Method: "GET", Path: "/api/v1/eval/suites", Summary: "The quality suites the control plane can run by name",
Response: []eval.SuiteInfo{}},
{Method: "POST", Path: "/api/v1/models/{name}/eval", Summary: "Ask a model a quality suite",
Description: "Started at once and answered with the run, to be followed at /api/v1/eval/runs/{id}.",
Request: eval.Request{}, Response: eval.Live{}, Status: http.StatusAccepted},
{Method: "GET", Path: "/api/v1/eval/runs", Summary: "Quality runs, the newest first",
Query: []Param{
{Name: "model", Description: "Only this model's runs."},
{Name: "limit", Type: "integer", Description: "How many; a sensible number when absent."},
},
Response: []eval.Live{}},
{Method: "GET", Path: "/api/v1/eval/runs/{id}", Summary: "One quality run, with each case",
Response: eval.Live{}},
{Method: "PUT", Path: "/api/v1/models/{name}/eval-gate", Summary: "Hold a model's updates and promotions to a quality suite",
Request: EvalGateRequest{}, Response: EvalGateView{}},
{Method: "DELETE", Path: "/api/v1/models/{name}/eval-gate", Summary: "Stop holding a model's updates to a suite",
Response: EvalGateView{}},
// Tenancy: quotas and guardrails.
{Method: "GET", Path: "/api/v1/quotas", Summary: "Every capacity quota, with what its scope holds now",
Response: []QuotaView{}},
{Method: "PUT", Path: "/api/v1/quotas", Summary: "Set a scope's capacity quota",
Description: "Replaces the scope's quota whole. The scope may be named by path or by ID; the answer names it by ID.",
Request: QuotaRequest{}, Response: QuotaSet{}},
{Method: "DELETE", Path: "/api/v1/quotas/{scope}", Summary: "Remove a scope's capacity quota",
Response: Removed{}},
{Method: "GET", Path: "/api/v1/guard", Summary: "The guardrail policies and where they are attached",
Response: GuardView{}},
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
// Usage, requests and the record of changes.
{Method: "GET", Path: "/api/v1/usage", Summary: "What each group of the tenancy tree spent",
Query: []Param{
{Name: "by", Description: "org, team or project; team when absent."},
{Name: "since", Description: "A duration back from now, like 24h; 24h when absent."},
},
Response: []ScopeUsageView{}},
{Method: "GET", Path: "/api/v1/usage/export", Summary: "Usage totals by period and dimension, as CSV or JSON",
Query: []Param{
{Name: "from", Description: "A date (2026-09-01) or an RFC 3339 time; the first of this month when absent."},
{Name: "to", Description: "A date or an RFC 3339 time; now when absent."},
{Name: "period", Description: "hour, day or month; day when absent."},
{Name: "by", Description: "Dimensions, comma-separated: org, team, project, user, key, model."},
{Name: "format", Description: "json for JSON; CSV otherwise."},
},
Response: []store.ExportRow{}, CSV: true},
{Method: "GET", Path: "/api/v1/requests/{id}", Summary: "What one request went through",
Response: RequestView{}},
{Method: "GET", Path: "/api/v1/requests/{id}/body", Summary: "A kept request and its answer",
Response: store.RequestBody{}},
{Method: "POST", Path: "/api/v1/requests/{id}/replay", Summary: "Send a kept request again, to any model",
Request: ReplayRequest{}, Response: ReplayView{}},
{Method: "GET", Path: "/api/v1/audit", Summary: "Changes made, newest first",
Query: []Param{
{Name: "since", Description: "How far back: 90m, 24h, 7d; 7d when absent."},
{Name: "limit", Type: "integer", Description: "The most events to answer with."},
{Name: "actor", Description: "Only changes by this actor."},
{Name: "contains", Description: "Only events whose action or target contains this."},
},
Response: []store.AuditEvent{}},
// Operations.
{Method: "POST", Path: "/api/v1/notify/test", Summary: "Send a test notification to every webhook",
Response: []notify.Result{}},
{Method: "POST", Path: "/api/v1/metrics/range", Summary: "Several Prometheus range queries at once",
Description: "Read from the Prometheus monitoring.prometheus names; configured is false without one.",
Request: MetricsRequest{}, Response: MetricsResult{}},
}
// serveOpenAPI answers with the document generated from Routes, as it was
// when this binary was built.
func serveOpenAPI(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(llmgw.AdminOpenAPI)
}
+119
View File
@@ -0,0 +1,119 @@
package adminapi
import (
"net/http"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// patterns records what Register mounts.
type patterns []string
func (p *patterns) HandleFunc(pattern string, _ func(http.ResponseWriter, *http.Request)) {
*p = append(*p, pattern)
}
func TestEveryRegisteredEndpointIsInTheRouteTable(t *testing.T) {
// The OpenAPI document is generated from Routes, so an endpoint missing
// from it is one no client generated from the document can call, and one
// in it that is not served is a promise the server breaks.
a, _ := setup(t, testModels())
var got patterns
a.Register(&got)
table := map[string]bool{}
for _, r := range Routes {
key := r.Method + " " + r.Path
if table[key] {
t.Errorf("%s is in Routes twice", key)
}
table[key] = true
}
served := map[string]bool{}
for _, p := range got {
served[p] = true
if !table[p] {
t.Errorf("%s is registered and not in Routes: describe it there", p)
}
}
for key := range table {
if !served[key] {
t.Errorf("%s is in Routes and not registered", key)
}
}
}
func TestEveryRouteSaysWhatItAnswers(t *testing.T) {
for _, r := range Routes {
if r.Summary == "" || r.Response == nil {
t.Errorf("%s %s needs a summary and a response type", r.Method, r.Path)
}
if r.Request != nil && (r.Method == "GET" || r.Method == "HEAD") {
t.Errorf("%s %s reads a body on a GET", r.Method, r.Path)
}
for _, seg := range strings.Split(strings.Trim(r.Path, "/"), "/") {
if !strings.HasPrefix(seg, "{") {
continue
}
found := false
for k := range PathParams {
if strings.HasSuffix(k, "/"+seg) && strings.Contains(r.Path, k) {
found = true
}
}
if !found {
t.Errorf("%s %s: %s is not described in PathParams", r.Method, r.Path, seg)
}
}
}
}
func TestTheAPIServesItsOwnDescription(t *testing.T) {
a, _ := setup(t, testModels())
var doc struct {
OpenAPI string `json:"openapi"`
Paths map[string]map[string]any `json:"paths"`
}
if code := call(t, a, "GET", "/api/v1/openapi.json", "", &doc); code != 200 {
t.Fatalf("status %d", code)
}
if doc.OpenAPI != "3.1.0" {
t.Errorf("openapi %q", doc.OpenAPI)
}
// What is built in is the document generated when it was last committed;
// internal/docgen's test is what fails when that is stale. This checks the
// two ends meet: every route the table has is in what is served.
for _, r := range Routes {
if _, ok := doc.Paths[r.Path][strings.ToLower(r.Method)]; !ok {
t.Errorf("the served document has no %s %s", r.Method, r.Path)
}
}
}
func TestOneModelsDefinitionIsReadByItsNameAlone(t *testing.T) {
a, _ := setup(t, testModels())
var m catalog.Model
if code := call(t, a, "GET", "/api/v1/models/llama", "", &m); code != 200 {
t.Fatalf("status %d", code)
}
if m.Name != "llama" || m.Source != "/m" || len(m.Profiles) != 1 || m.Profiles[0].TP != 8 {
t.Errorf("got %+v", m)
}
if m.Scaling.Min != 2 || m.Scaling.Max != 4 {
t.Errorf("scaling %+v", m.Scaling)
}
if !slices.Equal(m.Aliases, []string{"l"}) {
t.Errorf("aliases %v", m.Aliases)
}
// An alias answering would let a tool that manages "l" believe it exists
// when what it found is llama.
if code := call(t, a, "GET", "/api/v1/models/l", "", nil); code != http.StatusNotFound {
t.Errorf("an alias answered %d", code)
}
if code := call(t, a, "GET", "/api/v1/models/nothing", "", nil); code != http.StatusNotFound {
t.Errorf("a missing model answered %d", code)
}
}
+8 -6
View File
@@ -148,12 +148,14 @@ func (a *API) Summary() Summary {
byClass[c] = &ClassUse{Class: c}
}
}
// A node reports one class in practice; attribute its cards to it.
if len(n.Classes) == 1 {
u := byClass[n.Classes[0]]
u.Total += n.GPUTotal
u.Unhealthy += len(n.Unhealthy)
u.Used += n.GPUTotal - n.GPUFree - len(n.Unhealthy)
// Each class its own devices, which a node with a card in MIG mode
// needs: its instances are one class or several, and any whole card
// beside them another.
for _, c := range n.ClassGPUs {
u := byClass[c.Class]
u.Total += c.Total
u.Unhealthy += c.Unhealthy
u.Used += c.Total - c.Free - c.Unhealthy
}
}
for _, u := range byClass {
+17 -13
View File
@@ -65,6 +65,20 @@ func (a *API) listOrphans(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.Orphans())
}
// CollectRequest limits a sweep to one node. Empty means every node, which
// is the common case after removing a model from the catalog.
type CollectRequest struct {
Node string `json:"node,omitempty"`
}
// CollectResult is what was deleted on one node, or why nothing was.
type CollectResult struct {
Node string `json:"node"`
Files int `json:"files"`
Bytes int64 `json:"bytes"`
Error string `json:"error,omitempty"`
}
// collectWeights deletes them.
//
// A proposal, not an instruction: each node checks every path again before
@@ -79,26 +93,16 @@ func (a *API) collectWeights(w http.ResponseWriter, r *http.Request) {
return
}
var body struct {
// Node limits it to one node. Empty means every node, which is the
// common case after removing a model from the catalog.
Node string `json:"node,omitempty"`
}
var body CollectRequest
// An empty body is the "all nodes" case and must not be an error.
_ = json.NewDecoder(r.Body).Decode(&body)
type result struct {
Node string `json:"node"`
Files int `json:"files"`
Bytes int64 `json:"bytes"`
Error string `json:"error,omitempty"`
}
out := make([]result, 0)
out := make([]CollectResult, 0)
for _, o := range a.Orphans() {
if body.Node != "" && o.Node != body.Node {
continue
}
res := result{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes}
res := CollectResult{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes}
// A node that has gone since the list was built is reported rather than
// failing the whole sweep: the other nodes' space is still worth
// freeing, and this one will be offered again next time.
+21
View File
@@ -71,6 +71,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
TemperatureC: int(g.GetTemperatureC()),
Problems: g.GetProblems(),
},
MIG: migFrom(g.GetMig()),
})
}
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
@@ -131,6 +132,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
}
if m := g.MIG; m != nil {
r.Gpus[len(r.Gpus)-1].Mig = &llmgwv1.MIGDevice{
ParentIndex: int32(m.ParentIndex), ParentUuid: m.ParentUUID,
ParentMemoryTotalMb: int32(m.ParentMemoryTotalMB), Device: int32(m.Device),
Profile: m.Profile, GpuInstance: int32(m.GPUInstance), ComputeInstance: int32(m.ComputeInstance),
}
}
}
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
for _, f := range fabrics {
@@ -286,6 +294,19 @@ func int32Slice(v []int) []int32 {
return out
}
// migFrom is a MIG instance's parentage as the wire carries it, nil for a
// whole card and from every agent older than MIG support.
func migFrom(m *llmgwv1.MIGDevice) *discovery.MIGDevice {
if m == nil {
return nil
}
return &discovery.MIGDevice{
ParentIndex: int(m.GetParentIndex()), ParentUUID: m.GetParentUuid(),
ParentMemoryTotalMB: int(m.GetParentMemoryTotalMb()), Device: int(m.GetDevice()),
Profile: m.GetProfile(), GPUInstance: int(m.GetGpuInstance()), ComputeInstance: int(m.GetComputeInstance()),
}
}
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
// driver reported none.
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
+27
View File
@@ -0,0 +1,27 @@
package agentsvc
import (
"reflect"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
func TestAMIGInstanceKeepsItsParentageOnTheWire(t *testing.T) {
inst := discovery.GPU{Index: 1001, UUID: "MIG-bbb", Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentUUID: "GPU-card", ParentMemoryTotalMB: 81920,
Device: 1, Profile: "3g.40gb", GPUInstance: 1, ComputeInstance: 0}}
whole := discovery.GPU{Index: 1, UUID: "GPU-whole", Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920}
reg := RegisterFromInventory("n1", "h", "10.0.0.1", "v", nil, discovery.Inventory{GPUs: []discovery.GPU{inst, whole}}, nil)
n := NodeFromRegister(reg)
if len(n.GPUs) != 2 {
t.Fatalf("got %d GPUs", len(n.GPUs))
}
if !reflect.DeepEqual(n.GPUs[0].MIG, inst.MIG) || n.GPUs[0].Class() != "a100-80g-mig-3g.40gb" {
t.Errorf("the instance arrived as %s %+v", n.GPUs[0].Class(), n.GPUs[0].MIG)
}
if n.GPUs[1].MIG != nil || n.GPUs[1].Class() != "a100-80g" {
t.Errorf("the whole card arrived as %s %+v", n.GPUs[1].Class(), n.GPUs[1].MIG)
}
}
+34 -7
View File
@@ -14,6 +14,8 @@ import (
"time"
"connectrpc.com/connect"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
@@ -22,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// StatusSink receives shard transitions and recovers replica state from what
@@ -311,7 +314,20 @@ func (s *Service) Connect(
// unreachable in production while its unit tests, which call the reconciler
// directly, kept passing.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
// A trace of its own, and only for a node that reported something: a
// node running nothing registering is not an event anyone looks for.
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
if len(shards) > 0 {
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
trace.WithNewRoot(),
trace.WithAttributes(
attribute.String("llmgw.node", nodeID),
attribute.Int("llmgw.shards", len(shards)),
))
}
defer span.End()
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Trace: obs.Inject(tctx),
}:
case <-ctx.Done():
return
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
// and what every deployment before leadership had.
func (s *Service) SetLeadership(l Leader) { s.leader = l }
// tracerName names the spans this package opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
// ErrNotConnected means no agent stream is open for that node.
var ErrNotConnected = errors.New("no agent connected")
// StartShard implements reconciler.Dispatcher.
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
//
// The command carries the trace of ctx, so the agent's spans for this start
// — the weights, the engine's launch, its readiness — are in the trace of
// the replica they are starting rather than in one of their own.
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Trace: obs.Inject(ctx),
})
}
// StopShard implements reconciler.Dispatcher.
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
// DrainShard implements reconciler.Dispatcher.
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
+49
View File
@@ -12,6 +12,7 @@ import (
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
@@ -294,6 +295,10 @@ type Model struct {
// model — the same model at new weights — to compare the two before
// moving to it (`llmgw ctl model canary`).
Canary *Canary `json:"canary,omitempty" koanf:"canary"`
// Eval is a quality suite new weights must pass before they replace
// this model's: `llmgw ctl model update` and `model promote` run it
// against them first, and stop below its score.
Eval *EvalGate `json:"eval,omitempty" koanf:"eval"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
SLO SLO `json:"slo,omitempty" koanf:"slo"`
@@ -512,6 +517,18 @@ type Canary struct {
MaxTTFTRatio float64 `json:"max_ttft_ratio,omitempty" koanf:"max_ttft_ratio"`
}
// EvalGate is a quality suite a model's new weights must pass.
type EvalGate struct {
// Suite is a suite's name in control.eval_dir, or one the binary
// carries (arithmetic, json-format).
Suite string `json:"suite" koanf:"suite"`
// MinScore is the score, 0 to 1, to pass at; zero is the suite's own
// threshold.
MinScore float64 `json:"min_score,omitempty" koanf:"min_score"`
// Judge grades the suite's rubric cases, instead of the judge it names.
Judge string `json:"judge,omitempty" koanf:"judge"`
}
// Limits are the rollback thresholds with their defaults filled in.
func (c Canary) Limits() (minRequests int, maxErrorIncrease, maxTTFTRatio float64) {
minRequests, maxErrorIncrease, maxTTFTRatio = c.MinRequests, c.MaxErrorIncrease, c.MaxTTFTRatio
@@ -633,6 +650,14 @@ func (m Model) validate() error {
if !validClass(m.Class) {
return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class)
}
if g := m.Eval; g != nil {
if strings.TrimSpace(g.Suite) == "" {
return fmt.Errorf("model %q: eval names no suite", m.Name)
}
if g.MinScore < 0 || g.MinScore > 1 {
return fmt.Errorf("model %q: eval min_score %v is not 0 to 1", m.Name, g.MinScore)
}
}
switch m.Role {
case "":
if m.Guard != nil {
@@ -701,6 +726,17 @@ func (m Model) validate() error {
}
continue
}
// A MIG instance is a device of its own to CUDA, but a process sees
// only one, and instances have no NVLink or peer-to-peer between
// them: a group of them is neither tensor- nor layer-parallel, it is
// a replica that fails at start. One instance, one node, refused here
// rather than placed and found out.
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) {
return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+
"a process can open only one MIG device, and instances have no NVLink between them. "+
"Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile",
m.Name, p.Name, p.GPUClass)
}
// llama.cpp on GPUs splits a model across one node's cards by layer,
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
// parallelism, and a tp it was given was a width nothing placed — the
@@ -717,6 +753,19 @@ func (m Model) validate() error {
}
continue
}
// TensorRT-LLM spans machines only under MPI started from one of them,
// which the agents, each starting its own node, do not do; and it
// loads a Hugging Face checkpoint, not llama.cpp's format.
if p.Engine == "trtllm" {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
}
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") {
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
"serve a GGUF with llamacpp", m.Name, p.Name)
}
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
+38
View File
@@ -0,0 +1,38 @@
package catalog
import (
"strings"
"testing"
)
func TestAProfileOnAMIGClassRunsOnExactlyOneInstance(t *testing.T) {
// One instance, by default or said outright, is a profile like any other.
for _, p := range []Profile{
{Name: "vllm", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb"},
{Name: "llama", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, TP: 1},
} {
c, err := New([]Model{model("m", p)})
if err != nil {
t.Fatalf("%s on one MIG instance: %v", p.Name, err)
}
m, _ := c.Resolve("m")
if pl := m.PlacementProfiles()[0]; pl.GPUsPerNode != 1 || pl.Nodes != 1 {
t.Errorf("%s: placement asked for %+v", p.Name, pl)
}
}
// More than one is refused where the model is defined: a process opens
// one MIG device, and instances have no NVLink between them, so the
// replica would be placed and then fail at start.
for _, p := range []Profile{
{Name: "tp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2},
{Name: "pp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 1, PP: 2, GPUsPerNode: 2},
{Name: "nodes", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2, Nodes: 2, GPUsPerNode: 1},
{Name: "layers", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2},
{Name: "old-tp", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", TP: 2},
} {
_, err := New([]Model{model("m", p)})
if err == nil || !strings.Contains(err.Error(), "MIG instance") {
t.Errorf("%s: %v", p.Name, err)
}
}
}
+53
View File
@@ -0,0 +1,53 @@
package catalog
import (
"strings"
"testing"
)
func TestATensorRTLLMProfileIsAGPUProfileOnOneNode(t *testing.T) {
ok := Model{Name: "llama", Source: "meta-llama/Llama-3.1-70B-Instruct", Profiles: []Profile{{
Name: "h100", Engine: "trtllm", GPUClass: "h100-80g", TP: 4,
Args: []string{"--max_batch_size", "64"},
}}}
c, err := New([]Model{ok})
if err != nil {
t.Fatalf("a four-GPU TensorRT-LLM profile was refused: %v", err)
}
m, _ := c.Resolve("llama")
if p := m.PlacementProfiles()[0]; p.GPUsPerNode != 4 || p.Nodes != 1 {
t.Errorf("placed as %+v, expected four GPUs of one node", p)
}
for want, bad := range map[string]Model{
// Two nodes need MPI started from one of them.
"one node": {Name: "x", Source: "org/x", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8}}},
// No CPU build exists.
"gpu_class is required": {Name: "x", Source: "org/x", Profiles: []Profile{{Name: "p", Engine: "trtllm"}}},
// llama.cpp's format.
"not a .gguf": {Name: "x", Source: "http://mirror:8081/x/v1/x-q4.gguf", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g"}}},
// What its launcher closes, a profile may not open.
"--trust_remote_code is not allowed": {Name: "x", Source: "org/x", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", Args: []string{"--trust_remote_code"}}}},
} {
if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("expected %q, got %v", want, err)
}
}
}
func TestATensorRTLLMModelIsGivenTheRepositoryNotItsPage(t *testing.T) {
// trtllm-serve downloads a repository itself, as vLLM does.
m := Model{Name: "q", Source: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "l40s-48g"}}}
c, err := New([]Model{m})
if err != nil {
t.Fatal(err)
}
got, _ := c.Resolve("q")
if !strings.HasSuffix(got.Source, "Qwen/Qwen2.5-7B-Instruct") || strings.Contains(got.Source, "huggingface.co") {
t.Errorf("source %q", got.Source)
}
}
+12 -2
View File
@@ -430,8 +430,18 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
byClass[class]++
labels := []string{n.ID, strconv.Itoa(g.Index), class}
s, fresh := sensorFor(n.Load, g.Index, now)
if fresh && s.Busy() {
// The sensors are the card's. A MIG instance shares its card's
// heat and power, but not its memory, and has no utilisation of
// its own that any driver reports: its memory is the instance's
// own reading, and its utilisation is left out rather than
// published as the card's or as zero.
s, fresh := sensorFor(n.Load, g.CardIndex(), now)
if g.IsMIG() {
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
float64(g.MemoryTotalMB)*1024*1024, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
float64(g.MemoryUsedMB)*1024*1024, labels...)
} else if fresh && s.Busy() {
// What the card is doing now, sampled every heartbeat.
c.gpuBusy(ch, s, labels)
} else {
+36
View File
@@ -0,0 +1,36 @@
package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
// Card 0 in MIG mode: the sensor sample is the card's, 60 GB used across
// both instances and a utilisation the driver does not give per instance.
inst := discovery.GPU{Index: 1001, Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192, MemoryUsedMB: 21000,
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}}
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst},
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250,
UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1"
for _, want := range []string{
"llmgw_gpu_memory_used_bytes " + labels + " -> 22020096000\n",
"llmgw_gpu_memory_total_bytes " + labels + " -> 42144366592\n",
"llmgw_gpu_temperature_celsius " + labels + " -> 61\n",
"llmgw_gpu_power_watts " + labels + " -> 250\n",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
// Not the card's zero, not anything: absent.
if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) {
t.Errorf("a utilisation was published for a MIG instance:\n%s", out)
}
}
+51 -19
View File
@@ -56,25 +56,33 @@ func (s *Server) StartCanary(name string, cn catalog.Canary, by string) (Job, er
func (s *Server) runCanary(j *Job, cn catalog.Canary) {
ctx := context.Background()
name := j.Model
p, ok := s.prepare(ctx, j, name)
p, ok := s.prepare(ctx, j, j.Model)
if !ok {
return
}
if _, ok := s.addCandidate(ctx, j, p, cn); ok {
s.finish(j)
}
}
// addCandidate runs a model's prepared new weights beside it as its canary
// and gives it its share; it fails the job and says false when it cannot.
func (s *Server) addCandidate(ctx context.Context, j *Job, p prepared, cn catalog.Canary) (string, bool) {
name := j.Model
// 4. The candidate, beside the model: the same in everything but its
// weights, and one replica.
cand := canaryName(name)
s.step(j, "catalog", Running, "adding "+cand)
c := p.m
c.Name, c.Aliases, c.Fallbacks, c.Canary, c.Adapters = cand, nil, nil, nil, nil
c.Name, c.Aliases, c.Fallbacks, c.Canary, c.Adapters, c.Eval = cand, nil, nil, nil, nil, nil
c.Source = p.source
c.Origin = &catalog.Origin{Repo: p.repo.ID, Revision: p.repo.Revision, Files: originFiles(p.set)}
c.Scaling = catalog.Scaling{Min: 1, Max: 1}
c.Description = fmt.Sprintf("Canary of %s at %s.", name, p.repo.ShortRevision())
if err := s.deps.Models.Add(ctx, c, j.Request.By); err != nil {
s.fail(j, "catalog", err)
return
return "", false
}
s.step(j, "catalog", Done, cand+" runs "+p.repo.ID+" at "+p.repo.ShortRevision())
@@ -93,21 +101,22 @@ func (s *Server) runCanary(j *Job, cn catalog.Canary) {
if err != nil {
_ = s.deps.Models.Remove(ctx, cand)
s.fail(j, "placement", fmt.Errorf("%w; %s was taken out again", err, cand))
return
return "", false
}
// 6. Its share.
// 6. Its share: none, for new weights only being asked a suite, which
// hides it from clients all the same.
cn.Model = cand
if _, err := s.deps.Models.SetCanary(ctx, name, &cn, j.Request.By); err != nil {
_ = s.deps.Models.Remove(ctx, cand)
s.fail(j, "split", err)
return
return "", false
}
if s.deps.ResetCanary != nil {
s.deps.ResetCanary(name)
}
s.step(j, "split", Done, fmt.Sprintf("%d%% of %s's requests go to %s", cn.Percent, name, cand))
s.finish(j)
return cand, true
}
// SetCanaryPercent changes the share of a model's requests its canary is
@@ -152,8 +161,9 @@ func (s *Server) AbortCanary(ctx context.Context, name, by string) (string, erro
}
// StartPromote moves a model to its canary's weights — the update a canary
// was the trial of — and takes the canary out.
func (s *Server) StartPromote(name, by string) (Job, error) {
// was the trial of — and takes the canary out. Held to a quality suite, the
// canary is asked it first, and stays a canary below its score.
func (s *Server) StartPromote(name, by string, ch Change) (Job, error) {
m, err := s.current(name)
if err != nil {
return Job{}, err
@@ -161,28 +171,50 @@ func (s *Server) StartPromote(name, by string) (Job, error) {
if m.Canary == nil {
return Job{}, fmt.Errorf("%s has no canary to promote", name)
}
g, skipped := gateFor(m, ch)
if err := s.checkGate(m.Canary.Model, g); err != nil {
return Job{}, err
}
s.mu.Lock()
for _, j := range s.jobs {
if (j.Kind == "update" || j.Kind == "canary" || j.Kind == "promote") && j.Model == name && j.State == Running {
s.mu.Unlock()
return Job{}, fmt.Errorf("%s has a job running: %s", name, j.ID)
}
}
s.seq++
j := &Job{ID: fmt.Sprintf("promote-%d-%d", time.Now().Unix(), s.seq), Kind: "promote", Model: name,
Request: Request{Name: name, By: by}, State: Running, Started: time.Now()}
s.keepJob(j)
s.mu.Unlock()
go s.runPromote(j)
go func() {
ctx := context.Background()
cand := m.Canary.Model
if !s.gate(ctx, j, cand, g, skipped, func() string {
return fmt.Sprintf("%s stays a canary: `llmgw ctl model canary %s --abort` takes it out", cand, name)
}) {
return
}
if s.promoteTo(ctx, j) {
s.finish(j)
}
}()
return s.snapshot(j), nil
}
func (s *Server) runPromote(j *Job) {
ctx := context.Background()
// promoteTo moves a model to its canary's weights, replaces its replicas and
// takes the canary out; it fails the job and says false when it cannot.
func (s *Server) promoteTo(ctx context.Context, j *Job) bool {
name := j.Model
m, err := s.current(name)
if err != nil || m.Canary == nil {
s.fail(j, "catalog", fmt.Errorf("%s no longer has a canary", name))
return
return false
}
c, err := s.current(m.Canary.Model)
if err != nil {
s.fail(j, "catalog", err)
return
return false
}
s.step(j, "catalog", Running, fmt.Sprintf("moving %s to %s's weights", name, c.Name))
next := &catalog.Origin{}
@@ -195,18 +227,18 @@ func (s *Server) runPromote(j *Job) {
}
if _, err := s.deps.Models.SetSource(ctx, name, c.Source, next, j.Request.By); err != nil {
s.fail(j, "catalog", err)
return
return false
}
// Its share back to the model, which is about to run the same weights.
if _, err := s.deps.Models.SetCanary(ctx, name, nil, j.Request.By); err != nil {
s.fail(j, "catalog", err)
return
return false
}
s.step(j, "catalog", Done, "the catalog names "+c.Source)
if err := s.replace(ctx, j, name); err != nil {
s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back", err, name))
return
return false
}
if err := s.deps.Models.Remove(ctx, c.Name); err != nil {
s.step(j, "canary", Failed, fmt.Sprintf("%s could not be removed: %v", c.Name, err))
@@ -222,7 +254,7 @@ func (s *Server) runPromote(j *Job) {
Latest: c.Origin.Revision, Previous: next.Previous.Revision, CheckedAt: time.Now()}
s.mu.Unlock()
}
s.finish(j)
return true
}
// WatchCanaries takes out, every so often, each canary serving worse than
+1 -1
View File
@@ -49,7 +49,7 @@ func TestACanaryRunsTheNewWeightsBesideAndIsPromoted(t *testing.T) {
t.Errorf("share: %v", err)
}
job, err = s.StartPromote("qwen", "tester")
job, err = s.StartPromote("qwen", "tester", Change{})
if err != nil {
t.Fatal(err)
}
+96
View File
@@ -0,0 +1,96 @@
package modelserve
import (
"context"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// A gate: new weights asked a quality suite before they replace a model's,
// and refused below its score. A promotion asks the canary, which is
// already running; an update starts the new weights as a canary given none
// of the model's requests, asks it, and promotes it or takes it out — so
// the old weights serve until the new have passed.
// Evaluator runs quality suites: the control plane's eval service.
type Evaluator interface {
// Check says whether a suite can be run, before anything is started
// for it.
Check(model string, r eval.Request) error
// Run asks a model a suite and waits for its score.
Run(ctx context.Context, model string, r eval.Request, by, job string) (store.EvalRun, error)
}
// Change is how an update or a promotion is held to quality.
type Change struct {
// Eval is a suite the new weights must pass first. Nil takes the
// model's own (eval: in its catalog entry), if it has one.
Eval *eval.Request `json:"eval,omitempty"`
// SkipEval moves without the model's own suite; the job says so.
SkipEval bool `json:"skip_eval,omitempty"`
}
// gateFor is the suite a change is held to, and whether one was skipped.
func gateFor(m catalog.Model, ch Change) (r *eval.Request, skipped bool) {
switch {
case ch.Eval != nil:
return ch.Eval, false
case m.Eval == nil:
return nil, false
case ch.SkipEval:
return nil, true
}
return &eval.Request{Suite: m.Eval.Suite, MinScore: m.Eval.MinScore, Judge: m.Eval.Judge}, false
}
// checkGate refuses, before a job starts, a gate that cannot be run.
func (s *Server) checkGate(name string, r *eval.Request) error {
if r == nil {
return nil
}
if s.deps.Eval == nil {
return errors.New("this control plane cannot run quality suites: it routes no requests")
}
if err := s.deps.Eval.Check(name, *r); err != nil {
return fmt.Errorf("the quality suite: %w", err)
}
return nil
}
// gate asks the new weights, running as target, a suite, and says whether
// they may replace the old; when they may not, the job fails saying why,
// and what refused does about the new weights, which it says too.
func (s *Server) gate(ctx context.Context, j *Job, target string, r *eval.Request, skipped bool, refused func() string) bool {
if skipped {
s.step(j, "eval", Skipped, "the model's quality suite was skipped by "+orUnknown(j.Request.By))
return true
}
if r == nil {
return true
}
s.step(j, "eval", Running, fmt.Sprintf("asking %s the cases of %s", target, r.Named()))
run, err := s.deps.Eval.Run(ctx, target, *r, j.Request.By, j.ID)
if err != nil {
s.fail(j, "eval", fmt.Errorf("%w; %s", err, refused()))
return false
}
run.Results = nil
s.mu.Lock()
j.Eval = &run
s.mu.Unlock()
detail := fmt.Sprintf("%s scored %.2f on %s, %d of %d cases passed; at least %.2f is asked", target,
run.Score, run.Suite, run.Cases-run.Below, run.Cases, run.Threshold)
if run.Error != "" {
detail += "; " + run.Error
}
if !run.Passed {
s.fail(j, "eval", fmt.Errorf("%s; %s. `llmgw ctl eval --run %s` shows each case", detail, refused(), run.ID))
return false
}
s.step(j, "eval", Done, detail)
return true
}
+184
View File
@@ -0,0 +1,184 @@
package modelserve
import (
"context"
"errors"
"strings"
"sync"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// fakeEval scores whatever it is asked what it is told to.
type fakeEval struct {
mu sync.Mutex
score float64
asked []string
}
func (f *fakeEval) Check(_ string, r eval.Request) error {
if r.Suite == "missing" {
return errors.New("no suite called missing")
}
return nil
}
func (f *fakeEval) Run(_ context.Context, model string, r eval.Request, by, job string) (store.EvalRun, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.asked = append(f.asked, model+" "+r.Named())
threshold := r.MinScore
if threshold == 0 {
threshold = 0.8
}
below := 0
if f.score < 1 {
below = 1
}
return store.EvalRun{ID: "eval-1", Model: "qwen", Target: model, Suite: r.Named(), Score: f.score,
Threshold: threshold, Passed: f.score >= threshold, Cases: 4, Below: below, By: by, Job: job}, nil
}
func (f *fakeEval) said() []string {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string{}, f.asked...)
}
func stepOf(j Job, name string) Step {
for _, s := range j.Steps {
if s.Name == name {
return s
}
}
return Step{}
}
func TestAPromoteBelowTheSuitesScoreIsRefusedAndTheCanaryStays(t *testing.T) {
s, models, _ := canarySetup(t)
ev := &fakeEval{score: 0.5}
s.deps.Eval = ev
job, _ := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester")
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
oldSource := models.models["qwen"].Source
gate := Change{Eval: &eval.Request{Suite: "arithmetic", MinScore: 0.8}}
job, err := s.StartPromote("qwen", "tester", gate)
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Failed || stepOf(j, "eval").State != Failed ||
!strings.Contains(j.Error, "qwen-canary scored 0.50 on arithmetic") ||
!strings.Contains(j.Error, "stays a canary") || stepOf(j, "catalog").Name != "" {
t.Fatalf("a canary scoring 0.5 of 0.8 was promoted, or not said so: %s %+v", j.Error, j.Steps)
}
if j.Eval == nil || j.Eval.Score != 0.5 || j.Eval.Job != j.ID {
t.Errorf("the job does not carry the score: %+v", j.Eval)
}
m := models.models["qwen"]
if m.Source != oldSource || m.Canary == nil || m.Canary.Percent != 10 {
t.Fatalf("the model moved, or lost its canary: %s %+v", m.Source, m.Canary)
}
if _, ok := models.models["qwen-canary"]; !ok {
t.Fatal("the canary was taken out; a refused promotion leaves that to its owner")
}
if got := ev.said(); len(got) != 1 || got[0] != "qwen-canary arithmetic" {
t.Errorf("asked %v", got)
}
ev.score = 0.9
job, _ = s.StartPromote("qwen", "tester", gate)
if j := wait(t, s, job.ID); j.State != Done || !strings.Contains(stepOf(j, "eval").Detail, "scored 0.90") {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
if m := models.models["qwen"]; m.Source == oldSource || m.Canary != nil {
t.Fatalf("a canary that passed was not promoted: %s %+v", m.Source, m.Canary)
}
}
func TestAnUpdateHeldToTheModelsSuiteTriesTheNewWeightsFirstAndKeepsTheOldBelowIt(t *testing.T) {
s, models, fleet := canarySetup(t)
ev := &fakeEval{score: 0.4}
s.deps.Eval = ev
m := models.models["qwen"]
m.Eval = &catalog.EvalGate{Suite: "arithmetic", MinScore: 0.75}
models.models["qwen"] = m
oldSource := m.Source
job, err := s.StartUpdate("qwen", "tester", Change{})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Failed || !strings.Contains(j.Error, "scored 0.40") || !strings.Contains(j.Error, "taken out again") {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
if got := ev.said(); len(got) != 1 || got[0] != "qwen-canary arithmetic" {
t.Fatalf("the new weights were not what was asked: %v", got)
}
m = models.models["qwen"]
if m.Source != oldSource || m.Canary != nil {
t.Fatalf("the model moved to weights that failed, or kept their canary: %s %+v", m.Source, m.Canary)
}
if _, ok := models.models["qwen-canary"]; ok {
t.Fatal("the failed weights were left in the catalog")
}
for _, r := range fleet.Replicas() {
if r.Model == "qwen" && strings.Contains(r.ID, "new") {
t.Fatalf("%s was replaced onto weights that failed", r.ID)
}
}
// Scoring enough, they replace the old.
ev.score = 1
job, _ = s.StartUpdate("qwen", "tester", Change{})
j = wait(t, s, job.ID)
if j.State != Done || stepOf(j, "eval").State != Done || stepOf(j, "split").Detail != "0% of qwen's requests go to qwen-canary" {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
if m := models.models["qwen"]; m.Source == oldSource || !strings.HasSuffix(m.Source, "/def98765/q4.gguf") ||
m.Canary != nil || m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource || m.Eval == nil {
t.Fatalf("not moved to the new weights, or the gate was lost: %s %+v %+v", m.Source, m.Canary, m.Origin)
}
if _, ok := models.models["qwen-canary"]; ok {
t.Error("the canary was left in the catalog")
}
}
func TestAModelsSuiteCanBeSkippedOnceAndAMissingOneIsRefusedAtOnce(t *testing.T) {
s, models, _ := canarySetup(t)
ev := &fakeEval{score: 0}
s.deps.Eval = ev
m := models.models["qwen"]
m.Eval = &catalog.EvalGate{Suite: "arithmetic"}
models.models["qwen"] = m
if _, err := s.StartUpdate("qwen", "tester", Change{Eval: &eval.Request{Suite: "missing"}}); err == nil ||
!strings.Contains(err.Error(), "no suite called missing") {
t.Fatalf("an update held to a suite that is not there was started: %v", err)
}
job, err := s.StartUpdate("qwen", "tester", Change{SkipEval: true})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done || stepOf(j, "eval").State != Skipped || !strings.Contains(stepOf(j, "eval").Detail, "tester") {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
if len(ev.said()) != 0 {
t.Errorf("asked %v while skipped", ev.said())
}
// Without anything to run suites, a gate is refused rather than ignored.
s.deps.Eval = nil
if _, err := s.StartUpdate("qwen", "tester", Change{}); err == nil {
t.Error("an update held to the model's suite started with nothing to run it")
}
}
+14
View File
@@ -33,6 +33,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Registry reads a repository.
@@ -95,6 +96,9 @@ type Deps struct {
// canary is never rolled back by itself.
CanaryStats func() map[string]canary.Stat
ResetCanary func(model string)
// Eval runs the quality suites an update or a promotion is held to;
// without it, one held to a suite is refused.
Eval Evaluator
}
// errBusy is a model the cluster has the hardware for, all of it in use now.
@@ -178,6 +182,12 @@ type Job struct {
// how fast it said it.
Reply string `json:"reply,omitempty"`
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
// trace is the job's spans, when tracing is on.
trace jobTrace
// Eval is the quality suite the new weights were asked before they
// replaced the old, and how they scored, without each case.
Eval *store.EvalRun `json:"eval,omitempty"`
}
// Server runs jobs and remembers them.
@@ -324,6 +334,7 @@ func (s *Server) snapshot(j *Job) Job {
func (s *Server) step(j *Job, name, state, detail string) {
s.mu.Lock()
defer s.mu.Unlock()
j.traceStep(name, state, detail)
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
return
@@ -335,6 +346,7 @@ func (s *Server) fail(j *Job, name string, err error) {
s.step(j, name, Failed, err.Error())
s.mu.Lock()
j.State, j.Error = Failed, err.Error()
j.traceEnd(err)
s.mu.Unlock()
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
}
@@ -623,6 +635,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
func (s *Server) finish(j *Job) {
s.mu.Lock()
j.State = Done
j.traceEnd(nil)
s.mu.Unlock()
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
}
@@ -791,6 +804,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
continue
}
if r.State == reconciler.Ready {
s.linkReplica(j, r)
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.Endpoint))
return r, nil
+99
View File
@@ -0,0 +1,99 @@
package modelserve
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// A job is a trace of its own, and each of its steps a span beneath it —
// registry, mirror, check, catalog, placement, smoke test — opened when the
// step begins and closed when it is done, failed or skipped. The steps are
// what a job already records for `ctl model jobs`; the trace adds where the
// time went, and from the placement step a link to the replica's own start,
// whose trace goes on down to the node.
//
// Begun by the first step rather than where a job is made, so every kind of
// job is traced by recording steps and nothing else.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
// jobTrace is a job's spans. Read and written with Server.mu held.
type jobTrace struct {
span trace.Span
step trace.Span
stepName string
}
// traceStep records a step on the job's trace. Called with s.mu held.
func (j *Job) traceStep(name, state, detail string) {
if j.trace.span == nil {
kind := j.Kind
if kind == "" {
kind = "serve"
}
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
trace.WithAttributes(
attribute.String("llmgw.job", j.ID),
attribute.String("llmgw.repo", j.Request.Repo),
attribute.String("llmgw.model", j.Model),
))
}
if j.trace.step != nil && j.trace.stepName != name {
// A step left running when the next began: over, whatever it said.
j.trace.step.End()
j.trace.step = nil
}
if j.trace.step == nil {
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
j.trace.stepName = name
}
span := j.trace.step
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
switch state {
case Running:
return
case Failed:
span.SetStatus(codes.Error, detail)
}
span.End()
j.trace.step = nil
}
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
// held.
func (j *Job) traceEnd(err error) {
if j.trace.step != nil {
j.trace.step.End()
j.trace.step = nil
}
if j.trace.span == nil {
return
}
if j.Model != "" {
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
}
obs.End(j.trace.span, err)
j.trace.span = nil
}
// linkReplica links the step in progress to the start of a replica it waited
// for, which is in the reconciler's trace rather than this one.
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
sc := r.StartTrace()
if !sc.IsValid() {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if j.trace.step != nil {
j.trace.step.AddLink(trace.Link{SpanContext: sc,
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
}
}
+65
View File
@@ -0,0 +1,65 @@
package modelserve
import (
"strings"
"testing"
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("job %s: %s", j.State, j.Error)
}
root := rec.One(t, [16]byte{}, "serve model")
tid := root.SpanContext().TraceID()
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
t.Errorf("the job's span does not say which job: %v", a)
}
var steps []string
for _, sp := range rec.Ended() {
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
steps = append(steps, sp.Name())
}
}
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
t.Errorf("step spans %v", steps)
}
smoke := rec.One(t, tid, "smoke test")
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
t.Errorf("the smoke test's span says %v", a)
}
}
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
if j := wait(t, s, job.ID); j.State != Failed {
t.Fatalf("job %s", j.State)
}
root := rec.One(t, [16]byte{}, "serve model")
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
t.Errorf("the failed job's span says %+v", root.Status())
}
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
if mirror.Status().Code != codes.Error {
t.Errorf("the step that failed is %+v", mirror.Status())
}
if len(rec.Named("catalog")) != 0 {
t.Error("a step after the failure was traced")
}
}
+55 -4
View File
@@ -11,6 +11,7 @@ import (
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
@@ -429,33 +430,79 @@ func (s *Server) current(name string) (catalog.Model, error) {
}
// StartUpdate moves a model to its repository's latest weights, as a job.
func (s *Server) StartUpdate(name, by string) (Job, error) {
// Held to a quality suite, the new weights run first as a canary given none
// of its requests, are asked the suite, and replace the old only at its
// score.
func (s *Server) StartUpdate(name, by string, ch Change) (Job, error) {
if s.deps.Models == nil || s.deps.Restart == nil {
return Job{}, errors.New("updating needs the store, which this control plane lacks")
}
if _, err := s.current(name); err != nil {
m, err := s.current(name)
if err != nil {
return Job{}, err
}
if s.deps.Models.Declared(name) {
return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+
"or they would revert on the next restart", name)
}
g, skipped := gateFor(m, ch)
if g != nil && m.Canary != nil {
return Job{}, fmt.Errorf("%s has a canary, %s: an update held to a quality suite tries the new weights as "+
"one. Promote it (`llmgw ctl model promote %s --eval ...`) or take it out first", name, m.Canary.Model, name)
}
if err := s.checkGate(canaryName(name), g); err != nil {
return Job{}, err
}
s.mu.Lock()
for _, j := range s.jobs {
if j.Kind == "update" && j.Model == name && j.State == Running {
s.mu.Unlock()
return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID)
}
if g != nil && (j.Kind == "canary" || j.Kind == "promote") && j.Model == name && j.State == Running {
s.mu.Unlock()
return Job{}, fmt.Errorf("%s has a job running: %s", name, j.ID)
}
}
s.seq++
j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name,
Request: Request{Name: name, By: by}, State: Running, Started: time.Now()}
s.keepJob(j)
s.mu.Unlock()
go s.runUpdate(j)
if g != nil {
go s.runGatedUpdate(j, g)
} else {
go s.runUpdate(j, skipped)
}
return s.snapshot(j), nil
}
// runGatedUpdate is an update held to a quality suite: the new weights as a
// canary with no share, asked the suite, then promoted or taken out.
func (s *Server) runGatedUpdate(j *Job, g *eval.Request) {
ctx := context.Background()
name := j.Model
p, ok := s.prepare(ctx, j, name)
if !ok {
return
}
cand, ok := s.addCandidate(ctx, j, p, catalog.Canary{Percent: 0})
if !ok {
return
}
if !s.gate(ctx, j, cand, g, false, func() string {
if _, err := s.AbortCanary(ctx, name, "llmgw"); err != nil {
return fmt.Sprintf("the new weights could not be taken out (%v): `llmgw ctl model canary %s --abort`", err, name)
}
return fmt.Sprintf("the new weights were taken out again; %s still runs %s", name, catalog.ShortRevision(p.o.Revision))
}) {
return
}
if s.promoteTo(ctx, j) {
s.finish(j)
}
}
// prepared is a model's new weights, fetched and checked: what an update and
// a canary both start from.
type prepared struct {
@@ -559,13 +606,14 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
return prepared{m: m, o: o, repo: repo, set: set, source: source}, true
}
func (s *Server) runUpdate(j *Job) {
func (s *Server) runUpdate(j *Job, skipped bool) {
ctx := context.Background()
name := j.Model
p, ok := s.prepare(ctx, j, name)
if !ok {
return
}
s.gate(ctx, j, name, nil, skipped, nil)
m, o, repo, set, source := p.m, p.o, p.repo, p.set, p.source
// 4. The catalog: the new source, and the old one kept to go back to.
@@ -636,6 +684,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
}
}
if left == 0 && len(ready) > 0 {
for _, r := range ready {
s.linkReplica(j, r)
}
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)
+5 -5
View File
@@ -205,7 +205,7 @@ func updateSays(u UpdateInfo) string {
func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) {
s, mir, models, _, fleet := updSetup(t)
oldSource := models.models["qwen"].Source
job, err := s.StartUpdate("qwen", "tester")
job, err := s.StartUpdate("qwen", "tester", Change{})
if err != nil {
t.Fatal(err)
}
@@ -243,7 +243,7 @@ func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T
}
// And forward again, from files already on the mirror.
job, err = s.StartUpdate("qwen", "tester")
job, err = s.StartUpdate("qwen", "tester", Change{})
if err != nil {
t.Fatal(err)
}
@@ -251,7 +251,7 @@ func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
// Then there is nothing newer: refused, saying why.
job, _ = s.StartUpdate("qwen", "tester")
job, _ = s.StartUpdate("qwen", "tester", Change{})
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") {
t.Fatalf("%+v", j)
}
@@ -260,7 +260,7 @@ func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T
func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) {
s, _, models, _, _ := updSetup(t)
models.declared["qwen"] = true
if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") {
if _, err := s.StartUpdate("qwen", "t", Change{}); err == nil || !strings.Contains(err.Error(), "server.yaml") {
t.Fatalf("%v", err)
}
}
@@ -268,7 +268,7 @@ func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) {
func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) {
s, _, _, reg, _ := updSetup(t)
reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}}
job, _ := s.StartUpdate("qwen", "t")
job, _ := s.StartUpdate("qwen", "t", Change{})
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") {
t.Fatalf("%+v", j)
}
+17
View File
@@ -323,6 +323,23 @@ func (s *Service) SetCanary(ctx context.Context, name string, c *catalog.Canary,
return final, err
}
// SetEval sets the quality suite a model's new weights must pass before an
// update or a promotion moves it to them; nil takes it away.
func (s *Service) SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "set its eval gate", func(m *catalog.Model) error {
m.Eval = g
return nil
})
if err == nil {
if g == nil {
s.log.Info("model eval gate removed", "model", name, "by", by)
} else {
s.log.Info("model eval gate set", "model", name, "suite", g.Suite, "min_score", g.MinScore, "by", by)
}
}
return final, err
}
// SetOrigin records where a model's weights came from without changing them:
// for a model added before origins were kept. A model from configuration is
// left alone, its origin worked out again when it is needed.
+87
View File
@@ -0,0 +1,87 @@
package placement
import (
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// migNode is an A100 80GB node as discovery reports it with card 0 cut into
// two 3g.40gb instances and card 1 whole.
func migNode(id string) inventory.Node {
card := discovery.GPU{Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920, ComputeCapability: "8.0"}
nd := inventory.Node{ID: id, State: inventory.Ready}
for dev, uuid := range []string{"MIG-c6d4f1ef", "MIG-0f7c3a2e"} {
g := card
g.Index, g.UUID, g.MemoryTotalMB = discovery.MIGIndex(0, dev), uuid, 40192
g.Name += " MIG 3g.40gb"
g.MIG = &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: dev, Profile: "3g.40gb"}
nd.GPUs = append(nd.GPUs, g)
}
whole := card
whole.Index, whole.UUID = 1, "GPU-whole"
// NVLink to the card in MIG mode, as topo -m still reports it: nothing
// may take that for a link an instance can use.
whole.NVLinkPeers = []int{0}
nd.GPUs = append(nd.GPUs, whole)
return nd
}
func TestAModelIsPlacedOnAMIGInstanceOfItsClass(t *testing.T) {
st := store(migNode("n1"))
s := New(st)
p := Profile{Name: "mig", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1}
res := s.Place([]Profile{p}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
got := res.Placement.Nodes[0].GPUIndices
if !slices.Equal(got, []int{1000}) {
t.Fatalf("placed on %v, want MIG instance 1000", got)
}
if res.Placement.Pool != "none/a100-80g-mig-3g.40gb" {
t.Errorf("pool %q", res.Placement.Pool)
}
// The next replica takes the other instance, and the one after finds
// none: the whole card beside them is another class.
st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: got})
res = s.Place([]Profile{p}, nil)
if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1001}) {
t.Fatalf("second replica: %+v\n%s", res.Placement, res.Explain())
}
st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{1001}})
if res = s.Place([]Profile{p}, nil); res.Placement != nil {
t.Errorf("a third replica was placed on %v", res.Placement.Nodes[0].GPUIndices)
}
// A whole-card profile takes the whole card, never an instance.
res = s.Place([]Profile{{Name: "whole", GPUClass: "a100-80g", GPUsPerNode: 1}}, nil)
if res.Placement == nil || !slices.Equal(res.Placement.Nodes[0].GPUIndices, []int{1}) {
t.Errorf("whole-card profile: %+v\n%s", res.Placement, res.Explain())
}
}
func TestMoreThanOneMIGInstanceForAReplicaIsRefused(t *testing.T) {
// Two instances are free on one node, which is what a tp 2 profile would
// otherwise be given — and fail at start, since a process opens only one
// MIG device and instances have no NVLink between them.
s := New(store(migNode("n1"), migNode("n2")))
for _, p := range []Profile{
{Name: "tp2", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2},
{Name: "two-nodes", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, Nodes: 2},
} {
res := s.Place([]Profile{p}, nil)
if res.Placement != nil {
t.Errorf("%s was placed: %+v", p.Name, res.Placement.Nodes)
continue
}
if !strings.Contains(res.Explain(), "a replica runs on exactly one") {
t.Errorf("%s refused without saying why:\n%s", p.Name, res.Explain())
}
}
}
+9
View File
@@ -156,6 +156,15 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
for _, raw := range ordered {
p := raw.normalize()
// The catalog refuses these; a profile that reaches here some other
// way is refused again rather than placed on instances that cannot
// work together. MIG instances are placed one to a replica.
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1) {
res.Rejections = append(res.Rejections, Rejection{Profile: p.Name, Pool: "none/" + p.GPUClass,
Reason: fmt.Sprintf("asks for %d MIG instances on %d nodes, but a replica runs on exactly one",
p.GPUsPerNode*p.Nodes, p.Nodes)})
continue
}
if p.Nodes == 1 {
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
res.Placement = pl
+11 -1
View File
@@ -6,6 +6,8 @@ import (
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
@@ -53,7 +55,15 @@ func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
r.log.Info("evicting an idle replica to make room",
"stopping", best.ID, "idle_model", best.Model,
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
r.stopReplica(ctx, best.ID)
ctx, span := startSpan(ctx, "evict",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.replica", best.ID),
attribute.String("llmgw.model", best.Model),
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
attribute.Int("llmgw.gpus", best.GPUs),
attribute.Int("llmgw.candidates", len(candidates)))
defer span.End()
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
return true
}
+16 -1
View File
@@ -2,10 +2,13 @@ package reconciler
import (
"context"
"fmt"
"slices"
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
@@ -113,12 +116,24 @@ func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []vict
r.preempting[m.Name] = ids
r.mu.Unlock()
victims := make([]string, 0, len(chosen))
for _, v := range chosen {
r.log.Warn("preempting a replica to make room",
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
"for", m.Name, "for_class", m.EffectiveClass())
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
if v.AboveMin {
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
}
victims = append(victims, desc)
}
r.scaleDown(ctx, ids)
ctx, span := startSpan(ctx, "preempt",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.for_class", m.EffectiveClass()),
attribute.StringSlice("llmgw.victims", victims),
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
defer span.End()
r.scaleDown(ctx, ids, "preempted for "+m.Name)
}
// awaitingPreemption reports whether replicas stopped for a model are still
+184 -25
View File
@@ -11,6 +11,10 @@ import (
"sync/atomic"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -18,6 +22,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// ShardCommand is everything an agent needs to launch one shard.
@@ -405,7 +410,9 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
if r.leader != nil && !r.leader.IsLeader() {
return
}
r.expireNodes()
ctx, p := beginPass(ctx)
defer p.end()
r.expireNodes(ctx)
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
@@ -452,7 +459,7 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
r.stopReplica(ctx, rep.ID, "its model left the catalog")
}
}
@@ -523,6 +530,7 @@ func (r *Reconciler) expireStuckReplicas() {
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
rep.startDone(now)
stuck = append(stuck, rep)
}
r.mu.Unlock()
@@ -555,7 +563,7 @@ func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
func (r *Reconciler) expireNodes(ctx context.Context) {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
@@ -563,14 +571,21 @@ func (r *Reconciler) expireNodes() {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
_, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID),
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
span.End()
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
_, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID),
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.notify.Load().Notify(notify.NodeLost, n.ID,
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed",
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")))
span.End()
}
}
}
@@ -609,24 +624,26 @@ func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
for _, rep := range toDrain {
r.log.Info("draining replica off a node being taken out of service",
"replica", rep.ID, "nodes", rep.NodeIDs())
for _, s := range shards[rep.ID] {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained")
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
// failReplicasOn fails every replica on a node, and says which.
func (r *Reconciler) failReplicasOn(nodeID, reason string) []string {
r.mu.Lock()
defer r.mu.Unlock()
var ids []string
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
rep.startDone(r.now())
ids = append(ids, rep.ID)
}
slices.Sort(ids)
return ids
}
// reapFailed releases the resources of terminal replicas and tells the agents
@@ -639,8 +656,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
doomed = append(doomed, rep)
}
}
parkedNow := map[string]bool{}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
rep.startDone(r.now())
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
@@ -652,6 +671,7 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
r.parked[rep.Model] = parkRecord{
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
}
parkedNow[rep.ID] = true
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
@@ -667,23 +687,52 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...)
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
attempts := r.attempts[rep.Model]
retry := r.backoff(attempts)
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
span.SetAttributes(
attribute.String("llmgw.error", rep.Err),
attribute.Int("llmgw.failures_in_a_row", attempts),
attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()),
attribute.Bool("llmgw.parked", parkedNow[rep.ID]),
)
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
span.End()
}
}
// drainShards tells a replica's nodes to stop routing to its shards, as one
// "drain replica" span saying why.
func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) {
nodes := make([]string, 0, len(shards))
for _, s := range shards {
nodes = append(nodes, s.NodeID)
}
ctx, span := startSpan(ctx, "drain replica",
attribute.String("llmgw.replica", id),
attribute.StringSlice("llmgw.nodes", nodes),
attribute.String("llmgw.reason", why))
defer span.End()
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
@@ -727,12 +776,14 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
}
}
// The next restart an operator queued, once the last one is done.
restarting := ""
if len(retiringIDs) == 0 {
if id, ok := r.nextRestartLocked(m.Name); ok {
r.replicas[id].RetiringAt = r.now()
retiringIDs = append(retiringIDs, id)
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
readyLive--
restarting = id
r.log.Info("restarting a replica: placing its replacement first",
"replica", id, "model", m.Name)
}
@@ -749,6 +800,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
r.log.Info("model unparked: its definition changed", "model", m.Name)
}
r.mu.Unlock()
if restarting != "" {
_, span := startSpan(ctx, "restart replica",
attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name))
span.End()
}
desired := r.desiredFor(m)
outcome := notTried
@@ -765,9 +821,20 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// the real error in noise.
return
}
outcome = r.scaleUp(ctx, m, avoid)
sctx, span := startSpan(ctx, "scale up",
attribute.String("llmgw.model", m.Name),
attribute.Int("llmgw.replicas", len(liveIDs)),
attribute.Int("llmgw.desired", desired))
outcome = r.scaleUp(sctx, m, avoid)
span.SetAttributes(attribute.String("llmgw.outcome", outcome.String()))
span.End()
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
sctx, span := startSpan(ctx, "scale down",
attribute.String("llmgw.model", m.Name),
attribute.Int("llmgw.replicas", len(liveIDs)),
attribute.Int("llmgw.desired", desired))
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
span.End()
}
// A drained replica is stopped once its replacement is serving, or once the
@@ -790,17 +857,17 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
switch {
case !ok:
case replaced:
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "drained, and its replacement serves")
case expired && preemptedFor != "":
r.log.Info("stopping a preempted replica", "replica", id,
"for", preemptedFor, "after", grace)
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over")
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", grace)
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String())
}
}
@@ -813,6 +880,9 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
_, pspan := startSpan(ctx, "place",
attribute.String("llmgw.model", m.Name),
attribute.StringSlice("llmgw.avoid", avoid))
profiles := r.orderedProfiles(m)
res := r.sched.Place(profiles, avoid)
if res.Placement == nil && len(avoid) > 0 {
@@ -820,7 +890,10 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(profiles, nil)
pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil))
}
placementAttrs(pspan, res)
pspan.End()
r.mu.Lock()
r.lastExplain[m.Name] = res
@@ -852,10 +925,12 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
{Profile: res.Placement.Profile.Name, Reason: why}}}
r.mu.Unlock()
r.notePlacementFailure(m.Name, why)
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why))
return overLimit
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error())
return noRoom
}
return placed
@@ -872,6 +947,51 @@ const (
overLimit // there is room, but the owner's quota does not allow it
)
// String names an outcome for a trace.
func (o scaleOutcome) String() string {
switch o {
case placed:
return "placed"
case makingRoom:
return "making room"
case noRoom:
return "no room"
case overLimit:
return "over its quota"
default:
return "not tried"
}
}
// placementAttrs puts a placement's answer on its span: where it went, or
// every candidate it turned down and why, which is what `ctl
// explain-placement` would have printed at that moment.
func placementAttrs(span trace.Span, res placement.Result) {
span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections)))
pl := res.Placement
if pl == nil {
explain := res.Explain()
if len(explain) > 4096 {
explain = explain[:4096] + "…"
}
span.SetAttributes(attribute.String("llmgw.explain", explain))
span.SetStatus(codes.Error, "fits nowhere")
return
}
nodes := make([]string, 0, len(pl.Nodes))
for _, a := range pl.Nodes {
nodes = append(nodes, a.NodeID)
}
span.SetAttributes(
attribute.String("llmgw.profile", pl.Profile.Name),
attribute.String("llmgw.engine", pl.Profile.Engine),
attribute.String("llmgw.gpu_class", pl.Profile.GPUClass),
attribute.String("llmgw.pool", pl.Pool),
attribute.String("llmgw.fabric", pl.Fabric),
attribute.StringSlice("llmgw.nodes", nodes),
)
}
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
@@ -936,6 +1056,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
// Open until the replica serves, fails or is stopped, and the parent of
// every node's part of the start: the agents' spans arrive under it.
rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep),
attribute.String("llmgw.engine", prof.Engine),
attribute.String("llmgw.gpu_class", prof.GPUClass),
attribute.String("llmgw.pool", pl.Pool),
attribute.String("llmgw.fabric", pl.Fabric),
attribute.String("llmgw.endpoint", rep.Endpoint),
attribute.Int("llmgw.attempt", attempt),
attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()),
)...)
rep.startSpan, rep.startTrace = rspan, rspan.SpanContext()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
@@ -965,7 +1098,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
Fabric: pl.Fabric,
ProfileFP: prof.Fingerprint(),
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
// The command, sent: its context travels with it to the agent.
sctx, sspan := tracer().Start(rctx, "send start",
trace.WithSpanKind(trace.SpanKindProducer),
trace.WithAttributes(
attribute.String("llmgw.shard", s.ID),
attribute.String("llmgw.node", s.NodeID),
attribute.Int("llmgw.rank", s.Rank),
attribute.IntSlice("llmgw.gpus", s.GPUIndices),
attribute.Int("llmgw.port", s.HTTPPort),
))
err := r.disp.StartShard(sctx, cmd)
obs.End(sspan, err)
if err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
@@ -1031,7 +1176,8 @@ func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
// scaleDown drains replicas; why is for the trace.
func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
@@ -1045,15 +1191,12 @@ func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
r.drainShards(ctx, id, shards, why)
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
// stopReplica stops a replica's shards and forgets it; why is for the trace.
func (r *Reconciler) stopReplica(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
@@ -1062,6 +1205,10 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
rep.startDone(r.now())
ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep),
attribute.String("llmgw.reason", why))...)
defer span.End()
r.mu.Unlock()
for _, s := range shards {
@@ -1130,6 +1277,18 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
prev := rep.State
rep.deriveState(r.now())
if rep.startSpan != nil {
ev := []attribute.KeyValue{
attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()),
}
if errMsg != "" {
ev = append(ev, attribute.String("llmgw.error", errMsg))
}
rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...))
}
if rep.State == Ready || rep.State.Terminal() {
rep.startDone(r.now())
}
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
@@ -11,6 +11,8 @@ import (
"testing"
"time"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -24,9 +26,12 @@ type fakeDispatcher struct {
stopped []string
drained []string
failNext error
// traces are the span each start was sent under, which is what the
// real dispatcher puts on the command for the agent.
traces []trace.SpanContext
}
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.failNext != nil {
@@ -35,6 +40,7 @@ func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
return err
}
d.started = append(d.started, cmd)
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
return nil
}
+13
View File
@@ -7,6 +7,8 @@ import (
"slices"
"time"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
@@ -166,8 +168,19 @@ type Replica struct {
// carried across rebuilds so the backoff does not reset on every attempt.
Attempt int
Err string
// startSpan is the "replica start" span, open from placement until the
// replica serves, fails or is stopped; nil once closed, and for a replica
// adopted rather than started. startTrace names it for good.
startSpan trace.Span
startTrace trace.SpanContext
}
// StartTrace names the span of this replica's start, for something that
// waited on it — a job serving a model — to link to. Invalid for a replica
// adopted rather than started, and whenever tracing is off.
func (r Replica) StartTrace() trace.SpanContext { return r.startTrace }
// NodeIDs lists the nodes this replica occupies.
func (r Replica) NodeIDs() []string {
out := make([]string, 0, len(r.Shards))
Loaded 100 of 201 files, more files were not shown because too many files have changed in this diff. Show more