llmgw/internal/agent/provision.go

package agent

import (
	"context"
	"fmt"
	"log/slog"
	"maps"
	"slices"
	"strings"
	"time"

	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
)

// engines is what this node can run: what the agent installed and saw run,
// and what the configuration names and answers. Nothing merely found on the
// machine — a llama-server on PATH is somebody's, not necessarily one that
// works, and placing on it is how a replica fails on a node nobody watches.
func (a *Agent) engines(ctx context.Context) map[string]string {
	a.engineOnce.Do(func() {
		vers := a.reg.ProbeConfigured(ctx)
		a.mu.Lock()
		a.configuredVers, a.lastEngineProbe = vers, time.Now()
		a.mu.Unlock()
		if len(vers) > 0 {
			a.log.Info("engines named in the configuration",
				"engines", engines.ProbedNames(vers))
		}
	})
	a.mu.Lock()
	defer a.mu.Unlock()
	vers := a.engineReportLocked()
	// Whatever registration sends is what the control plane now knows, so a
	// heartbeat repeats it only once it differs.
	a.enginesSent = enginesFingerprint(vers)
	return vers
}

// engineReportLocked merges the two sources. Installed wins: it is the one
// that was seen to run. Called with mu held.
func (a *Agent) engineReportLocked() map[string]string {
	out := maps.Clone(a.configuredVers)
	if out == nil {
		out = map[string]string{}
	}
	maps.Copy(out, a.installedVers)
	maps.DeleteFunc(out, func(e, _ string) bool { return a.disabled(e) })
	return out
}

// byHand reports whether agent.yaml names this engine's binary, which opts
// it out of being installed.
func (a *Agent) byHand(engine string) bool {
	if a.reg == nil {
		return false
	}
	e, err := a.reg.Get(engine)
	if err != nil {
		return false
	}
	c, ok := e.(engines.Configured)
	return ok && c.Configured()
}

// disabled reports whether agent.yaml says this node never runs an engine:
// not installed when asked, and not advertised when found.
func (a *Agent) disabled(engine string) bool {
	return slices.Contains(a.cfg.Engines.Disabled, engine)
}

// provisionEvery is how often installed engines are checked again. A library
// removed by an unrelated upgrade is found here rather than by the next
// replica to start.
const provisionEvery = 10 * time.Minute

// setProvision takes the control plane's list of wanted engines.
func (a *Agent) setProvision(p *llmgwv1.Provision) {
	wants := []provision.Want{}
	for _, e := range p.GetEngines() {
		if a.disabled(e.GetEngine()) {
			continue // a control plane older than the node's say in it
		}
		if a.byHand(e.GetEngine()) {
			// The operator named a binary for it: a local build, a CUDA
			// runtime of their own. Installing one beside it would win —
			// the installed build is used first — and quietly replace theirs.
			continue
		}
		wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()})
	}
	a.mu.Lock()
	a.provWants = wants
	a.mu.Unlock()
	select {
	case a.provKick <- struct{}{}:
	default:
	}
}

// provisionLoop installs and re-checks the wanted engines, one pass at a time.
func (a *Agent) provisionLoop(ctx context.Context) {
	// After a failure the next pass comes sooner, and backs off: a lock held
	// by a new machine's own updates clears in a minute or two, while ten
	// minutes' wait left a node idle for no reason.
	next := provisionEvery
	retry := time.Minute
	for {
		select {
		case <-ctx.Done():
			return
		case <-a.provKick:
		case <-time.After(next):
		}
		a.mu.Lock()
		wants := slices.Clone(a.provWants)
		managed := a.provWants != nil
		a.mu.Unlock()
		if !managed {
			continue
		}

		statuses := a.prov.Converge(ctx, wants)
		next = provisionEvery
		for _, st := range statuses {
			if st.State == provision.Failed {
				next, retry = retry, min(retry*2, provisionEvery)
				break
			}
		}
		if next == provisionEvery {
			retry = time.Minute
		}
		installed := map[string]string{}
		keep := map[string]bool{}
		for _, st := range statuses {
			keep[st.Engine] = true
			if st.State == provision.Ready {
				a.reg.UseInstalled(st.Engine, st.Binary)
				installed[st.Engine] = st.Version
			}
		}
		a.mu.Lock()
		if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
			a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
		}
		a.installedVers = installed
		// An engine no longer asked for is no longer reported on.
		for e := range a.provStatus {
			if !keep[e] {
				delete(a.provStatus, e)
			}
		}
		a.mu.Unlock()
	}
}

// provisionStatus records a change in one engine's installation.
func (a *Agent) provisionStatus(st provision.Status) {
	a.mu.Lock()
	prev, had := a.provStatus[st.Engine]
	a.provStatus[st.Engine] = st
	a.mu.Unlock()
	if !had || prev.State != st.State || prev.Detail != st.Detail {
		lvl := slog.LevelInfo
		if st.State == provision.Failed || st.State == provision.Unsupported {
			lvl = slog.LevelWarn
		}
		a.log.Log(context.Background(), lvl, "engine installation",
			"engine", st.Engine, "version", st.Version, "variant", st.Variant,
			"state", st.State, "detail", st.Detail)
	}
}

func provisionFingerprint(m map[string]provision.Status) string {
	var b strings.Builder
	for _, k := range slices.Sorted(maps.Keys(m)) {
		st := m[k]
		fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
	}
	return b.String()
}

// engineReprobe is how often the installed engines are looked for again.
//
// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
// minute's delay in noticing is invisible, while SGLang's probe starts a
// Python interpreter that imports it.
const engineReprobe = time.Minute

// reprobeEngines looks for installed engines again when it is time, off the
// heartbeat goroutine so a slow probe never delays a heartbeat.
func (a *Agent) reprobeEngines() {
	a.mu.Lock()
	due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
		time.Since(a.lastEngineProbe) >= engineReprobe
	if due {
		a.probingEngines = true
	}
	a.mu.Unlock()
	if !due {
		return
	}
	go func() {
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		vers := a.reg.ProbeConfigured(ctx)

		a.mu.Lock()
		changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
		a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
		a.mu.Unlock()
		if changed {
			a.log.Info("engines named in the configuration changed",
				"engines", engines.ProbedNames(vers))
		}
	}()
}

func enginesFingerprint(vers map[string]string) string {
	var b strings.Builder
	for _, k := range slices.Sorted(maps.Keys(vers)) {
		fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
	}
	return b.String()
}