llmgw/internal/onboard/onboard.go

// Package onboard makes a machine a node of the cluster, over SSH, from the
// machine holding the certificate authority.
//
// It is what deploy/bootstrap/onboard-node.sh did, moved into the binary so
// there is nothing else to fetch, and extended by what a script could not
// easily do: it asks the machine its own name and architecture instead of
// being told, runs llmgw doctor there before changing anything, and suggests
// the fabric its adapters are on.
//
// It reports each step as an Event, so a plain terminal and a full-screen one
// show the same thing from the same code.
package onboard

import (
	"context"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/release"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
	"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
	"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)

// Fabric is one InfiniBand or RoCE fabric the node is on, and its adapters.
type Fabric struct {
	Name string
	HCAs []string
	// Key is what the adapters report the fabric to be — its subnet manager,
	// or a RoCE network — on a suggestion, so a name given for one node can
	// be given again to the next on the same fabric.
	Key string
}

// ParseFabric reads "name:hca1,hca2", or "name" alone.
func ParseFabric(s string) (Fabric, error) {
	name, hcas, _ := strings.Cut(strings.TrimSpace(s), ":")
	if name == "" {
		return Fabric{}, fmt.Errorf("fabric %q has no name: give it as name:mlx5_0,mlx5_1", s)
	}
	f := Fabric{Name: name}
	for _, h := range strings.Split(hcas, ",") {
		if h = strings.TrimSpace(h); h != "" {
			f.HCAs = append(f.HCAs, h)
		}
	}
	return f, nil
}

// Options say which machine, and what to tell its agent.
type Options struct {
	// NoNodeExporter leaves Prometheus' node exporter off the machine.
	NoNodeExporter bool
	// NoFirewall leaves the machine's firewall as it is. Otherwise its llmgw
	// ports — FirewallPorts — answer only the networks it reaches the
	// control planes on, and FirewallAllow.
	NoFirewall bool
	// MirrorBase is where the mirror serves model files, http://host:8081:
	// the node is asked to reach it before it is added, since a node that
	// cannot fetches no model.
	MirrorBase    string
	FirewallPorts []firewall.Port
	FirewallAllow []netip.Prefix

	// Addr is how to reach the machine over SSH, and User who to log in as.
	Addr, User string
	// Node is its id in the cluster. Empty means its hostname, which is what
	// it should be: the node id, the hostname and the certificate's name
	// being one string saves an afternoon each time they would differ.
	Node string
	// Servers are the control planes the agent dials. Empty means this
	// machine, at the address it has on the route to the node.
	Servers []string
	Fabrics []Fabric

	MgmtAddr, MgmtIface string
	NvidiaSMI, IBRoot   string
	LlamaCppBin         string
	VLLMBin             string
	SGLangPython        string

	// PKI is the authority's directory; EtcDir is where this machine's own
	// server certificate is, to check the chosen address is named in it.
	PKI, EtcDir string
	// Binary is the llmgw to install when the node cannot download the
	// release, and Version what it is.
	Binary, Version string
	// Releases is where release files are downloaded from.
	Releases string
	// ReleaseClient fetches them; nil is a default client.
	ReleaseClient *http.Client
	// IgnoreChecks goes on when doctor finds a problem.
	IgnoreChecks bool
	// DisableEngines are engines this node never installs or advertises,
	// written into its agent.yaml, and not checked by doctor: vLLM on a node
	// of cards it does not run on, meant for llama.cpp alone. Narrower than
	// IgnoreChecks, which also passes over everything else.
	DisableEngines []string
	// VLLMVersion is the vLLM the control plane has nodes install, for
	// doctor to check against; "none" when it has them install none, empty
	// for doctor's own default.
	VLLMVersion string
	// SSHWait is how long an unreachable machine is waited for.
	SSHWait time.Duration

	// AskFabrics, when set, is asked which fabrics to declare for a node
	// given none whose adapters are up, with the adapters grouped by the
	// fabric they report. It returns the fabrics to use, none to go on
	// without. Nil leaves it at a warning, as a script needs.
	AskFabrics func(suggested []Fabric) []Fabric
	// AskContinue, when set, is asked whether to go on after doctor found
	// problems. Nil stops unless IgnoreChecks.
	AskContinue func(fails int) bool
}

// Levels of an Event.
const (
	Running = "running"
	Done    = "done"
	Info    = "info"
	Warn    = "warn"
	Fail    = "fail"
)

// Event is one thing that happened: a step starting or ending, or something
// worth saying along the way.
type Event struct {
	Step   string
	Level  string
	Detail string
	// Fix is what to do about a warning or a failure, when there is a known
	// answer.
	Fix string
}

// The steps, in order.
const (
	StepConnect = "connect"
	StepCheck   = "check"
	StepBinary  = "binary"
	StepCert    = "certificate"
	StepConfig  = "configuration"
	StepStart   = "start"
	// StepJoin and StepEngines are watched from the control plane once the
	// agent runs; the caller does that, since it has the admin API.
	StepJoin    = "join"
	StepEngines = "engines"
)

// Steps lists them for a display that shows what is still to come.
var Steps = []string{StepConnect, StepBinary, StepCheck, StepCert, StepConfig, StepStart, StepJoin, StepEngines}

// Result is what onboarding learned and did.
type Result struct {
	Node     string
	Arch     string
	Hostname string
	Servers  []string
	Facts    doctor.Facts
	Findings []doctor.Finding
	// Suggested is the fabrics the node's active adapters are on, for a node
	// given none: grouped by what the adapters report, named for the operator
	// to choose.
	Suggested []Fabric
	// EngineProblems are engines named by path that will not run there.
	EngineProblems []string
}

// Runner onboards one machine.
type Runner struct {
	Opts   Options
	Remote Remote
	// Emit hears every event. Nil discards them.
	Emit func(Event)

	ca *tlsconf.CA
}

// New makes a Runner that reaches the machine with the system's ssh.
func New(o Options, emit func(Event)) *Runner {
	return &Runner{Opts: o, Remote: &SSH{Addr: o.Addr, User: o.User}, Emit: emit}
}

func (r *Runner) emit(step, level, detail, fix string) {
	if r.Emit != nil {
		r.Emit(Event{Step: step, Level: level, Detail: detail, Fix: fix})
	}
}

// ErrChecks is returned when doctor found problems and IgnoreChecks is off.
var ErrChecks = errors.New("the node has problems to fix first")

// Run does every step, stopping at the first that fails.
func (r *Runner) Run(ctx context.Context) (*Result, error) {
	res := &Result{}
	// The authority first: without it nothing can be finished, and finding
	// out after the node has been changed leaves it half done.
	pki := r.Opts.PKI
	ca, err := tlsconf.LoadCA(filepath.Join(pki, "ca.pem"), filepath.Join(pki, "ca-key.pem"))
	if err != nil {
		err = fmt.Errorf("cannot read the certificate authority in %s: %v. Run this on the machine that holds it, as root (sudo), or give --pki", pki, err)
		r.emit(StepConnect, Fail, err.Error(), "")
		return res, err
	}
	r.ca = ca
	for _, step := range []struct {
		name string
		fn   func(context.Context, *Result) error
	}{
		{StepConnect, r.connect},
		{StepBinary, r.binary},
		{StepCheck, r.check},
		{StepCert, r.certificate},
		{StepConfig, r.configure},
		{StepStart, r.start},
	} {
		if err := step.fn(ctx, res); err != nil {
			r.emit(step.name, Fail, err.Error(), "")
			return res, err
		}
	}
	return res, nil
}

// --- connect ---

var nodeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`)

func (r *Runner) connect(ctx context.Context, res *Result) error {
	r.emit(StepConnect, Running, "connecting to "+r.target(), "")
	wait := r.Opts.SSHWait
	if wait == 0 {
		wait = 3 * time.Minute
	}
	// Waited for rather than failed on: a machine a minute old restarts sshd
	// after cloud-init regenerates its host keys, and refuses connections for
	// a few seconds after answering.
	deadline := time.Now().Add(wait)
	var last error
	for {
		out, err := r.Remote.Run(ctx, "hostname && uname -m", nil)
		if err == nil {
			lines := strings.Fields(string(out))
			if len(lines) >= 2 {
				res.Hostname, res.Arch = lines[0], lines[1]
				break
			}
			err = fmt.Errorf("unexpected answer: %q", out)
		}
		last = err
		if time.Now().After(deadline) {
			return fmt.Errorf("cannot log in to %s: %v. Check that `ssh %s true` works from this machine, as the user running this, without a password",
				r.target(), last, r.target())
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(5 * time.Second):
		}
	}
	if _, err := r.Remote.Run(ctx, "sudo -n true", nil); err != nil {
		return fmt.Errorf("%s can log in but cannot use sudo without a password, which installing the agent needs", r.target())
	}
	switch res.Arch {
	case "x86_64", "amd64":
		res.Arch = "amd64"
	case "aarch64", "arm64":
		res.Arch = "arm64"
	default:
		return fmt.Errorf("%s is %s; llmgw is built for x86-64 and ARM64", r.target(), res.Arch)
	}
	res.Node = r.Opts.Node
	if res.Node == "" {
		res.Node = res.Hostname
	}
	if !nodeName.MatchString(res.Node) {
		return fmt.Errorf("%q cannot be a node id: letters, digits, dots, dashes and underscores; give one with --node", res.Node)
	}
	servers, err := r.servers()
	if err != nil {
		return err
	}
	res.Servers = servers
	r.emit(StepConnect, Done, fmt.Sprintf("%s (%s), joining as %s, agent will dial %s",
		res.Hostname, res.Arch, res.Node, strings.Join(servers, ", ")), "")
	return nil
}

func (r *Runner) target() string {
	if r.Opts.User == "" {
		return r.Opts.Addr
	}
	return r.Opts.User + "@" + r.Opts.Addr
}

// servers are the given control planes, or this machine at the address it
// uses to reach the node — which is the one the node can reach it at, and
// is named in the server certificate when init issued it here.
func (r *Runner) servers() ([]string, error) {
	if len(r.Opts.Servers) > 0 {
		return r.Opts.Servers, nil
	}
	host := r.Opts.Addr
	if h, _, err := net.SplitHostPort(host); err == nil {
		host = h
	}
	c, err := net.Dial("udp", net.JoinHostPort(host, "7443"))
	if err != nil {
		return nil, fmt.Errorf("cannot tell which address the node would reach this machine at: %v; give it with --server", err)
	}
	local := c.LocalAddr().(*net.UDPAddr).IP.String()
	_ = c.Close()
	if names, err := certNames(filepath.Join(r.etcDir(), "server.pem")); err == nil && !slices.Contains(names, local) {
		return nil, fmt.Errorf("the node would reach this machine at %s, which the server certificate does not name (%s): "+
			"run `sudo llmgw server init --host %s` to add it, or give the address with --server",
			local, strings.Join(names, ", "), local)
	}
	return []string{"https://" + net.JoinHostPort(local, "7443")}, nil
}

func (r *Runner) etcDir() string {
	if r.Opts.EtcDir != "" {
		return r.Opts.EtcDir
	}
	return "/etc/llmgw"
}

func certNames(path string) ([]string, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	blk, _ := pem.Decode(b)
	if blk == nil {
		return nil, errors.New("no certificate in " + path)
	}
	c, err := x509.ParseCertificate(blk.Bytes)
	if err != nil {
		return nil, err
	}
	names := slices.Clone(c.DNSNames)
	for _, ip := range c.IPAddresses {
		names = append(names, ip.String())
	}
	return names, nil
}

// --- binary ---

// releaseTag is a version CI published: v0.10.4, or a release candidate of
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
// release to download.
var releaseTag = release.Tag

func (r *Runner) binary(ctx context.Context, res *Result) error {
	v := r.Opts.Version
	if releaseTag.MatchString(v) {
		// Fetched here, for the node's architecture, and checked here —
		// signature and checksum, by the same code that upgrades this
		// machine — then handed to the node. The node checking a release
		// itself could only compare it with a checksum from the same
		// server, which proves nothing about who put both there.
		r.emit(StepBinary, Running, "downloading llmgw "+v+" for "+res.Arch+" and checking it", "")
		b, err := release.Fetch(ctx, r.Opts.ReleaseClient, r.Opts.Releases, v, "linux", goArch(res.Arch))
		if err == nil {
			if err := r.installBinary(ctx, b); err != nil {
				return err
			}
			what := "checksum verified"
			if release.TrustedKey() != "" {
				what = "signature and checksum verified"
			}
			r.emit(StepBinary, Done, "llmgw "+v+" from the release, "+what, "")
			return nil
		}
		r.emit(StepBinary, Warn, "the release could not be fetched and checked here: "+err.Error(), "")
	}
	// A development build has no release to fetch, and a node that cannot
	// reach the releases gets this machine's copy — when it can run it.
	if res.Arch != hostArch() {
		return fmt.Errorf("the node is %s and this machine %s, so its binary cannot be copied: "+
			"let the node reach %s, or install llmgw there yourself", res.Arch, hostArch(), r.Opts.Releases)
	}
	r.emit(StepBinary, Running, "copying this machine's llmgw ("+orUnknown(v)+")", "")
	b, err := os.ReadFile(r.Opts.Binary)
	if err != nil {
		return fmt.Errorf("reading %s: %w", r.Opts.Binary, err)
	}
	if err := r.installBinary(ctx, b); err != nil {
		return err
	}
	r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "")
	return nil
}

// installBinary puts bytes at /usr/local/bin/llmgw on the node.
func (r *Runner) installBinary(ctx context.Context, b []byte) error {
	if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
		return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out))
	}
	return nil
}

// goArch is Go's name for what uname -m says.
func goArch(uname string) string {
	switch uname {
	case "x86_64", "amd64":
		return "amd64"
	case "aarch64", "arm64":
		return "arm64"
	}
	return uname
}

// --- check ---

func (r *Runner) check(ctx context.Context, res *Result) error {
	r.emit(StepCheck, Running, "running llmgw doctor on the node", "")
	args := "sudo llmgw doctor --json"
	if r.Opts.NvidiaSMI != "" {
		args += " --nvidia-smi " + shellQuote(r.Opts.NvidiaSMI)
	}
	if r.Opts.IBRoot != "" {
		args += " --ib-root " + shellQuote(r.Opts.IBRoot)
	}
	// The fabrics given, checked as the ones they will be; any adapter they
	// do not name is checked as one nothing uses.
	for _, fab := range r.Opts.Fabrics {
		args += " --fabric " + shellQuote(fab.Name+":"+strings.Join(fab.HCAs, ","))
	}
	switch {
	case slices.Contains(r.Opts.DisableEngines, "vllm"):
		args += " --vllm none"
		r.emit(StepCheck, Info, "vLLM is disabled on this node: it joins with llama.cpp alone", "")
	case r.Opts.VLLMVersion != "":
		args += " --vllm " + shellQuote(r.Opts.VLLMVersion)
	}
	out, err := r.Remote.Run(ctx, args, nil)
	var rep struct {
		Findings []doctor.Finding `json:"findings"`
		Facts    doctor.Facts     `json:"facts"`
	}
	if jerr := json.Unmarshal(out, &rep); jerr != nil {
		if err == nil {
			err = jerr
		}
		return fmt.Errorf("llmgw doctor did not run on the node: %v: %s", err, lastLine(out))
	}
	if !r.Opts.NoFirewall && len(r.Opts.FirewallPorts) > 0 {
		// A missing firewall is what this run is about to write.
		rep.Findings = slices.DeleteFunc(rep.Findings, func(f doctor.Finding) bool { return f.Area == "firewall" })
	}
	if f, ok := r.mirrorReach(ctx); ok {
		rep.Findings = append(rep.Findings, f)
	}
	res.Facts, res.Findings = rep.Facts, rep.Findings

	fails := 0
	for _, f := range rep.Findings {
		switch f.Level {
		case doctor.Fail:
			fails++
			r.emit(StepCheck, Fail, f.Area+": "+f.Message, f.Fix)
		case doctor.Warn:
			r.emit(StepCheck, Warn, f.Area+": "+f.Message, f.Fix)
		case doctor.OK:
			r.emit(StepCheck, Info, f.Area+": "+f.Message, "")
		}
	}
	res.Suggested = suggestFabrics(rep.Facts.HCAs)
	if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 && r.Opts.AskFabrics != nil {
		r.Opts.Fabrics = r.Opts.AskFabrics(res.Suggested)
		for _, f := range r.Opts.Fabrics {
			r.emit(StepCheck, Info, "fabric "+f.Name+": "+strings.Join(f.HCAs, ", "), "")
		}
	}
	if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 {
		var flags []string
		for _, f := range res.Suggested {
			flags = append(flags, "--fabric "+f.Name+":"+strings.Join(f.HCAs, ","))
		}
		r.emit(StepCheck, Warn, "InfiniBand or RoCE adapters are up, but no fabric was given: this node would never be given a replica spanning nodes",
			"add "+strings.Join(flags, " ")+", naming each fabric the same on every node attached to it")
	}
	goOn := r.Opts.IgnoreChecks
	if fails > 0 && !goOn && r.Opts.AskContinue != nil && r.Opts.AskContinue(fails) {
		r.emit(StepCheck, Warn, fmt.Sprintf("going on despite %d problem(s)", fails), "")
		goOn = true
	}
	if fails > 0 && !goOn {
		return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails)
	}
	r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "")
	return nil
}

// mirrorReach asks the node to reach the mirror's files: a base address the
// node cannot reach (another machine's interface, Docker's bridge, a port
// closed to it) otherwise shows up as every model failing to download.
func (r *Runner) mirrorReach(ctx context.Context) (doctor.Finding, bool) {
	if r.Opts.MirrorBase == "" {
		return doctor.Finding{}, false
	}
	u, err := url.Parse(r.Opts.MirrorBase)
	if err != nil || u.Hostname() == "" {
		return doctor.Finding{}, false
	}
	port := u.Port()
	if port == "" {
		port = "80"
	}
	base := strings.TrimRight(r.Opts.MirrorBase, "/") + "/"
	out, _ := r.Remote.Run(ctx, `if command -v curl >/dev/null 2>&1; then
  curl -s -o /dev/null -w '%{http_code}' --max-time 5 `+shellQuote(base)+` || true
else
  timeout 5 bash -c `+shellQuote("</dev/tcp/"+u.Hostname()+"/"+port)+` 2>/dev/null && echo 200 || echo 000
fi`, nil)
	code := strings.TrimSpace(lastLine(out))
	if code != "" && code != "000" {
		return doctor.Finding{Level: doctor.OK, Area: "mirror", Message: "reaches the mirror's files at " + base}, true
	}
	return doctor.Finding{Level: doctor.Fail, Area: "mirror",
		Message: "cannot reach the mirror's files at " + base + ": every model it is given would fail to download",
		Fix: "on the mirror's machine, give llmgw-mirror a --base this node reaches (systemctl cat llmgw-mirror), " +
			"and let this node through to port " + port}, true
}

// suggestFabrics groups the active adapters by the fabric they report being
// on: the subnet manager they see, or the network of their RoCE address.
// Names are placeholders for the operator: a name is what ties one node's
// adapters to another's, and nothing on one node can know what the others
// were called.
func suggestFabrics(hcas []discovery.HCA) []Fabric {
	byKey := map[string][]string{}
	var keys []string
	for _, h := range hcas {
		for _, p := range h.ActivePorts() {
			k := p.FabricKey()
			if k == "" {
				k = "unknown"
			}
			if _, ok := byKey[k]; !ok {
				keys = append(keys, k)
			}
			if !slices.Contains(byKey[k], h.Name) {
				byKey[k] = append(byKey[k], h.Name)
			}
		}
	}
	var out []Fabric
	for i, k := range keys {
		name := "fabric"
		if len(keys) > 1 {
			name = fmt.Sprintf("fabric%d", i+1)
		}
		out = append(out, Fabric{Name: name, HCAs: byKey[k], Key: k})
	}
	return out
}

// --- certificate ---

func (r *Runner) certificate(ctx context.Context, res *Result) error {
	r.emit(StepCert, Running, "issuing a certificate for "+res.Node, "")
	ca := r.ca
	// Issued fresh every time: onboarding again should not depend on finding
	// an old key, and the old one keeps working until revoked.
	leaf, err := ca.IssueClient(res.Node)
	if err != nil {
		return err
	}
	for _, f := range []struct {
		path, mode string
		body       []byte
	}{
		{"/etc/llmgw/ca.pem", "0644", ca.PEM},
		{"/etc/llmgw/agent.pem", "0644", leaf.CertPEM},
		{"/etc/llmgw/agent-key.pem", "0600", leaf.KeyPEM},
	} {
		if err := r.put(ctx, f.path, f.mode, f.body); err != nil {
			return err
		}
	}
	// The agent trusts the cluster's root through its own configuration only.
	// Not system-wide: that root signs whatever the control plane asks it to,
	// and every TLS client on the machine would then take it for any site.
	// A node onboarded by an older version had it added; take it back out.
	_, _ = r.Remote.Run(ctx, `if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then
  sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt && sudo update-ca-certificates --fresh >/dev/null 2>&1
fi
if [ -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem ]; then
  sudo rm -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem && sudo update-ca-trust >/dev/null 2>&1
fi; true`, nil)
	r.emit(StepCert, Done, "certificate for "+res.Node+" installed", "")
	return nil
}

// put writes a file on the node as root, never readable by others on the
// way when its mode says it should not be.
func (r *Runner) put(ctx context.Context, path, mode string, body []byte) error {
	script := fmt.Sprintf(`set -e
sudo install -d -m 0755 %[1]s
sudo sh -c 'umask 077; cat > %[2]s.new'
sudo chmod %[3]s %[2]s.new
sudo chown root:root %[2]s.new
sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode)
	if out, err := r.Remote.Run(ctx, script, body); err != nil {
		return fmt.Errorf("writing %s: %v: %s", path, err, lastLine(out))
	}
	return nil
}

// --- configuration ---

// AgentUnit is the agent's systemd unit, the same one server init writes.
var AgentUnit string

func (r *Runner) configure(ctx context.Context, res *Result) error {
	r.emit(StepConfig, Running, "writing /etc/llmgw/agent.yaml", "")
	if err := r.put(ctx, "/etc/llmgw/agent.yaml", "0644", []byte(AgentYAML(r.Opts, res))); err != nil {
		return err
	}
	if AgentUnit == "" {
		return errors.New("onboard: AgentUnit is not set")
	}
	if err := r.put(ctx, "/etc/systemd/system/llmgw-agent.service", "0644", []byte(AgentUnit)); err != nil {
		return err
	}
	res.EngineProblems = r.checkEngines(ctx)
	for _, p := range res.EngineProblems {
		r.emit(StepConfig, Warn, p, "")
	}
	r.emit(StepConfig, Done, "agent.yaml and the systemd unit", "")
	return nil
}

// AgentYAML is the node's configuration: only what the node cannot find out
// for itself. Its address, GPUs, adapters and engines it discovers, and every
// line written here is one somebody has to keep true by hand.
func AgentYAML(o Options, res *Result) string {
	var b strings.Builder
	fmt.Fprintf(&b, "# %s, written by 'llmgw node add'.\n", res.Node)
	if len(res.Servers) == 1 {
		fmt.Fprintf(&b, "server: %q\n", res.Servers[0])
	} else {
		// Every control plane it may fail over to; a standby refuses it and
		// it moves on to the next.
		b.WriteString("servers:\n")
		for _, s := range res.Servers {
			fmt.Fprintf(&b, "  - %q\n", s)
		}
	}
	b.WriteString("\ntls:\n  ca: /etc/llmgw/ca.pem\n  cert: /etc/llmgw/agent.pem\n  key: /etc/llmgw/agent-key.pem\n")
	// The control plane refuses a node whose id differs from its
	// certificate's name; said only when the hostname would say otherwise.
	if res.Node != res.Hostname {
		fmt.Fprintf(&b, "node_id: %q\n", res.Node)
	}
	if o.MgmtAddr != "" {
		fmt.Fprintf(&b, "mgmt_addr: %q\n", o.MgmtAddr)
	}
	if o.MgmtIface != "" {
		fmt.Fprintf(&b, "mgmt_interface: %q\n", o.MgmtIface)
	}
	if len(o.Fabrics) > 0 {
		// The one thing a node cannot find out: which name its fabric goes
		// by on every other node.
		b.WriteString("fabrics:\n")
		for _, f := range o.Fabrics {
			fmt.Fprintf(&b, "  - name: %q\n    hcas: [%s]\n", f.Name, strings.Join(f.HCAs, ", "))
		}
	}
	if o.NvidiaSMI != "" || o.IBRoot != "" {
		b.WriteString("discovery:\n")
		if o.NvidiaSMI != "" {
			fmt.Fprintf(&b, "  nvidia_smi: %q\n", o.NvidiaSMI)
		}
		if o.IBRoot != "" {
			fmt.Fprintf(&b, "  ib_root: %q\n", o.IBRoot)
		}
	}
	if o.LlamaCppBin != "" || o.VLLMBin != "" || o.SGLangPython != "" || len(o.DisableEngines) > 0 {
		b.WriteString("engines:\n")
		if len(o.DisableEngines) > 0 {
			fmt.Fprintf(&b, "  disabled: [%s]\n", strings.Join(o.DisableEngines, ", "))
		}
		if o.LlamaCppBin != "" {
			fmt.Fprintf(&b, "  llamacpp:\n    bin: %q\n", o.LlamaCppBin)
		}
		if o.VLLMBin != "" {
			fmt.Fprintf(&b, "  vllm:\n    bin: %q\n", o.VLLMBin)
		}
		if o.SGLangPython != "" {
			fmt.Fprintf(&b, "  sglang:\n    python: %q\n", o.SGLangPython)
		}
	}
	return b.String()
}

// checkEngines runs each engine named by path. A wrong one registers as
// healthy and fails every shard placed on it, with the reason three logs
// away: llama-server on node05 was present, executable, and missing libgomp.
func (r *Runner) checkEngines(ctx context.Context) []string {
	var problems []string
	for _, e := range []struct{ name, path string }{
		{"llama.cpp", r.Opts.LlamaCppBin}, {"vLLM", r.Opts.VLLMBin}, {"SGLang", r.Opts.SGLangPython},
	} {
		if e.path == "" {
			continue
		}
		q := shellQuote(e.path)
		if _, err := r.Remote.Run(ctx, "test -x "+q, nil); err != nil {
			problems = append(problems, fmt.Sprintf("%s is set to %s, which is not an executable on the node", e.name, e.path))
			continue
		}
		if _, err := r.Remote.Run(ctx, q+" --version >/dev/null 2>&1 || "+q+" --help >/dev/null 2>&1", nil); err != nil {
			problems = append(problems, fmt.Sprintf("%s at %s exists but will not run; usually a missing library: ldd %s | grep 'not found'",
				e.name, e.path, e.path))
		}
	}
	return problems
}

// --- start ---

func (r *Runner) start(ctx context.Context, res *Result) error {
	// The user engines run as, so the agent drops root before starting one:
	// in the groups that own the GPU devices.
	if out, err := r.Remote.Run(ctx, `set -e
id llmgw-engine >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin llmgw-engine
for g in video render; do getent group "$g" >/dev/null && sudo usermod -aG "$g" llmgw-engine; done
true`, nil); err != nil {
		r.emit(StepStart, Warn, "the user engines run as could not be made, so they run as root: "+lastLine(out), "")
	}
	r.emit(StepStart, Running, "starting the agent", "")
	if out, err := r.Remote.Run(ctx, `set -e
sudo systemctl daemon-reload
sudo systemctl enable llmgw-agent >/dev/null 2>&1
sudo systemctl restart llmgw-agent
sleep 2
systemctl is-active llmgw-agent`, nil); err != nil {
		return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out))
	}
	r.emit(StepStart, Done, "llmgw-agent is running", "")
	if !r.Opts.NoNodeExporter {
		r.nodeExporter(ctx)
	}
	if !r.Opts.NoFirewall && len(r.Opts.FirewallPorts) > 0 {
		r.firewall(ctx, res)
	}
	return nil
}

// firewall keeps the node's llmgw ports — its engines, its metrics — to the
// networks it reaches the control planes on. A warning when it cannot: the
// node serves either way, and says how to see why.
func (r *Runner) firewall(ctx context.Context, res *Result) {
	var toward []string
	for _, s := range res.Servers {
		u, err := url.Parse(s)
		if err != nil {
			continue
		}
		if _, err := netip.ParseAddr(u.Hostname()); err == nil {
			toward = append(toward, u.Hostname())
		} else if ips, err := net.LookupIP(u.Hostname()); err == nil && len(ips) > 0 {
			toward = append(toward, ips[0].String())
		}
	}
	nets, err := firewall.Networks(ctx, r.Remote, toward)
	if err == nil {
		nets = append(nets, r.Opts.FirewallAllow...)
		_, err = firewall.Install(ctx, r.Remote, firewall.Spec{Ports: r.Opts.FirewallPorts, Allow: nets})
	}
	if err != nil {
		r.emit(StepStart, Warn, "the firewall was not set: "+err.Error()+
			"; the node's engine ports answer every network it is on", "")
		return
	}
	var shown []string
	for _, n := range nets {
		shown = append(shown, n.String())
	}
	r.emit(StepStart, Info, "llmgw's ports on the node answer "+strings.Join(shown, ", ")+" only ("+firewall.RulesFile+")", "")
}

// nodeExporter installs Prometheus' node exporter from the distribution, for
// what llmgw does not measure: network and disk traffic, filesystems. Best
// effort: a node without it serves as well, so a failure is a warning.
func (r *Runner) nodeExporter(ctx context.Context) {
	out, err := r.Remote.Run(ctx, `set -e
if systemctl is-active --quiet prometheus-node-exporter 2>/dev/null || systemctl is-active --quiet node_exporter 2>/dev/null; then
  echo already
elif command -v apt-get >/dev/null 2>&1; then
  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq prometheus-node-exporter >/dev/null
  sudo systemctl enable --now prometheus-node-exporter >/dev/null 2>&1
  echo installed
elif command -v dnf >/dev/null 2>&1; then
  sudo dnf install -y -q golang-github-prometheus-node-exporter >/dev/null
  sudo systemctl enable --now node_exporter >/dev/null 2>&1
  echo installed
else
  echo "no apt-get or dnf"; exit 1
fi`, nil)
	switch {
	case err != nil:
		r.emit(StepStart, Warn, "the Prometheus node exporter was not installed: "+lastLine(out)+
			"; the node serves without it, and its network and disk are not charted", "")
	case strings.Contains(string(out), "already"):
		r.emit(StepStart, Info, "the Prometheus node exporter was already running", "")
	default:
		r.emit(StepStart, Info, "the Prometheus node exporter is installed, on port 9100", "")
	}
}

// --- helpers ---

func lastLine(b []byte) string {
	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
	return strings.TrimSpace(lines[len(lines)-1])
}

func orUnknown(s string) string {
	if s == "" {
		return "unversioned"
	}
	return s
}

func countLevel(fs []doctor.Finding, l doctor.Level) int {
	n := 0
	for _, f := range fs {
		if f.Level == l {
			n++
		}
	}
	return n
}

// shellQuote quotes a string for a POSIX shell.
func shellQuote(s string) string {
	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}