// Package onboard makes a machine a node of the cluster, over SSH, from the // machine holding the certificate authority. // // It is what deploy/bootstrap/onboard-node.sh did, moved into the binary so // there is nothing else to fetch, and extended by what a script could not // easily do: it asks the machine its own name and architecture instead of // being told, runs llmgw doctor there before changing anything, and suggests // the fabric its adapters are on. // // It reports each step as an Event, so a plain terminal and a full-screen one // show the same thing from the same code. package onboard import ( "context" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "net" "net/http" "net/netip" "net/url" "os" "path/filepath" "regexp" "slices" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/doctor" "gitea.mynodes.xyz/ilker/llmgw/internal/firewall" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Fabric is one InfiniBand or RoCE fabric the node is on, and its adapters. type Fabric struct { Name string HCAs []string // Key is what the adapters report the fabric to be — its subnet manager, // or a RoCE network — on a suggestion, so a name given for one node can // be given again to the next on the same fabric. Key string } // ParseFabric reads "name:hca1,hca2", or "name" alone. func ParseFabric(s string) (Fabric, error) { name, hcas, _ := strings.Cut(strings.TrimSpace(s), ":") if name == "" { return Fabric{}, fmt.Errorf("fabric %q has no name: give it as name:mlx5_0,mlx5_1", s) } f := Fabric{Name: name} for _, h := range strings.Split(hcas, ",") { if h = strings.TrimSpace(h); h != "" { f.HCAs = append(f.HCAs, h) } } return f, nil } // Options say which machine, and what to tell its agent. type Options struct { // NoNodeExporter leaves Prometheus' node exporter off the machine. NoNodeExporter bool // NoFirewall leaves the machine's firewall as it is. Otherwise its llmgw // ports — FirewallPorts — answer only the networks it reaches the // control planes on, and FirewallAllow. NoFirewall bool // MirrorBase is where the mirror serves model files, http://host:8081: // the node is asked to reach it before it is added, since a node that // cannot fetches no model. MirrorBase string FirewallPorts []firewall.Port FirewallAllow []netip.Prefix // Addr is how to reach the machine over SSH, and User who to log in as. Addr, User string // Node is its id in the cluster. Empty means its hostname, which is what // it should be: the node id, the hostname and the certificate's name // being one string saves an afternoon each time they would differ. Node string // Servers are the control planes the agent dials. Empty means this // machine, at the address it has on the route to the node. Servers []string Fabrics []Fabric MgmtAddr, MgmtIface string NvidiaSMI, IBRoot string LlamaCppBin string VLLMBin string SGLangPython string // PKI is the authority's directory; EtcDir is where this machine's own // server certificate is, to check the chosen address is named in it. PKI, EtcDir string // Binary is the llmgw to install when the node cannot download the // release, and Version what it is. Binary, Version string // Releases is where release files are downloaded from. Releases string // ReleaseClient fetches them; nil is a default client. ReleaseClient *http.Client // IgnoreChecks goes on when doctor finds a problem. IgnoreChecks bool // DisableEngines are engines this node never installs or advertises, // written into its agent.yaml, and not checked by doctor: vLLM on a node // of cards it does not run on, meant for llama.cpp alone. Narrower than // IgnoreChecks, which also passes over everything else. DisableEngines []string // VLLMVersion is the vLLM the control plane has nodes install, for // doctor to check against; "none" when it has them install none, empty // for doctor's own default. VLLMVersion string // SSHWait is how long an unreachable machine is waited for. SSHWait time.Duration // AskFabrics, when set, is asked which fabrics to declare for a node // given none whose adapters are up, with the adapters grouped by the // fabric they report. It returns the fabrics to use, none to go on // without. Nil leaves it at a warning, as a script needs. AskFabrics func(suggested []Fabric) []Fabric // AskContinue, when set, is asked whether to go on after doctor found // problems. Nil stops unless IgnoreChecks. AskContinue func(fails int) bool } // Levels of an Event. const ( Running = "running" Done = "done" Info = "info" Warn = "warn" Fail = "fail" ) // Event is one thing that happened: a step starting or ending, or something // worth saying along the way. type Event struct { Step string Level string Detail string // Fix is what to do about a warning or a failure, when there is a known // answer. Fix string } // The steps, in order. const ( StepConnect = "connect" StepCheck = "check" StepBinary = "binary" StepCert = "certificate" StepConfig = "configuration" StepStart = "start" // StepJoin and StepEngines are watched from the control plane once the // agent runs; the caller does that, since it has the admin API. StepJoin = "join" StepEngines = "engines" ) // Steps lists them for a display that shows what is still to come. var Steps = []string{StepConnect, StepBinary, StepCheck, StepCert, StepConfig, StepStart, StepJoin, StepEngines} // Result is what onboarding learned and did. type Result struct { Node string Arch string Hostname string Servers []string Facts doctor.Facts Findings []doctor.Finding // Suggested is the fabrics the node's active adapters are on, for a node // given none: grouped by what the adapters report, named for the operator // to choose. Suggested []Fabric // EngineProblems are engines named by path that will not run there. EngineProblems []string } // Runner onboards one machine. type Runner struct { Opts Options Remote Remote // Emit hears every event. Nil discards them. Emit func(Event) ca *tlsconf.CA } // New makes a Runner that reaches the machine with the system's ssh. func New(o Options, emit func(Event)) *Runner { return &Runner{Opts: o, Remote: &SSH{Addr: o.Addr, User: o.User}, Emit: emit} } func (r *Runner) emit(step, level, detail, fix string) { if r.Emit != nil { r.Emit(Event{Step: step, Level: level, Detail: detail, Fix: fix}) } } // ErrChecks is returned when doctor found problems and IgnoreChecks is off. var ErrChecks = errors.New("the node has problems to fix first") // Run does every step, stopping at the first that fails. func (r *Runner) Run(ctx context.Context) (*Result, error) { res := &Result{} // The authority first: without it nothing can be finished, and finding // out after the node has been changed leaves it half done. pki := r.Opts.PKI ca, err := tlsconf.LoadCA(filepath.Join(pki, "ca.pem"), filepath.Join(pki, "ca-key.pem")) if err != nil { err = fmt.Errorf("cannot read the certificate authority in %s: %v. Run this on the machine that holds it, as root (sudo), or give --pki", pki, err) r.emit(StepConnect, Fail, err.Error(), "") return res, err } r.ca = ca for _, step := range []struct { name string fn func(context.Context, *Result) error }{ {StepConnect, r.connect}, {StepBinary, r.binary}, {StepCheck, r.check}, {StepCert, r.certificate}, {StepConfig, r.configure}, {StepStart, r.start}, } { if err := step.fn(ctx, res); err != nil { r.emit(step.name, Fail, err.Error(), "") return res, err } } return res, nil } // --- connect --- var nodeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`) func (r *Runner) connect(ctx context.Context, res *Result) error { r.emit(StepConnect, Running, "connecting to "+r.target(), "") wait := r.Opts.SSHWait if wait == 0 { wait = 3 * time.Minute } // Waited for rather than failed on: a machine a minute old restarts sshd // after cloud-init regenerates its host keys, and refuses connections for // a few seconds after answering. deadline := time.Now().Add(wait) var last error for { out, err := r.Remote.Run(ctx, "hostname && uname -m", nil) if err == nil { lines := strings.Fields(string(out)) if len(lines) >= 2 { res.Hostname, res.Arch = lines[0], lines[1] break } err = fmt.Errorf("unexpected answer: %q", out) } last = err if time.Now().After(deadline) { return fmt.Errorf("cannot log in to %s: %v. Check that `ssh %s true` works from this machine, as the user running this, without a password", r.target(), last, r.target()) } select { case <-ctx.Done(): return ctx.Err() case <-time.After(5 * time.Second): } } if _, err := r.Remote.Run(ctx, "sudo -n true", nil); err != nil { return fmt.Errorf("%s can log in but cannot use sudo without a password, which installing the agent needs", r.target()) } switch res.Arch { case "x86_64", "amd64": res.Arch = "amd64" case "aarch64", "arm64": res.Arch = "arm64" default: return fmt.Errorf("%s is %s; llmgw is built for x86-64 and ARM64", r.target(), res.Arch) } res.Node = r.Opts.Node if res.Node == "" { res.Node = res.Hostname } if !nodeName.MatchString(res.Node) { return fmt.Errorf("%q cannot be a node id: letters, digits, dots, dashes and underscores; give one with --node", res.Node) } servers, err := r.servers() if err != nil { return err } res.Servers = servers r.emit(StepConnect, Done, fmt.Sprintf("%s (%s), joining as %s, agent will dial %s", res.Hostname, res.Arch, res.Node, strings.Join(servers, ", ")), "") return nil } func (r *Runner) target() string { if r.Opts.User == "" { return r.Opts.Addr } return r.Opts.User + "@" + r.Opts.Addr } // servers are the given control planes, or this machine at the address it // uses to reach the node — which is the one the node can reach it at, and // is named in the server certificate when init issued it here. func (r *Runner) servers() ([]string, error) { if len(r.Opts.Servers) > 0 { return r.Opts.Servers, nil } host := r.Opts.Addr if h, _, err := net.SplitHostPort(host); err == nil { host = h } c, err := net.Dial("udp", net.JoinHostPort(host, "7443")) if err != nil { return nil, fmt.Errorf("cannot tell which address the node would reach this machine at: %v; give it with --server", err) } local := c.LocalAddr().(*net.UDPAddr).IP.String() _ = c.Close() if names, err := certNames(filepath.Join(r.etcDir(), "server.pem")); err == nil && !slices.Contains(names, local) { return nil, fmt.Errorf("the node would reach this machine at %s, which the server certificate does not name (%s): "+ "run `sudo llmgw server init --host %s` to add it, or give the address with --server", local, strings.Join(names, ", "), local) } return []string{"https://" + net.JoinHostPort(local, "7443")}, nil } func (r *Runner) etcDir() string { if r.Opts.EtcDir != "" { return r.Opts.EtcDir } return "/etc/llmgw" } func certNames(path string) ([]string, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } blk, _ := pem.Decode(b) if blk == nil { return nil, errors.New("no certificate in " + path) } c, err := x509.ParseCertificate(blk.Bytes) if err != nil { return nil, err } names := slices.Clone(c.DNSNames) for _, ip := range c.IPAddresses { names = append(names, ip.String()) } return names, nil } // --- binary --- // releaseTag is a version CI published: v0.10.4, or a release candidate of // it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no // release to download. var releaseTag = release.Tag func (r *Runner) binary(ctx context.Context, res *Result) error { v := r.Opts.Version if releaseTag.MatchString(v) { // Fetched here, for the node's architecture, and checked here — // signature and checksum, by the same code that upgrades this // machine — then handed to the node. The node checking a release // itself could only compare it with a checksum from the same // server, which proves nothing about who put both there. r.emit(StepBinary, Running, "downloading llmgw "+v+" for "+res.Arch+" and checking it", "") b, err := release.Fetch(ctx, r.Opts.ReleaseClient, r.Opts.Releases, v, "linux", goArch(res.Arch)) if err == nil { if err := r.installBinary(ctx, b); err != nil { return err } what := "checksum verified" if release.TrustedKey() != "" { what = "signature and checksum verified" } r.emit(StepBinary, Done, "llmgw "+v+" from the release, "+what, "") return nil } r.emit(StepBinary, Warn, "the release could not be fetched and checked here: "+err.Error(), "") } // A development build has no release to fetch, and a node that cannot // reach the releases gets this machine's copy — when it can run it. if res.Arch != hostArch() { return fmt.Errorf("the node is %s and this machine %s, so its binary cannot be copied: "+ "let the node reach %s, or install llmgw there yourself", res.Arch, hostArch(), r.Opts.Releases) } r.emit(StepBinary, Running, "copying this machine's llmgw ("+orUnknown(v)+")", "") b, err := os.ReadFile(r.Opts.Binary) if err != nil { return fmt.Errorf("reading %s: %w", r.Opts.Binary, err) } if err := r.installBinary(ctx, b); err != nil { return err } r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "") return nil } // installBinary puts bytes at /usr/local/bin/llmgw on the node. func (r *Runner) installBinary(ctx context.Context, b []byte) error { if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil { return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out)) } return nil } // goArch is Go's name for what uname -m says. func goArch(uname string) string { switch uname { case "x86_64", "amd64": return "amd64" case "aarch64", "arm64": return "arm64" } return uname } // --- check --- func (r *Runner) check(ctx context.Context, res *Result) error { r.emit(StepCheck, Running, "running llmgw doctor on the node", "") args := "sudo llmgw doctor --json" if r.Opts.NvidiaSMI != "" { args += " --nvidia-smi " + shellQuote(r.Opts.NvidiaSMI) } if r.Opts.IBRoot != "" { args += " --ib-root " + shellQuote(r.Opts.IBRoot) } // The fabrics given, checked as the ones they will be; any adapter they // do not name is checked as one nothing uses. for _, fab := range r.Opts.Fabrics { args += " --fabric " + shellQuote(fab.Name+":"+strings.Join(fab.HCAs, ",")) } switch { case slices.Contains(r.Opts.DisableEngines, "vllm"): args += " --vllm none" r.emit(StepCheck, Info, "vLLM is disabled on this node: it joins with llama.cpp alone", "") case r.Opts.VLLMVersion != "": args += " --vllm " + shellQuote(r.Opts.VLLMVersion) } out, err := r.Remote.Run(ctx, args, nil) var rep struct { Findings []doctor.Finding `json:"findings"` Facts doctor.Facts `json:"facts"` } if jerr := json.Unmarshal(out, &rep); jerr != nil { if err == nil { err = jerr } return fmt.Errorf("llmgw doctor did not run on the node: %v: %s", err, lastLine(out)) } if !r.Opts.NoFirewall && len(r.Opts.FirewallPorts) > 0 { // A missing firewall is what this run is about to write. rep.Findings = slices.DeleteFunc(rep.Findings, func(f doctor.Finding) bool { return f.Area == "firewall" }) } if f, ok := r.mirrorReach(ctx); ok { rep.Findings = append(rep.Findings, f) } res.Facts, res.Findings = rep.Facts, rep.Findings fails := 0 for _, f := range rep.Findings { switch f.Level { case doctor.Fail: fails++ r.emit(StepCheck, Fail, f.Area+": "+f.Message, f.Fix) case doctor.Warn: r.emit(StepCheck, Warn, f.Area+": "+f.Message, f.Fix) case doctor.OK: r.emit(StepCheck, Info, f.Area+": "+f.Message, "") } } res.Suggested = suggestFabrics(rep.Facts.HCAs) if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 && r.Opts.AskFabrics != nil { r.Opts.Fabrics = r.Opts.AskFabrics(res.Suggested) for _, f := range r.Opts.Fabrics { r.emit(StepCheck, Info, "fabric "+f.Name+": "+strings.Join(f.HCAs, ", "), "") } } if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 { var flags []string for _, f := range res.Suggested { flags = append(flags, "--fabric "+f.Name+":"+strings.Join(f.HCAs, ",")) } r.emit(StepCheck, Warn, "InfiniBand or RoCE adapters are up, but no fabric was given: this node would never be given a replica spanning nodes", "add "+strings.Join(flags, " ")+", naming each fabric the same on every node attached to it") } goOn := r.Opts.IgnoreChecks if fails > 0 && !goOn && r.Opts.AskContinue != nil && r.Opts.AskContinue(fails) { r.emit(StepCheck, Warn, fmt.Sprintf("going on despite %d problem(s)", fails), "") goOn = true } if fails > 0 && !goOn { return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails) } r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "") return nil } // mirrorReach asks the node to reach the mirror's files: a base address the // node cannot reach (another machine's interface, Docker's bridge, a port // closed to it) otherwise shows up as every model failing to download. func (r *Runner) mirrorReach(ctx context.Context) (doctor.Finding, bool) { if r.Opts.MirrorBase == "" { return doctor.Finding{}, false } u, err := url.Parse(r.Opts.MirrorBase) if err != nil || u.Hostname() == "" { return doctor.Finding{}, false } port := u.Port() if port == "" { port = "80" } base := strings.TrimRight(r.Opts.MirrorBase, "/") + "/" out, _ := r.Remote.Run(ctx, `if command -v curl >/dev/null 2>&1; then curl -s -o /dev/null -w '%{http_code}' --max-time 5 `+shellQuote(base)+` || true else timeout 5 bash -c `+shellQuote("/dev/null && echo 200 || echo 000 fi`, nil) code := strings.TrimSpace(lastLine(out)) if code != "" && code != "000" { return doctor.Finding{Level: doctor.OK, Area: "mirror", Message: "reaches the mirror's files at " + base}, true } return doctor.Finding{Level: doctor.Fail, Area: "mirror", Message: "cannot reach the mirror's files at " + base + ": every model it is given would fail to download", Fix: "on the mirror's machine, give llmgw-mirror a --base this node reaches (systemctl cat llmgw-mirror), " + "and let this node through to port " + port}, true } // suggestFabrics groups the active adapters by the fabric they report being // on: the subnet manager they see, or the network of their RoCE address. // Names are placeholders for the operator: a name is what ties one node's // adapters to another's, and nothing on one node can know what the others // were called. func suggestFabrics(hcas []discovery.HCA) []Fabric { byKey := map[string][]string{} var keys []string for _, h := range hcas { for _, p := range h.ActivePorts() { k := p.FabricKey() if k == "" { k = "unknown" } if _, ok := byKey[k]; !ok { keys = append(keys, k) } if !slices.Contains(byKey[k], h.Name) { byKey[k] = append(byKey[k], h.Name) } } } var out []Fabric for i, k := range keys { name := "fabric" if len(keys) > 1 { name = fmt.Sprintf("fabric%d", i+1) } out = append(out, Fabric{Name: name, HCAs: byKey[k], Key: k}) } return out } // --- certificate --- func (r *Runner) certificate(ctx context.Context, res *Result) error { r.emit(StepCert, Running, "issuing a certificate for "+res.Node, "") ca := r.ca // Issued fresh every time: onboarding again should not depend on finding // an old key, and the old one keeps working until revoked. leaf, err := ca.IssueClient(res.Node) if err != nil { return err } for _, f := range []struct { path, mode string body []byte }{ {"/etc/llmgw/ca.pem", "0644", ca.PEM}, {"/etc/llmgw/agent.pem", "0644", leaf.CertPEM}, {"/etc/llmgw/agent-key.pem", "0600", leaf.KeyPEM}, } { if err := r.put(ctx, f.path, f.mode, f.body); err != nil { return err } } // The agent trusts the cluster's root through its own configuration only. // Not system-wide: that root signs whatever the control plane asks it to, // and every TLS client on the machine would then take it for any site. // A node onboarded by an older version had it added; take it back out. _, _ = r.Remote.Run(ctx, `if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt && sudo update-ca-certificates --fresh >/dev/null 2>&1 fi if [ -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem ]; then sudo rm -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem && sudo update-ca-trust >/dev/null 2>&1 fi; true`, nil) r.emit(StepCert, Done, "certificate for "+res.Node+" installed", "") return nil } // put writes a file on the node as root, never readable by others on the // way when its mode says it should not be. func (r *Runner) put(ctx context.Context, path, mode string, body []byte) error { script := fmt.Sprintf(`set -e sudo install -d -m 0755 %[1]s sudo sh -c 'umask 077; cat > %[2]s.new' sudo chmod %[3]s %[2]s.new sudo chown root:root %[2]s.new sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode) if out, err := r.Remote.Run(ctx, script, body); err != nil { return fmt.Errorf("writing %s: %v: %s", path, err, lastLine(out)) } return nil } // --- configuration --- // AgentUnit is the agent's systemd unit, the same one server init writes. var AgentUnit string func (r *Runner) configure(ctx context.Context, res *Result) error { r.emit(StepConfig, Running, "writing /etc/llmgw/agent.yaml", "") if err := r.put(ctx, "/etc/llmgw/agent.yaml", "0644", []byte(AgentYAML(r.Opts, res))); err != nil { return err } if AgentUnit == "" { return errors.New("onboard: AgentUnit is not set") } if err := r.put(ctx, "/etc/systemd/system/llmgw-agent.service", "0644", []byte(AgentUnit)); err != nil { return err } res.EngineProblems = r.checkEngines(ctx) for _, p := range res.EngineProblems { r.emit(StepConfig, Warn, p, "") } r.emit(StepConfig, Done, "agent.yaml and the systemd unit", "") return nil } // AgentYAML is the node's configuration: only what the node cannot find out // for itself. Its address, GPUs, adapters and engines it discovers, and every // line written here is one somebody has to keep true by hand. func AgentYAML(o Options, res *Result) string { var b strings.Builder fmt.Fprintf(&b, "# %s, written by 'llmgw node add'.\n", res.Node) if len(res.Servers) == 1 { fmt.Fprintf(&b, "server: %q\n", res.Servers[0]) } else { // Every control plane it may fail over to; a standby refuses it and // it moves on to the next. b.WriteString("servers:\n") for _, s := range res.Servers { fmt.Fprintf(&b, " - %q\n", s) } } b.WriteString("\ntls:\n ca: /etc/llmgw/ca.pem\n cert: /etc/llmgw/agent.pem\n key: /etc/llmgw/agent-key.pem\n") // The control plane refuses a node whose id differs from its // certificate's name; said only when the hostname would say otherwise. if res.Node != res.Hostname { fmt.Fprintf(&b, "node_id: %q\n", res.Node) } if o.MgmtAddr != "" { fmt.Fprintf(&b, "mgmt_addr: %q\n", o.MgmtAddr) } if o.MgmtIface != "" { fmt.Fprintf(&b, "mgmt_interface: %q\n", o.MgmtIface) } if len(o.Fabrics) > 0 { // The one thing a node cannot find out: which name its fabric goes // by on every other node. b.WriteString("fabrics:\n") for _, f := range o.Fabrics { fmt.Fprintf(&b, " - name: %q\n hcas: [%s]\n", f.Name, strings.Join(f.HCAs, ", ")) } } if o.NvidiaSMI != "" || o.IBRoot != "" { b.WriteString("discovery:\n") if o.NvidiaSMI != "" { fmt.Fprintf(&b, " nvidia_smi: %q\n", o.NvidiaSMI) } if o.IBRoot != "" { fmt.Fprintf(&b, " ib_root: %q\n", o.IBRoot) } } if o.LlamaCppBin != "" || o.VLLMBin != "" || o.SGLangPython != "" || len(o.DisableEngines) > 0 { b.WriteString("engines:\n") if len(o.DisableEngines) > 0 { fmt.Fprintf(&b, " disabled: [%s]\n", strings.Join(o.DisableEngines, ", ")) } if o.LlamaCppBin != "" { fmt.Fprintf(&b, " llamacpp:\n bin: %q\n", o.LlamaCppBin) } if o.VLLMBin != "" { fmt.Fprintf(&b, " vllm:\n bin: %q\n", o.VLLMBin) } if o.SGLangPython != "" { fmt.Fprintf(&b, " sglang:\n python: %q\n", o.SGLangPython) } } return b.String() } // checkEngines runs each engine named by path. A wrong one registers as // healthy and fails every shard placed on it, with the reason three logs // away: llama-server on node05 was present, executable, and missing libgomp. func (r *Runner) checkEngines(ctx context.Context) []string { var problems []string for _, e := range []struct{ name, path string }{ {"llama.cpp", r.Opts.LlamaCppBin}, {"vLLM", r.Opts.VLLMBin}, {"SGLang", r.Opts.SGLangPython}, } { if e.path == "" { continue } q := shellQuote(e.path) if _, err := r.Remote.Run(ctx, "test -x "+q, nil); err != nil { problems = append(problems, fmt.Sprintf("%s is set to %s, which is not an executable on the node", e.name, e.path)) continue } if _, err := r.Remote.Run(ctx, q+" --version >/dev/null 2>&1 || "+q+" --help >/dev/null 2>&1", nil); err != nil { problems = append(problems, fmt.Sprintf("%s at %s exists but will not run; usually a missing library: ldd %s | grep 'not found'", e.name, e.path, e.path)) } } return problems } // --- start --- func (r *Runner) start(ctx context.Context, res *Result) error { // The user engines run as, so the agent drops root before starting one: // in the groups that own the GPU devices. if out, err := r.Remote.Run(ctx, `set -e id llmgw-engine >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin llmgw-engine for g in video render; do getent group "$g" >/dev/null && sudo usermod -aG "$g" llmgw-engine; done true`, nil); err != nil { r.emit(StepStart, Warn, "the user engines run as could not be made, so they run as root: "+lastLine(out), "") } r.emit(StepStart, Running, "starting the agent", "") if out, err := r.Remote.Run(ctx, `set -e sudo systemctl daemon-reload sudo systemctl enable llmgw-agent >/dev/null 2>&1 sudo systemctl restart llmgw-agent sleep 2 systemctl is-active llmgw-agent`, nil); err != nil { return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out)) } r.emit(StepStart, Done, "llmgw-agent is running", "") if !r.Opts.NoNodeExporter { r.nodeExporter(ctx) } if !r.Opts.NoFirewall && len(r.Opts.FirewallPorts) > 0 { r.firewall(ctx, res) } return nil } // firewall keeps the node's llmgw ports — its engines, its metrics — to the // networks it reaches the control planes on. A warning when it cannot: the // node serves either way, and says how to see why. func (r *Runner) firewall(ctx context.Context, res *Result) { var toward []string for _, s := range res.Servers { u, err := url.Parse(s) if err != nil { continue } if _, err := netip.ParseAddr(u.Hostname()); err == nil { toward = append(toward, u.Hostname()) } else if ips, err := net.LookupIP(u.Hostname()); err == nil && len(ips) > 0 { toward = append(toward, ips[0].String()) } } nets, err := firewall.Networks(ctx, r.Remote, toward) if err == nil { nets = append(nets, r.Opts.FirewallAllow...) _, err = firewall.Install(ctx, r.Remote, firewall.Spec{Ports: r.Opts.FirewallPorts, Allow: nets}) } if err != nil { r.emit(StepStart, Warn, "the firewall was not set: "+err.Error()+ "; the node's engine ports answer every network it is on", "") return } var shown []string for _, n := range nets { shown = append(shown, n.String()) } r.emit(StepStart, Info, "llmgw's ports on the node answer "+strings.Join(shown, ", ")+" only ("+firewall.RulesFile+")", "") } // nodeExporter installs Prometheus' node exporter from the distribution, for // what llmgw does not measure: network and disk traffic, filesystems. Best // effort: a node without it serves as well, so a failure is a warning. func (r *Runner) nodeExporter(ctx context.Context) { out, err := r.Remote.Run(ctx, `set -e if systemctl is-active --quiet prometheus-node-exporter 2>/dev/null || systemctl is-active --quiet node_exporter 2>/dev/null; then echo already elif command -v apt-get >/dev/null 2>&1; then sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq prometheus-node-exporter >/dev/null sudo systemctl enable --now prometheus-node-exporter >/dev/null 2>&1 echo installed elif command -v dnf >/dev/null 2>&1; then sudo dnf install -y -q golang-github-prometheus-node-exporter >/dev/null sudo systemctl enable --now node_exporter >/dev/null 2>&1 echo installed else echo "no apt-get or dnf"; exit 1 fi`, nil) switch { case err != nil: r.emit(StepStart, Warn, "the Prometheus node exporter was not installed: "+lastLine(out)+ "; the node serves without it, and its network and disk are not charted", "") case strings.Contains(string(out), "already"): r.emit(StepStart, Info, "the Prometheus node exporter was already running", "") default: r.emit(StepStart, Info, "the Prometheus node exporter is installed, on port 9100", "") } } // --- helpers --- func lastLine(b []byte) string { lines := strings.Split(strings.TrimSpace(string(b)), "\n") return strings.TrimSpace(lines[len(lines)-1]) } func orUnknown(s string) string { if s == "" { return "unversioned" } return s } func countLevel(fs []doctor.Finding, l doctor.Level) int { n := 0 for _, f := range fs { if f.Level == l { n++ } } return n } // shellQuote quotes a string for a POSIX shell. func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }