CPU replicas start on half the machine's CPUs, not all of them: two busy llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece, where at 4 threads they keep 5.5. llamacpp.threads sets it per node. Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with admission, schedule and upstream spans and traceparent sent to the engine. Security: onboarding no longer puts the cluster root in the system trust store and removes it where it was; the mirror's mutual-TLS API can be limited to named certificates (--api-client); init.log no longer keeps the administrator's password; a warning when auth is off on a public listen; the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and the remaining engine gaps are documented. reconciler.go and agent.go are split by topic, the status mapping of each failure is tested, and the plan's checklist is brought up to date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
682 lines
22 KiB
Go
682 lines
22 KiB
Go
// Package onboard makes a machine a node of the cluster, over SSH, from the
|
|
// machine holding the certificate authority.
|
|
//
|
|
// It is what deploy/bootstrap/onboard-node.sh did, moved into the binary so
|
|
// there is nothing else to fetch, and extended by what a script could not
|
|
// easily do: it asks the machine its own name and architecture instead of
|
|
// being told, runs llmgw doctor there before changing anything, and suggests
|
|
// the fabric its adapters are on.
|
|
//
|
|
// It reports each step as an Event, so a plain terminal and a full-screen one
|
|
// show the same thing from the same code.
|
|
package onboard
|
|
|
|
import (
|
|
"context"
|
|
"crypto/x509"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
)
|
|
|
|
// Fabric is one InfiniBand or RoCE fabric the node is on, and its adapters.
|
|
type Fabric struct {
|
|
Name string
|
|
HCAs []string
|
|
// Key is what the adapters report the fabric to be — its subnet manager,
|
|
// or a RoCE network — on a suggestion, so a name given for one node can
|
|
// be given again to the next on the same fabric.
|
|
Key string
|
|
}
|
|
|
|
// ParseFabric reads "name:hca1,hca2", or "name" alone.
|
|
func ParseFabric(s string) (Fabric, error) {
|
|
name, hcas, _ := strings.Cut(strings.TrimSpace(s), ":")
|
|
if name == "" {
|
|
return Fabric{}, fmt.Errorf("fabric %q has no name: give it as name:mlx5_0,mlx5_1", s)
|
|
}
|
|
f := Fabric{Name: name}
|
|
for _, h := range strings.Split(hcas, ",") {
|
|
if h = strings.TrimSpace(h); h != "" {
|
|
f.HCAs = append(f.HCAs, h)
|
|
}
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
// Options say which machine, and what to tell its agent.
|
|
type Options struct {
|
|
// Addr is how to reach the machine over SSH, and User who to log in as.
|
|
Addr, User string
|
|
// Node is its id in the cluster. Empty means its hostname, which is what
|
|
// it should be: the node id, the hostname and the certificate's name
|
|
// being one string saves an afternoon each time they would differ.
|
|
Node string
|
|
// Servers are the control planes the agent dials. Empty means this
|
|
// machine, at the address it has on the route to the node.
|
|
Servers []string
|
|
Fabrics []Fabric
|
|
|
|
MgmtAddr, MgmtIface string
|
|
NvidiaSMI, IBRoot string
|
|
LlamaCppBin string
|
|
VLLMBin string
|
|
SGLangPython string
|
|
|
|
// PKI is the authority's directory; EtcDir is where this machine's own
|
|
// server certificate is, to check the chosen address is named in it.
|
|
PKI, EtcDir string
|
|
// Binary is the llmgw to install when the node cannot download the
|
|
// release, and Version what it is.
|
|
Binary, Version string
|
|
// Releases is where release files are downloaded from.
|
|
Releases string
|
|
// IgnoreChecks goes on when doctor finds a problem.
|
|
IgnoreChecks bool
|
|
// SSHWait is how long an unreachable machine is waited for.
|
|
SSHWait time.Duration
|
|
|
|
// AskFabrics, when set, is asked which fabrics to declare for a node
|
|
// given none whose adapters are up, with the adapters grouped by the
|
|
// fabric they report. It returns the fabrics to use, none to go on
|
|
// without. Nil leaves it at a warning, as a script needs.
|
|
AskFabrics func(suggested []Fabric) []Fabric
|
|
// AskContinue, when set, is asked whether to go on after doctor found
|
|
// problems. Nil stops unless IgnoreChecks.
|
|
AskContinue func(fails int) bool
|
|
}
|
|
|
|
// Levels of an Event.
|
|
const (
|
|
Running = "running"
|
|
Done = "done"
|
|
Info = "info"
|
|
Warn = "warn"
|
|
Fail = "fail"
|
|
)
|
|
|
|
// Event is one thing that happened: a step starting or ending, or something
|
|
// worth saying along the way.
|
|
type Event struct {
|
|
Step string
|
|
Level string
|
|
Detail string
|
|
// Fix is what to do about a warning or a failure, when there is a known
|
|
// answer.
|
|
Fix string
|
|
}
|
|
|
|
// The steps, in order.
|
|
const (
|
|
StepConnect = "connect"
|
|
StepCheck = "check"
|
|
StepBinary = "binary"
|
|
StepCert = "certificate"
|
|
StepConfig = "configuration"
|
|
StepStart = "start"
|
|
// StepJoin and StepEngines are watched from the control plane once the
|
|
// agent runs; the caller does that, since it has the admin API.
|
|
StepJoin = "join"
|
|
StepEngines = "engines"
|
|
)
|
|
|
|
// Steps lists them for a display that shows what is still to come.
|
|
var Steps = []string{StepConnect, StepBinary, StepCheck, StepCert, StepConfig, StepStart, StepJoin, StepEngines}
|
|
|
|
// Result is what onboarding learned and did.
|
|
type Result struct {
|
|
Node string
|
|
Arch string
|
|
Hostname string
|
|
Servers []string
|
|
Facts doctor.Facts
|
|
Findings []doctor.Finding
|
|
// Suggested is the fabrics the node's active adapters are on, for a node
|
|
// given none: grouped by what the adapters report, named for the operator
|
|
// to choose.
|
|
Suggested []Fabric
|
|
// EngineProblems are engines named by path that will not run there.
|
|
EngineProblems []string
|
|
}
|
|
|
|
// Runner onboards one machine.
|
|
type Runner struct {
|
|
Opts Options
|
|
Remote Remote
|
|
// Emit hears every event. Nil discards them.
|
|
Emit func(Event)
|
|
|
|
ca *tlsconf.CA
|
|
}
|
|
|
|
// New makes a Runner that reaches the machine with the system's ssh.
|
|
func New(o Options, emit func(Event)) *Runner {
|
|
return &Runner{Opts: o, Remote: &SSH{Addr: o.Addr, User: o.User}, Emit: emit}
|
|
}
|
|
|
|
func (r *Runner) emit(step, level, detail, fix string) {
|
|
if r.Emit != nil {
|
|
r.Emit(Event{Step: step, Level: level, Detail: detail, Fix: fix})
|
|
}
|
|
}
|
|
|
|
// ErrChecks is returned when doctor found problems and IgnoreChecks is off.
|
|
var ErrChecks = errors.New("the node has problems to fix first")
|
|
|
|
// Run does every step, stopping at the first that fails.
|
|
func (r *Runner) Run(ctx context.Context) (*Result, error) {
|
|
res := &Result{}
|
|
// The authority first: without it nothing can be finished, and finding
|
|
// out after the node has been changed leaves it half done.
|
|
pki := r.Opts.PKI
|
|
ca, err := tlsconf.LoadCA(filepath.Join(pki, "ca.pem"), filepath.Join(pki, "ca-key.pem"))
|
|
if err != nil {
|
|
err = fmt.Errorf("cannot read the certificate authority in %s: %v. Run this on the machine that holds it, as root (sudo), or give --pki", pki, err)
|
|
r.emit(StepConnect, Fail, err.Error(), "")
|
|
return res, err
|
|
}
|
|
r.ca = ca
|
|
for _, step := range []struct {
|
|
name string
|
|
fn func(context.Context, *Result) error
|
|
}{
|
|
{StepConnect, r.connect},
|
|
{StepBinary, r.binary},
|
|
{StepCheck, r.check},
|
|
{StepCert, r.certificate},
|
|
{StepConfig, r.configure},
|
|
{StepStart, r.start},
|
|
} {
|
|
if err := step.fn(ctx, res); err != nil {
|
|
r.emit(step.name, Fail, err.Error(), "")
|
|
return res, err
|
|
}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// --- connect ---
|
|
|
|
var nodeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`)
|
|
|
|
func (r *Runner) connect(ctx context.Context, res *Result) error {
|
|
r.emit(StepConnect, Running, "connecting to "+r.target(), "")
|
|
wait := r.Opts.SSHWait
|
|
if wait == 0 {
|
|
wait = 3 * time.Minute
|
|
}
|
|
// Waited for rather than failed on: a machine a minute old restarts sshd
|
|
// after cloud-init regenerates its host keys, and refuses connections for
|
|
// a few seconds after answering.
|
|
deadline := time.Now().Add(wait)
|
|
var last error
|
|
for {
|
|
out, err := r.Remote.Run(ctx, "hostname && uname -m", nil)
|
|
if err == nil {
|
|
lines := strings.Fields(string(out))
|
|
if len(lines) >= 2 {
|
|
res.Hostname, res.Arch = lines[0], lines[1]
|
|
break
|
|
}
|
|
err = fmt.Errorf("unexpected answer: %q", out)
|
|
}
|
|
last = err
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("cannot log in to %s: %v. Check that `ssh %s true` works from this machine, as the user running this, without a password",
|
|
r.target(), last, r.target())
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
}
|
|
if _, err := r.Remote.Run(ctx, "sudo -n true", nil); err != nil {
|
|
return fmt.Errorf("%s can log in but cannot use sudo without a password, which installing the agent needs", r.target())
|
|
}
|
|
switch res.Arch {
|
|
case "x86_64", "amd64":
|
|
res.Arch = "amd64"
|
|
case "aarch64", "arm64":
|
|
res.Arch = "arm64"
|
|
default:
|
|
return fmt.Errorf("%s is %s; llmgw is built for x86-64 and ARM64", r.target(), res.Arch)
|
|
}
|
|
res.Node = r.Opts.Node
|
|
if res.Node == "" {
|
|
res.Node = res.Hostname
|
|
}
|
|
if !nodeName.MatchString(res.Node) {
|
|
return fmt.Errorf("%q cannot be a node id: letters, digits, dots, dashes and underscores; give one with --node", res.Node)
|
|
}
|
|
servers, err := r.servers()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
res.Servers = servers
|
|
r.emit(StepConnect, Done, fmt.Sprintf("%s (%s), joining as %s, agent will dial %s",
|
|
res.Hostname, res.Arch, res.Node, strings.Join(servers, ", ")), "")
|
|
return nil
|
|
}
|
|
|
|
func (r *Runner) target() string {
|
|
if r.Opts.User == "" {
|
|
return r.Opts.Addr
|
|
}
|
|
return r.Opts.User + "@" + r.Opts.Addr
|
|
}
|
|
|
|
// servers are the given control planes, or this machine at the address it
|
|
// uses to reach the node — which is the one the node can reach it at, and
|
|
// is named in the server certificate when init issued it here.
|
|
func (r *Runner) servers() ([]string, error) {
|
|
if len(r.Opts.Servers) > 0 {
|
|
return r.Opts.Servers, nil
|
|
}
|
|
host := r.Opts.Addr
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
c, err := net.Dial("udp", net.JoinHostPort(host, "7443"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot tell which address the node would reach this machine at: %v; give it with --server", err)
|
|
}
|
|
local := c.LocalAddr().(*net.UDPAddr).IP.String()
|
|
_ = c.Close()
|
|
if names, err := certNames(filepath.Join(r.etcDir(), "server.pem")); err == nil && !slices.Contains(names, local) {
|
|
return nil, fmt.Errorf("the node would reach this machine at %s, which the server certificate does not name (%s): "+
|
|
"run `sudo llmgw server init --host %s` to add it, or give the address with --server",
|
|
local, strings.Join(names, ", "), local)
|
|
}
|
|
return []string{"https://" + net.JoinHostPort(local, "7443")}, nil
|
|
}
|
|
|
|
func (r *Runner) etcDir() string {
|
|
if r.Opts.EtcDir != "" {
|
|
return r.Opts.EtcDir
|
|
}
|
|
return "/etc/llmgw"
|
|
}
|
|
|
|
func certNames(path string) ([]string, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
blk, _ := pem.Decode(b)
|
|
if blk == nil {
|
|
return nil, errors.New("no certificate in " + path)
|
|
}
|
|
c, err := x509.ParseCertificate(blk.Bytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
names := slices.Clone(c.DNSNames)
|
|
for _, ip := range c.IPAddresses {
|
|
names = append(names, ip.String())
|
|
}
|
|
return names, nil
|
|
}
|
|
|
|
// --- binary ---
|
|
|
|
// releaseTag is a version CI published: v0.10.4, or a release candidate of
|
|
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
|
|
// release to download.
|
|
var releaseTag = release.Tag
|
|
|
|
func (r *Runner) binary(ctx context.Context, res *Result) error {
|
|
v := r.Opts.Version
|
|
if releaseTag.MatchString(v) {
|
|
r.emit(StepBinary, Running, "downloading llmgw "+v+" on the node", "")
|
|
name := "llmgw-" + v + "-linux-" + res.Arch
|
|
base := strings.TrimSuffix(r.Opts.Releases, "/") + "/" + v
|
|
// Fetched by the node, for its own architecture, and checked against
|
|
// the release's checksums, rather than handed whatever this machine
|
|
// happens to run.
|
|
script := fmt.Sprintf(`set -e
|
|
d=$(mktemp -d); cd "$d"
|
|
curl -fsSL -o %[1]s '%[2]s/%[1]s'
|
|
curl -fsSL -o checksums.txt '%[2]s/checksums.txt'
|
|
grep " %[1]s$" checksums.txt | sha256sum -c - >/dev/null
|
|
sudo install -m 0755 %[1]s /usr/local/bin/llmgw
|
|
cd /; rm -rf "$d"`, name, base)
|
|
out, err := r.Remote.Run(ctx, script, nil)
|
|
if err == nil {
|
|
r.emit(StepBinary, Done, "llmgw "+v+" from the release, checksum verified", "")
|
|
return nil
|
|
}
|
|
r.emit(StepBinary, Warn, "the node could not download the release: "+lastLine(out), "")
|
|
}
|
|
// A development build has no release to fetch, and a node that cannot
|
|
// reach the releases gets this machine's copy — when it can run it.
|
|
if res.Arch != hostArch() {
|
|
return fmt.Errorf("the node is %s and this machine %s, so its binary cannot be copied: "+
|
|
"let the node reach %s, or install llmgw there yourself", res.Arch, hostArch(), r.Opts.Releases)
|
|
}
|
|
r.emit(StepBinary, Running, "copying this machine's llmgw ("+orUnknown(v)+")", "")
|
|
b, err := os.ReadFile(r.Opts.Binary)
|
|
if err != nil {
|
|
return fmt.Errorf("reading %s: %w", r.Opts.Binary, err)
|
|
}
|
|
if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
|
|
return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out))
|
|
}
|
|
r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "")
|
|
return nil
|
|
}
|
|
|
|
// --- check ---
|
|
|
|
func (r *Runner) check(ctx context.Context, res *Result) error {
|
|
r.emit(StepCheck, Running, "running llmgw doctor on the node", "")
|
|
args := "sudo llmgw doctor --json"
|
|
if r.Opts.NvidiaSMI != "" {
|
|
args += " --nvidia-smi " + shellQuote(r.Opts.NvidiaSMI)
|
|
}
|
|
if r.Opts.IBRoot != "" {
|
|
args += " --ib-root " + shellQuote(r.Opts.IBRoot)
|
|
}
|
|
out, err := r.Remote.Run(ctx, args, nil)
|
|
var rep struct {
|
|
Findings []doctor.Finding `json:"findings"`
|
|
Facts doctor.Facts `json:"facts"`
|
|
}
|
|
if jerr := json.Unmarshal(out, &rep); jerr != nil {
|
|
if err == nil {
|
|
err = jerr
|
|
}
|
|
return fmt.Errorf("llmgw doctor did not run on the node: %v: %s", err, lastLine(out))
|
|
}
|
|
res.Facts, res.Findings = rep.Facts, rep.Findings
|
|
|
|
fails := 0
|
|
for _, f := range rep.Findings {
|
|
switch f.Level {
|
|
case doctor.Fail:
|
|
fails++
|
|
r.emit(StepCheck, Fail, f.Area+": "+f.Message, f.Fix)
|
|
case doctor.Warn:
|
|
r.emit(StepCheck, Warn, f.Area+": "+f.Message, f.Fix)
|
|
case doctor.OK:
|
|
r.emit(StepCheck, Info, f.Area+": "+f.Message, "")
|
|
}
|
|
}
|
|
res.Suggested = suggestFabrics(rep.Facts.HCAs)
|
|
if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 && r.Opts.AskFabrics != nil {
|
|
r.Opts.Fabrics = r.Opts.AskFabrics(res.Suggested)
|
|
for _, f := range r.Opts.Fabrics {
|
|
r.emit(StepCheck, Info, "fabric "+f.Name+": "+strings.Join(f.HCAs, ", "), "")
|
|
}
|
|
}
|
|
if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 {
|
|
var flags []string
|
|
for _, f := range res.Suggested {
|
|
flags = append(flags, "--fabric "+f.Name+":"+strings.Join(f.HCAs, ","))
|
|
}
|
|
r.emit(StepCheck, Warn, "InfiniBand or RoCE adapters are up, but no fabric was given: this node would never be given a replica spanning nodes",
|
|
"add "+strings.Join(flags, " ")+", naming each fabric the same on every node attached to it")
|
|
}
|
|
goOn := r.Opts.IgnoreChecks
|
|
if fails > 0 && !goOn && r.Opts.AskContinue != nil && r.Opts.AskContinue(fails) {
|
|
r.emit(StepCheck, Warn, fmt.Sprintf("going on despite %d problem(s)", fails), "")
|
|
goOn = true
|
|
}
|
|
if fails > 0 && !goOn {
|
|
return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails)
|
|
}
|
|
r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "")
|
|
return nil
|
|
}
|
|
|
|
// suggestFabrics groups the active adapters by the fabric they report being
|
|
// on: the subnet manager they see, or the network of their RoCE address.
|
|
// Names are placeholders for the operator: a name is what ties one node's
|
|
// adapters to another's, and nothing on one node can know what the others
|
|
// were called.
|
|
func suggestFabrics(hcas []discovery.HCA) []Fabric {
|
|
byKey := map[string][]string{}
|
|
var keys []string
|
|
for _, h := range hcas {
|
|
for _, p := range h.ActivePorts() {
|
|
k := p.FabricKey()
|
|
if k == "" {
|
|
k = "unknown"
|
|
}
|
|
if _, ok := byKey[k]; !ok {
|
|
keys = append(keys, k)
|
|
}
|
|
if !slices.Contains(byKey[k], h.Name) {
|
|
byKey[k] = append(byKey[k], h.Name)
|
|
}
|
|
}
|
|
}
|
|
var out []Fabric
|
|
for i, k := range keys {
|
|
name := "fabric"
|
|
if len(keys) > 1 {
|
|
name = fmt.Sprintf("fabric%d", i+1)
|
|
}
|
|
out = append(out, Fabric{Name: name, HCAs: byKey[k], Key: k})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// --- certificate ---
|
|
|
|
func (r *Runner) certificate(ctx context.Context, res *Result) error {
|
|
r.emit(StepCert, Running, "issuing a certificate for "+res.Node, "")
|
|
ca := r.ca
|
|
// Issued fresh every time: onboarding again should not depend on finding
|
|
// an old key, and the old one keeps working until revoked.
|
|
leaf, err := ca.IssueClient(res.Node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, f := range []struct {
|
|
path, mode string
|
|
body []byte
|
|
}{
|
|
{"/etc/llmgw/ca.pem", "0644", ca.PEM},
|
|
{"/etc/llmgw/agent.pem", "0644", leaf.CertPEM},
|
|
{"/etc/llmgw/agent-key.pem", "0600", leaf.KeyPEM},
|
|
} {
|
|
if err := r.put(ctx, f.path, f.mode, f.body); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// The agent trusts the cluster's root through its own configuration only.
|
|
// Not system-wide: that root signs whatever the control plane asks it to,
|
|
// and every TLS client on the machine would then take it for any site.
|
|
// A node onboarded by an older version had it added; take it back out.
|
|
_, _ = r.Remote.Run(ctx, `if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then
|
|
sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt && sudo update-ca-certificates --fresh >/dev/null 2>&1
|
|
fi
|
|
if [ -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem ]; then
|
|
sudo rm -f /etc/pki/ca-trust/source/anchors/llmgw-ca.pem && sudo update-ca-trust >/dev/null 2>&1
|
|
fi; true`, nil)
|
|
r.emit(StepCert, Done, "certificate for "+res.Node+" installed", "")
|
|
return nil
|
|
}
|
|
|
|
// put writes a file on the node as root, never readable by others on the
|
|
// way when its mode says it should not be.
|
|
func (r *Runner) put(ctx context.Context, path, mode string, body []byte) error {
|
|
script := fmt.Sprintf(`set -e
|
|
sudo install -d -m 0755 %[1]s
|
|
sudo sh -c 'umask 077; cat > %[2]s.new'
|
|
sudo chmod %[3]s %[2]s.new
|
|
sudo chown root:root %[2]s.new
|
|
sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode)
|
|
if out, err := r.Remote.Run(ctx, script, body); err != nil {
|
|
return fmt.Errorf("writing %s: %v: %s", path, err, lastLine(out))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// --- configuration ---
|
|
|
|
// AgentUnit is the agent's systemd unit, the same one server init writes.
|
|
var AgentUnit string
|
|
|
|
func (r *Runner) configure(ctx context.Context, res *Result) error {
|
|
r.emit(StepConfig, Running, "writing /etc/llmgw/agent.yaml", "")
|
|
if err := r.put(ctx, "/etc/llmgw/agent.yaml", "0644", []byte(AgentYAML(r.Opts, res))); err != nil {
|
|
return err
|
|
}
|
|
if AgentUnit == "" {
|
|
return errors.New("onboard: AgentUnit is not set")
|
|
}
|
|
if err := r.put(ctx, "/etc/systemd/system/llmgw-agent.service", "0644", []byte(AgentUnit)); err != nil {
|
|
return err
|
|
}
|
|
res.EngineProblems = r.checkEngines(ctx)
|
|
for _, p := range res.EngineProblems {
|
|
r.emit(StepConfig, Warn, p, "")
|
|
}
|
|
r.emit(StepConfig, Done, "agent.yaml and the systemd unit", "")
|
|
return nil
|
|
}
|
|
|
|
// AgentYAML is the node's configuration: only what the node cannot find out
|
|
// for itself. Its address, GPUs, adapters and engines it discovers, and every
|
|
// line written here is one somebody has to keep true by hand.
|
|
func AgentYAML(o Options, res *Result) string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "# %s, written by 'llmgw node add'.\n", res.Node)
|
|
if len(res.Servers) == 1 {
|
|
fmt.Fprintf(&b, "server: %q\n", res.Servers[0])
|
|
} else {
|
|
// Every control plane it may fail over to; a standby refuses it and
|
|
// it moves on to the next.
|
|
b.WriteString("servers:\n")
|
|
for _, s := range res.Servers {
|
|
fmt.Fprintf(&b, " - %q\n", s)
|
|
}
|
|
}
|
|
b.WriteString("\ntls:\n ca: /etc/llmgw/ca.pem\n cert: /etc/llmgw/agent.pem\n key: /etc/llmgw/agent-key.pem\n")
|
|
// The control plane refuses a node whose id differs from its
|
|
// certificate's name; said only when the hostname would say otherwise.
|
|
if res.Node != res.Hostname {
|
|
fmt.Fprintf(&b, "node_id: %q\n", res.Node)
|
|
}
|
|
if o.MgmtAddr != "" {
|
|
fmt.Fprintf(&b, "mgmt_addr: %q\n", o.MgmtAddr)
|
|
}
|
|
if o.MgmtIface != "" {
|
|
fmt.Fprintf(&b, "mgmt_interface: %q\n", o.MgmtIface)
|
|
}
|
|
if len(o.Fabrics) > 0 {
|
|
// The one thing a node cannot find out: which name its fabric goes
|
|
// by on every other node.
|
|
b.WriteString("fabrics:\n")
|
|
for _, f := range o.Fabrics {
|
|
fmt.Fprintf(&b, " - name: %q\n hcas: [%s]\n", f.Name, strings.Join(f.HCAs, ", "))
|
|
}
|
|
}
|
|
if o.NvidiaSMI != "" || o.IBRoot != "" {
|
|
b.WriteString("discovery:\n")
|
|
if o.NvidiaSMI != "" {
|
|
fmt.Fprintf(&b, " nvidia_smi: %q\n", o.NvidiaSMI)
|
|
}
|
|
if o.IBRoot != "" {
|
|
fmt.Fprintf(&b, " ib_root: %q\n", o.IBRoot)
|
|
}
|
|
}
|
|
if o.LlamaCppBin != "" || o.VLLMBin != "" || o.SGLangPython != "" {
|
|
b.WriteString("engines:\n")
|
|
if o.LlamaCppBin != "" {
|
|
fmt.Fprintf(&b, " llamacpp:\n bin: %q\n", o.LlamaCppBin)
|
|
}
|
|
if o.VLLMBin != "" {
|
|
fmt.Fprintf(&b, " vllm:\n bin: %q\n", o.VLLMBin)
|
|
}
|
|
if o.SGLangPython != "" {
|
|
fmt.Fprintf(&b, " sglang:\n python: %q\n", o.SGLangPython)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// checkEngines runs each engine named by path. A wrong one registers as
|
|
// healthy and fails every shard placed on it, with the reason three logs
|
|
// away: llama-server on node05 was present, executable, and missing libgomp.
|
|
func (r *Runner) checkEngines(ctx context.Context) []string {
|
|
var problems []string
|
|
for _, e := range []struct{ name, path string }{
|
|
{"llama.cpp", r.Opts.LlamaCppBin}, {"vLLM", r.Opts.VLLMBin}, {"SGLang", r.Opts.SGLangPython},
|
|
} {
|
|
if e.path == "" {
|
|
continue
|
|
}
|
|
q := shellQuote(e.path)
|
|
if _, err := r.Remote.Run(ctx, "test -x "+q, nil); err != nil {
|
|
problems = append(problems, fmt.Sprintf("%s is set to %s, which is not an executable on the node", e.name, e.path))
|
|
continue
|
|
}
|
|
if _, err := r.Remote.Run(ctx, q+" --version >/dev/null 2>&1 || "+q+" --help >/dev/null 2>&1", nil); err != nil {
|
|
problems = append(problems, fmt.Sprintf("%s at %s exists but will not run; usually a missing library: ldd %s | grep 'not found'",
|
|
e.name, e.path, e.path))
|
|
}
|
|
}
|
|
return problems
|
|
}
|
|
|
|
// --- start ---
|
|
|
|
func (r *Runner) start(ctx context.Context, _ *Result) error {
|
|
r.emit(StepStart, Running, "starting the agent", "")
|
|
if out, err := r.Remote.Run(ctx, `set -e
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable llmgw-agent >/dev/null 2>&1
|
|
sudo systemctl restart llmgw-agent
|
|
sleep 2
|
|
systemctl is-active llmgw-agent`, nil); err != nil {
|
|
return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out))
|
|
}
|
|
r.emit(StepStart, Done, "llmgw-agent is running", "")
|
|
return nil
|
|
}
|
|
|
|
// --- helpers ---
|
|
|
|
func lastLine(b []byte) string {
|
|
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
|
return strings.TrimSpace(lines[len(lines)-1])
|
|
}
|
|
|
|
func orUnknown(s string) string {
|
|
if s == "" {
|
|
return "unversioned"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func countLevel(fs []doctor.Finding, l doctor.Level) int {
|
|
n := 0
|
|
for _, f := range fs {
|
|
if f.Level == l {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// shellQuote quotes a string for a POSIX shell.
|
|
func shellQuote(s string) string {
|
|
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
|
}
|