Files
llmgw/internal/onboard/onboard.go
T
Ilker ManapandClaude Opus 5.5 a1b5daceb8 Let server init ask for what it needs, and node add take several nodes
In a terminal and without --admin, server init shows a short form: the
administrator's email, the other names nodes will use (with the addresses
it found), SQLite or PostgreSQL, and the mirror, node and starter model.
Its own lines then fill in as a list — read from the writer init already
uses, so nothing in init changed — with downloads updating in place and the
password in a box until the end. The whole run, password included, is
printed again when the screen closes, so it is in the terminal's history.
The last screen says how to add a node and to back up the authority's key.

node add takes several addresses and adds them in turn, with a summary.
Suggested fabrics carry what the adapters report them to be, so a name
given for one node is used for the next on the same fabric without asking.

Tried on empty VMs: init from the form through to the starter model
serving, and two nodes added to that cluster in one command. The first
attempt at the form lost its keystrokes to the terminal colour query in a
pseudo-terminal that never answers it; a real terminal does.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 21:54:18 +02:00

675 lines
22 KiB
Go

// Package onboard makes a machine a node of the cluster, over SSH, from the
// machine holding the certificate authority.
//
// It is what deploy/bootstrap/onboard-node.sh did, moved into the binary so
// there is nothing else to fetch, and extended by what a script could not
// easily do: it asks the machine its own name and architecture instead of
// being told, runs llmgw doctor there before changing anything, and suggests
// the fabric its adapters are on.
//
// It reports each step as an Event, so a plain terminal and a full-screen one
// show the same thing from the same code.
package onboard
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// Fabric is one InfiniBand or RoCE fabric the node is on, and its adapters.
type Fabric struct {
Name string
HCAs []string
// Key is what the adapters report the fabric to be — its subnet manager,
// or a RoCE network — on a suggestion, so a name given for one node can
// be given again to the next on the same fabric.
Key string
}
// ParseFabric reads "name:hca1,hca2", or "name" alone.
func ParseFabric(s string) (Fabric, error) {
name, hcas, _ := strings.Cut(strings.TrimSpace(s), ":")
if name == "" {
return Fabric{}, fmt.Errorf("fabric %q has no name: give it as name:mlx5_0,mlx5_1", s)
}
f := Fabric{Name: name}
for _, h := range strings.Split(hcas, ",") {
if h = strings.TrimSpace(h); h != "" {
f.HCAs = append(f.HCAs, h)
}
}
return f, nil
}
// Options say which machine, and what to tell its agent.
type Options struct {
// Addr is how to reach the machine over SSH, and User who to log in as.
Addr, User string
// Node is its id in the cluster. Empty means its hostname, which is what
// it should be: the node id, the hostname and the certificate's name
// being one string saves an afternoon each time they would differ.
Node string
// Servers are the control planes the agent dials. Empty means this
// machine, at the address it has on the route to the node.
Servers []string
Fabrics []Fabric
MgmtAddr, MgmtIface string
NvidiaSMI, IBRoot string
LlamaCppBin string
VLLMBin string
SGLangPython string
// PKI is the authority's directory; EtcDir is where this machine's own
// server certificate is, to check the chosen address is named in it.
PKI, EtcDir string
// Binary is the llmgw to install when the node cannot download the
// release, and Version what it is.
Binary, Version string
// Releases is where release files are downloaded from.
Releases string
// IgnoreChecks goes on when doctor finds a problem.
IgnoreChecks bool
// SSHWait is how long an unreachable machine is waited for.
SSHWait time.Duration
// AskFabrics, when set, is asked which fabrics to declare for a node
// given none whose adapters are up, with the adapters grouped by the
// fabric they report. It returns the fabrics to use, none to go on
// without. Nil leaves it at a warning, as a script needs.
AskFabrics func(suggested []Fabric) []Fabric
// AskContinue, when set, is asked whether to go on after doctor found
// problems. Nil stops unless IgnoreChecks.
AskContinue func(fails int) bool
}
// Levels of an Event.
const (
Running = "running"
Done = "done"
Info = "info"
Warn = "warn"
Fail = "fail"
)
// Event is one thing that happened: a step starting or ending, or something
// worth saying along the way.
type Event struct {
Step string
Level string
Detail string
// Fix is what to do about a warning or a failure, when there is a known
// answer.
Fix string
}
// The steps, in order.
const (
StepConnect = "connect"
StepCheck = "check"
StepBinary = "binary"
StepCert = "certificate"
StepConfig = "configuration"
StepStart = "start"
// StepJoin and StepEngines are watched from the control plane once the
// agent runs; the caller does that, since it has the admin API.
StepJoin = "join"
StepEngines = "engines"
)
// Steps lists them for a display that shows what is still to come.
var Steps = []string{StepConnect, StepBinary, StepCheck, StepCert, StepConfig, StepStart, StepJoin, StepEngines}
// Result is what onboarding learned and did.
type Result struct {
Node string
Arch string
Hostname string
Servers []string
Facts doctor.Facts
Findings []doctor.Finding
// Suggested is the fabrics the node's active adapters are on, for a node
// given none: grouped by what the adapters report, named for the operator
// to choose.
Suggested []Fabric
// EngineProblems are engines named by path that will not run there.
EngineProblems []string
}
// Runner onboards one machine.
type Runner struct {
Opts Options
Remote Remote
// Emit hears every event. Nil discards them.
Emit func(Event)
ca *tlsconf.CA
}
// New makes a Runner that reaches the machine with the system's ssh.
func New(o Options, emit func(Event)) *Runner {
return &Runner{Opts: o, Remote: &SSH{Addr: o.Addr, User: o.User}, Emit: emit}
}
func (r *Runner) emit(step, level, detail, fix string) {
if r.Emit != nil {
r.Emit(Event{Step: step, Level: level, Detail: detail, Fix: fix})
}
}
// ErrChecks is returned when doctor found problems and IgnoreChecks is off.
var ErrChecks = errors.New("the node has problems to fix first")
// Run does every step, stopping at the first that fails.
func (r *Runner) Run(ctx context.Context) (*Result, error) {
res := &Result{}
// The authority first: without it nothing can be finished, and finding
// out after the node has been changed leaves it half done.
pki := r.Opts.PKI
ca, err := tlsconf.LoadCA(filepath.Join(pki, "ca.pem"), filepath.Join(pki, "ca-key.pem"))
if err != nil {
err = fmt.Errorf("cannot read the certificate authority in %s: %v. Run this on the machine that holds it, as root (sudo), or give --pki", pki, err)
r.emit(StepConnect, Fail, err.Error(), "")
return res, err
}
r.ca = ca
for _, step := range []struct {
name string
fn func(context.Context, *Result) error
}{
{StepConnect, r.connect},
{StepBinary, r.binary},
{StepCheck, r.check},
{StepCert, r.certificate},
{StepConfig, r.configure},
{StepStart, r.start},
} {
if err := step.fn(ctx, res); err != nil {
r.emit(step.name, Fail, err.Error(), "")
return res, err
}
}
return res, nil
}
// --- connect ---
var nodeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`)
func (r *Runner) connect(ctx context.Context, res *Result) error {
r.emit(StepConnect, Running, "connecting to "+r.target(), "")
wait := r.Opts.SSHWait
if wait == 0 {
wait = 3 * time.Minute
}
// Waited for rather than failed on: a machine a minute old restarts sshd
// after cloud-init regenerates its host keys, and refuses connections for
// a few seconds after answering.
deadline := time.Now().Add(wait)
var last error
for {
out, err := r.Remote.Run(ctx, "hostname && uname -m", nil)
if err == nil {
lines := strings.Fields(string(out))
if len(lines) >= 2 {
res.Hostname, res.Arch = lines[0], lines[1]
break
}
err = fmt.Errorf("unexpected answer: %q", out)
}
last = err
if time.Now().After(deadline) {
return fmt.Errorf("cannot log in to %s: %v. Check that `ssh %s true` works from this machine, as the user running this, without a password",
r.target(), last, r.target())
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(5 * time.Second):
}
}
if _, err := r.Remote.Run(ctx, "sudo -n true", nil); err != nil {
return fmt.Errorf("%s can log in but cannot use sudo without a password, which installing the agent needs", r.target())
}
switch res.Arch {
case "x86_64", "amd64":
res.Arch = "amd64"
case "aarch64", "arm64":
res.Arch = "arm64"
default:
return fmt.Errorf("%s is %s; llmgw is built for x86-64 and ARM64", r.target(), res.Arch)
}
res.Node = r.Opts.Node
if res.Node == "" {
res.Node = res.Hostname
}
if !nodeName.MatchString(res.Node) {
return fmt.Errorf("%q cannot be a node id: letters, digits, dots, dashes and underscores; give one with --node", res.Node)
}
servers, err := r.servers()
if err != nil {
return err
}
res.Servers = servers
r.emit(StepConnect, Done, fmt.Sprintf("%s (%s), joining as %s, agent will dial %s",
res.Hostname, res.Arch, res.Node, strings.Join(servers, ", ")), "")
return nil
}
func (r *Runner) target() string {
if r.Opts.User == "" {
return r.Opts.Addr
}
return r.Opts.User + "@" + r.Opts.Addr
}
// servers are the given control planes, or this machine at the address it
// uses to reach the node — which is the one the node can reach it at, and
// is named in the server certificate when init issued it here.
func (r *Runner) servers() ([]string, error) {
if len(r.Opts.Servers) > 0 {
return r.Opts.Servers, nil
}
host := r.Opts.Addr
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
c, err := net.Dial("udp", net.JoinHostPort(host, "7443"))
if err != nil {
return nil, fmt.Errorf("cannot tell which address the node would reach this machine at: %v; give it with --server", err)
}
local := c.LocalAddr().(*net.UDPAddr).IP.String()
_ = c.Close()
if names, err := certNames(filepath.Join(r.etcDir(), "server.pem")); err == nil && !slices.Contains(names, local) {
return nil, fmt.Errorf("the node would reach this machine at %s, which the server certificate does not name (%s): "+
"run `sudo llmgw server init --host %s` to add it, or give the address with --server",
local, strings.Join(names, ", "), local)
}
return []string{"https://" + net.JoinHostPort(local, "7443")}, nil
}
func (r *Runner) etcDir() string {
if r.Opts.EtcDir != "" {
return r.Opts.EtcDir
}
return "/etc/llmgw"
}
func certNames(path string) ([]string, error) {
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
blk, _ := pem.Decode(b)
if blk == nil {
return nil, errors.New("no certificate in " + path)
}
c, err := x509.ParseCertificate(blk.Bytes)
if err != nil {
return nil, err
}
names := slices.Clone(c.DNSNames)
for _, ip := range c.IPAddresses {
names = append(names, ip.String())
}
return names, nil
}
// --- binary ---
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
func (r *Runner) binary(ctx context.Context, res *Result) error {
v := r.Opts.Version
if releaseTag.MatchString(v) {
r.emit(StepBinary, Running, "downloading llmgw "+v+" on the node", "")
name := "llmgw-" + v + "-linux-" + res.Arch
base := strings.TrimSuffix(r.Opts.Releases, "/") + "/" + v
// Fetched by the node, for its own architecture, and checked against
// the release's checksums, rather than handed whatever this machine
// happens to run.
script := fmt.Sprintf(`set -e
d=$(mktemp -d); cd "$d"
curl -fsSL -o %[1]s '%[2]s/%[1]s'
curl -fsSL -o checksums.txt '%[2]s/checksums.txt'
grep " %[1]s$" checksums.txt | sha256sum -c - >/dev/null
sudo install -m 0755 %[1]s /usr/local/bin/llmgw
cd /; rm -rf "$d"`, name, base)
out, err := r.Remote.Run(ctx, script, nil)
if err == nil {
r.emit(StepBinary, Done, "llmgw "+v+" from the release, checksum verified", "")
return nil
}
r.emit(StepBinary, Warn, "the node could not download the release: "+lastLine(out), "")
}
// A development build has no release to fetch, and a node that cannot
// reach the releases gets this machine's copy — when it can run it.
if res.Arch != hostArch() {
return fmt.Errorf("the node is %s and this machine %s, so its binary cannot be copied: "+
"let the node reach %s, or install llmgw there yourself", res.Arch, hostArch(), r.Opts.Releases)
}
r.emit(StepBinary, Running, "copying this machine's llmgw ("+orUnknown(v)+")", "")
b, err := os.ReadFile(r.Opts.Binary)
if err != nil {
return fmt.Errorf("reading %s: %w", r.Opts.Binary, err)
}
if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out))
}
r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "")
return nil
}
// --- check ---
func (r *Runner) check(ctx context.Context, res *Result) error {
r.emit(StepCheck, Running, "running llmgw doctor on the node", "")
args := "sudo llmgw doctor --json"
if r.Opts.NvidiaSMI != "" {
args += " --nvidia-smi " + shellQuote(r.Opts.NvidiaSMI)
}
if r.Opts.IBRoot != "" {
args += " --ib-root " + shellQuote(r.Opts.IBRoot)
}
out, err := r.Remote.Run(ctx, args, nil)
var rep struct {
Findings []doctor.Finding `json:"findings"`
Facts doctor.Facts `json:"facts"`
}
if jerr := json.Unmarshal(out, &rep); jerr != nil {
if err == nil {
err = jerr
}
return fmt.Errorf("llmgw doctor did not run on the node: %v: %s", err, lastLine(out))
}
res.Facts, res.Findings = rep.Facts, rep.Findings
fails := 0
for _, f := range rep.Findings {
switch f.Level {
case doctor.Fail:
fails++
r.emit(StepCheck, Fail, f.Area+": "+f.Message, f.Fix)
case doctor.Warn:
r.emit(StepCheck, Warn, f.Area+": "+f.Message, f.Fix)
case doctor.OK:
r.emit(StepCheck, Info, f.Area+": "+f.Message, "")
}
}
res.Suggested = suggestFabrics(rep.Facts.HCAs)
if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 && r.Opts.AskFabrics != nil {
r.Opts.Fabrics = r.Opts.AskFabrics(res.Suggested)
for _, f := range r.Opts.Fabrics {
r.emit(StepCheck, Info, "fabric "+f.Name+": "+strings.Join(f.HCAs, ", "), "")
}
}
if len(r.Opts.Fabrics) == 0 && len(res.Suggested) > 0 {
var flags []string
for _, f := range res.Suggested {
flags = append(flags, "--fabric "+f.Name+":"+strings.Join(f.HCAs, ","))
}
r.emit(StepCheck, Warn, "InfiniBand or RoCE adapters are up, but no fabric was given: this node would never be given a replica spanning nodes",
"add "+strings.Join(flags, " ")+", naming each fabric the same on every node attached to it")
}
goOn := r.Opts.IgnoreChecks
if fails > 0 && !goOn && r.Opts.AskContinue != nil && r.Opts.AskContinue(fails) {
r.emit(StepCheck, Warn, fmt.Sprintf("going on despite %d problem(s)", fails), "")
goOn = true
}
if fails > 0 && !goOn {
return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails)
}
r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "")
return nil
}
// suggestFabrics groups the active adapters by the fabric they report being
// on: the subnet manager they see, or the network of their RoCE address.
// Names are placeholders for the operator: a name is what ties one node's
// adapters to another's, and nothing on one node can know what the others
// were called.
func suggestFabrics(hcas []discovery.HCA) []Fabric {
byKey := map[string][]string{}
var keys []string
for _, h := range hcas {
for _, p := range h.ActivePorts() {
k := p.FabricKey()
if k == "" {
k = "unknown"
}
if _, ok := byKey[k]; !ok {
keys = append(keys, k)
}
if !slices.Contains(byKey[k], h.Name) {
byKey[k] = append(byKey[k], h.Name)
}
}
}
var out []Fabric
for i, k := range keys {
name := "fabric"
if len(keys) > 1 {
name = fmt.Sprintf("fabric%d", i+1)
}
out = append(out, Fabric{Name: name, HCAs: byKey[k], Key: k})
}
return out
}
// --- certificate ---
func (r *Runner) certificate(ctx context.Context, res *Result) error {
r.emit(StepCert, Running, "issuing a certificate for "+res.Node, "")
ca := r.ca
// Issued fresh every time: onboarding again should not depend on finding
// an old key, and the old one keeps working until revoked.
leaf, err := ca.IssueClient(res.Node)
if err != nil {
return err
}
for _, f := range []struct {
path, mode string
body []byte
}{
{"/etc/llmgw/ca.pem", "0644", ca.PEM},
{"/etc/llmgw/agent.pem", "0644", leaf.CertPEM},
{"/etc/llmgw/agent-key.pem", "0600", leaf.KeyPEM},
} {
if err := r.put(ctx, f.path, f.mode, f.body); err != nil {
return err
}
}
// Trusting the root too, so other tools on the machine do not treat the
// cluster's certificates as strangers. Best effort: either family's way.
_, _ = r.Remote.Run(ctx, `if [ -d /usr/local/share/ca-certificates ]; then
sudo install -m 0644 /etc/llmgw/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt && sudo update-ca-certificates >/dev/null 2>&1
elif [ -d /etc/pki/ca-trust/source/anchors ]; then
sudo install -m 0644 /etc/llmgw/ca.pem /etc/pki/ca-trust/source/anchors/llmgw-ca.pem && sudo update-ca-trust >/dev/null 2>&1
fi; true`, nil)
r.emit(StepCert, Done, "certificate for "+res.Node+" installed, the cluster's root trusted", "")
return nil
}
// put writes a file on the node as root, never readable by others on the
// way when its mode says it should not be.
func (r *Runner) put(ctx context.Context, path, mode string, body []byte) error {
script := fmt.Sprintf(`set -e
sudo install -d -m 0755 %[1]s
sudo sh -c 'umask 077; cat > %[2]s.new'
sudo chmod %[3]s %[2]s.new
sudo chown root:root %[2]s.new
sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode)
if out, err := r.Remote.Run(ctx, script, body); err != nil {
return fmt.Errorf("writing %s: %v: %s", path, err, lastLine(out))
}
return nil
}
// --- configuration ---
// AgentUnit is the agent's systemd unit, the same one server init writes.
var AgentUnit string
func (r *Runner) configure(ctx context.Context, res *Result) error {
r.emit(StepConfig, Running, "writing /etc/llmgw/agent.yaml", "")
if err := r.put(ctx, "/etc/llmgw/agent.yaml", "0644", []byte(AgentYAML(r.Opts, res))); err != nil {
return err
}
if AgentUnit == "" {
return errors.New("onboard: AgentUnit is not set")
}
if err := r.put(ctx, "/etc/systemd/system/llmgw-agent.service", "0644", []byte(AgentUnit)); err != nil {
return err
}
res.EngineProblems = r.checkEngines(ctx)
for _, p := range res.EngineProblems {
r.emit(StepConfig, Warn, p, "")
}
r.emit(StepConfig, Done, "agent.yaml and the systemd unit", "")
return nil
}
// AgentYAML is the node's configuration: only what the node cannot find out
// for itself. Its address, GPUs, adapters and engines it discovers, and every
// line written here is one somebody has to keep true by hand.
func AgentYAML(o Options, res *Result) string {
var b strings.Builder
fmt.Fprintf(&b, "# %s, written by 'llmgw node add'.\n", res.Node)
if len(res.Servers) == 1 {
fmt.Fprintf(&b, "server: %q\n", res.Servers[0])
} else {
// Every control plane it may fail over to; a standby refuses it and
// it moves on to the next.
b.WriteString("servers:\n")
for _, s := range res.Servers {
fmt.Fprintf(&b, " - %q\n", s)
}
}
b.WriteString("\ntls:\n ca: /etc/llmgw/ca.pem\n cert: /etc/llmgw/agent.pem\n key: /etc/llmgw/agent-key.pem\n")
// The control plane refuses a node whose id differs from its
// certificate's name; said only when the hostname would say otherwise.
if res.Node != res.Hostname {
fmt.Fprintf(&b, "node_id: %q\n", res.Node)
}
if o.MgmtAddr != "" {
fmt.Fprintf(&b, "mgmt_addr: %q\n", o.MgmtAddr)
}
if o.MgmtIface != "" {
fmt.Fprintf(&b, "mgmt_interface: %q\n", o.MgmtIface)
}
if len(o.Fabrics) > 0 {
// The one thing a node cannot find out: which name its fabric goes
// by on every other node.
b.WriteString("fabrics:\n")
for _, f := range o.Fabrics {
fmt.Fprintf(&b, " - name: %q\n hcas: [%s]\n", f.Name, strings.Join(f.HCAs, ", "))
}
}
if o.NvidiaSMI != "" || o.IBRoot != "" {
b.WriteString("discovery:\n")
if o.NvidiaSMI != "" {
fmt.Fprintf(&b, " nvidia_smi: %q\n", o.NvidiaSMI)
}
if o.IBRoot != "" {
fmt.Fprintf(&b, " ib_root: %q\n", o.IBRoot)
}
}
if o.LlamaCppBin != "" || o.VLLMBin != "" || o.SGLangPython != "" {
b.WriteString("engines:\n")
if o.LlamaCppBin != "" {
fmt.Fprintf(&b, " llamacpp:\n bin: %q\n", o.LlamaCppBin)
}
if o.VLLMBin != "" {
fmt.Fprintf(&b, " vllm:\n bin: %q\n", o.VLLMBin)
}
if o.SGLangPython != "" {
fmt.Fprintf(&b, " sglang:\n python: %q\n", o.SGLangPython)
}
}
return b.String()
}
// checkEngines runs each engine named by path. A wrong one registers as
// healthy and fails every shard placed on it, with the reason three logs
// away: llama-server on node05 was present, executable, and missing libgomp.
func (r *Runner) checkEngines(ctx context.Context) []string {
var problems []string
for _, e := range []struct{ name, path string }{
{"llama.cpp", r.Opts.LlamaCppBin}, {"vLLM", r.Opts.VLLMBin}, {"SGLang", r.Opts.SGLangPython},
} {
if e.path == "" {
continue
}
q := shellQuote(e.path)
if _, err := r.Remote.Run(ctx, "test -x "+q, nil); err != nil {
problems = append(problems, fmt.Sprintf("%s is set to %s, which is not an executable on the node", e.name, e.path))
continue
}
if _, err := r.Remote.Run(ctx, q+" --version >/dev/null 2>&1 || "+q+" --help >/dev/null 2>&1", nil); err != nil {
problems = append(problems, fmt.Sprintf("%s at %s exists but will not run; usually a missing library: ldd %s | grep 'not found'",
e.name, e.path, e.path))
}
}
return problems
}
// --- start ---
func (r *Runner) start(ctx context.Context, _ *Result) error {
r.emit(StepStart, Running, "starting the agent", "")
if out, err := r.Remote.Run(ctx, `set -e
sudo systemctl daemon-reload
sudo systemctl enable llmgw-agent >/dev/null 2>&1
sudo systemctl restart llmgw-agent
sleep 2
systemctl is-active llmgw-agent`, nil); err != nil {
return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out))
}
r.emit(StepStart, Done, "llmgw-agent is running", "")
return nil
}
// --- helpers ---
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
func orUnknown(s string) string {
if s == "" {
return "unversioned"
}
return s
}
func countLevel(fs []doctor.Finding, l doctor.Level) int {
n := 0
for _, f := range fs {
if f.Level == l {
n++
}
}
return n
}
// shellQuote quotes a string for a POSIX shell.
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}