The firewall turns off and on with the cluster's settings; llmgw wipe

firewall.enabled (default true) is a cluster setting, applied live: the
leader sends a Firewall message to every agent when it registers and when
the setting or firewall_networks changes, and the agent (root) takes llmgw's
nft table out or installs it. A control plane writes the state to
/var/lib/llmgw/firewall.json, which llmgw-firewall-sync.path applies as root
with `llmgw firewall sync`; server init and server add install the units,
`llmgw firewall sync --install` on one set up before. node add and server
add put no firewall on a machine while the setting is off.

`llmgw wipe` removes llmgw from a machine after listing what goes and asking
for its name: units, engines' processes, the nft table, /etc/llmgw,
/var/lib/llmgw, the agent's state, the sudo grant, the CA from the trust
store, keepalived's configuration when llmgw wrote it, and the users.
--models, --store (Patroni member, etcd, or the local database) and --all
take more. The console has it under Do.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-06 08:00:06 +02:00
1 parent 0730d23b77
commit 32c3942040
29 files changed
+1446 -230

No files matched your search

+3 -1
View File
@@ -237,8 +237,10 @@ func (w *settingsWatch) check(ctx context.Context) {
// again: what it reads at every use, or not at all.
var liveSettings = []string{
"guard",
// Sent to the nodes and this machine's firewall unit as they change.
"firewall", "firewall_networks",
// Read by server init, node add and backups, not by the server.
"firewall_networks", "backup_recipients", "backup_copy_to",
"backup_recipients", "backup_copy_to",
}
// onlyLive is whether every changed key is one of liveSettings.
+176
View File
@@ -0,0 +1,176 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/netip"
"os"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
)
// firewallStateFile is where a control plane writes what the cluster's
// settings say of the firewall, for llmgw-firewall-sync to apply as root:
// the server itself runs as llmgw and cannot change the ruleset.
const firewallStateFile = "/var/lib/llmgw/firewall.json"
// firewallState is what is written there.
type firewallState struct {
Enabled bool `json:"enabled"`
Networks []string `json:"networks,omitempty"`
}
// writeFirewallState records the settings' firewall for this machine's sync
// unit. Without the state directory — a server not run by systemd — there
// is no unit to read it, and nothing is written.
func writeFirewallState(cfg config.Server) error {
if _, err := os.Stat("/var/lib/llmgw"); err != nil {
return nil //nolint:nilerr // not a control plane set up by init
}
b, _ := json.Marshal(firewallState{Enabled: cfg.Firewall.Enabled, Networks: cfg.FirewallNetworks})
if old, err := os.ReadFile(firewallStateFile); err == nil && string(old) == string(b) {
return nil
}
tmp := firewallStateFile + ".new"
if err := os.WriteFile(tmp, b, 0o644); err != nil {
return err
}
return os.Rename(tmp, firewallStateFile)
}
// The units that apply it: a path unit watching the file, and the oneshot it
// starts.
const (
firewallSyncPathUnit = "llmgw-firewall-sync.path"
firewallSyncUnitFile = `[Unit]
Description=llmgw firewall: apply the cluster's settings on this control plane
[Service]
Type=oneshot
ExecStart=/usr/local/bin/llmgw firewall sync
`
firewallSyncPathFile = `[Unit]
Description=llmgw firewall: follow the cluster's settings on this control plane
[Path]
PathChanged=` + firewallStateFile + `
[Install]
WantedBy=multi-user.target
`
)
func newFirewallCmd() *cobra.Command {
c := &cobra.Command{
Use: "firewall",
Short: "llmgw's firewall on this machine",
Long: "Whether llmgw keeps a firewall is the cluster's setting firewall.enabled\n" +
"(llmgw ctl config set firewall.enabled false). Nodes are told by the\n" +
"control plane; a control plane writes it to " + firewallStateFile + ",\n" +
"which " + firewallSyncPathUnit + " applies with 'llmgw firewall sync'.",
}
var install bool
sync := &cobra.Command{
Use: "sync",
Short: "Apply what the cluster's settings say of the firewall, here (root)",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if os.Geteuid() != 0 {
return errors.New("the firewall is changed as root: run it with sudo")
}
if install {
if err := installFirewallSync(cmd.Context(), firewall.Local{}); err != nil {
return err
}
fmt.Fprintln(cmd.OutOrStdout(), firewallSyncPathUnit+" follows the cluster's settings here")
if _, err := os.Stat(firewallStateFile); err != nil {
return nil // the server writes it when it starts
}
}
b, err := os.ReadFile(firewallStateFile)
if err != nil {
return err
}
var st firewallState
if err := json.Unmarshal(b, &st); err != nil {
return fmt.Errorf("%s: %w", firewallStateFile, err)
}
ctx, cancel := context.WithTimeout(cmd.Context(), 2*time.Minute)
defer cancel()
msg, err := syncFirewall(ctx, firewall.Local{}, st)
if err != nil {
return err
}
fmt.Fprintln(cmd.OutOrStdout(), msg)
return nil
},
}
sync.Flags().BoolVar(&install, "install", false, "first install the units that run this whenever the settings change")
c.AddCommand(sync)
return c
}
// installFirewallSync writes the sync units on a machine and has the path
// unit follow the state file.
func installFirewallSync(ctx context.Context, r firewall.Runner) error {
body := firewallSyncUnitFile + "\x00" + firewallSyncPathFile
out, err := r.Run(ctx, `set -e
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
cat > "$t/all"
python3 - "$t" <<'PY'
import sys
d = sys.argv[1]
a, b = open(d + "/all", "rb").read().split(b"\x00")
open(d + "/service", "wb").write(a)
open(d + "/path", "wb").write(b)
PY
sudo install -m 644 "$t/service" /etc/systemd/system/llmgw-firewall-sync.service
sudo install -m 644 "$t/path" /etc/systemd/system/`+firewallSyncPathUnit+`
sudo systemctl daemon-reload
sudo systemctl enable --now `+firewallSyncPathUnit+` >/dev/null 2>&1`, []byte(body))
if err != nil {
return fmt.Errorf("installing %s: %v: %s", firewallSyncPathUnit, err, string(out))
}
return nil
}
// syncFirewall puts a firewall state in force on this machine.
func syncFirewall(ctx context.Context, r firewall.Runner, st firewallState) (string, error) {
if !st.Enabled {
had, err := firewall.Disable(ctx, r)
if err != nil {
return "", err
}
if had {
return "llmgw's firewall is off here", nil
}
return "llmgw's firewall was already off here", nil
}
loaded, err := firewall.Enable(ctx, r)
if err != nil {
return "", err
}
if !loaded {
return "no llmgw rules on this machine to put back; 'llmgw server init' writes them", nil
}
var extra []netip.Prefix
for _, n := range st.Networks {
if p, err := netip.ParsePrefix(n); err == nil {
extra = append(extra, p)
} else if a, err := netip.ParseAddr(n); err == nil {
extra = append(extra, netip.PrefixFrom(a, a.BitLen()))
}
}
if len(extra) > 0 {
if _, err := firewall.Install(ctx, r, firewall.Spec{Allow: extra}); err != nil {
return "", err
}
}
return "llmgw's firewall is on here", nil
}
+51
View File
@@ -0,0 +1,51 @@
package cli
import (
"context"
"strings"
"testing"
)
type scriptLog struct {
scripts []string
reply func(script string) string
}
func (s *scriptLog) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
s.scripts = append(s.scripts, script)
if s.reply != nil {
return []byte(s.reply(script)), nil
}
return nil, nil
}
func TestAControlPlaneFollowsTheFirewallSetting(t *testing.T) {
off := &scriptLog{}
if _, err := syncFirewall(context.Background(), off, firewallState{Enabled: false}); err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(off.scripts, "\n"), "nft delete table inet llmgw") {
t.Errorf("off did not take the table out: %q", off.scripts)
}
none := &scriptLog{reply: func(s string) string {
if strings.Contains(s, "echo loaded") {
return "none\n"
}
return ""
}}
msg, err := syncFirewall(context.Background(), none, firewallState{Enabled: true})
if err != nil || !strings.Contains(msg, "server init") {
t.Errorf("on, with no rules here: %q %v", msg, err)
}
on := &scriptLog{reply: func(s string) string {
if strings.Contains(s, "echo loaded") {
return "loaded\n"
}
return ""
}}
if msg, err := syncFirewall(context.Background(), on, firewallState{Enabled: true}); err != nil || msg != "llmgw's firewall is on here" {
t.Errorf("on: %q %v", msg, err)
}
}
+5 -2
View File
@@ -174,8 +174,11 @@ func (na *nodeAdd) prepare() error {
}
}
}
// The node's ports — its engines', its metrics — for the cluster only.
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && !o.NoFirewall {
// The node's ports — its engines', its metrics — for the cluster only,
// unless the cluster's settings turned llmgw's firewall off.
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && !cfg.Firewall.Enabled {
o.NoFirewall = true
} else if err == nil && !o.NoFirewall {
from, to := cfg.EnginePorts()
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
+2
View File
@@ -59,6 +59,8 @@ func Root() *cobra.Command {
newSelfUpgradeCmd(),
newReleaseCmd(),
newJoinCmd(),
newFirewallCmd(),
newWipeCmd(),
newDocsCmd(),
)
return root
+25
View File
@@ -223,6 +223,10 @@ func newServerCmd() *cobra.Command {
// The control plane replaces the static endpoint list: agents
// register, the reconciler places replicas, and Ready ones are
// published to the router (plan §6.4).
// The settings in force, for what reads them at each use.
var current atomic.Pointer[config.Server]
current.Store(&cfg)
var agentSvc *agentsvc.Service
var agentHandler http.Handler
var agentPath string
var admin *control.Controller
@@ -247,6 +251,14 @@ func newServerCmd() *cobra.Command {
return err
}
defer ctl.Close()
agentSvc = svc
// Whether nodes keep llmgw's firewall, as the settings say now.
svc.SetFirewall(func() *llmgwv1.Firewall {
c := current.Load()
from, to := c.EnginePorts()
return &llmgwv1.Firewall{Enabled: c.Firewall.Enabled, Networks: c.FirewallNetworks,
EnginePortsFrom: int32(from), EnginePortsTo: int32(to)} //nolint:gosec // ports
})
ctl.Reconciler().SetNotifier(notifier)
admin = ctl
agentPath, agentHandler = svc.Handler()
@@ -653,11 +665,24 @@ func newServerCmd() *cobra.Command {
// Every change to the settings in force: what can change live
// does, the rest starts this process again, which systemd brings
// back on them.
// And this machine's own firewall, through its sync unit.
if err := writeFirewallState(cfg); err != nil {
log.Warn("could not record the firewall's settings for this machine", "err", err)
}
watch := &settingsWatch{path: configPath, st: rt.Store(), cur: boot, log: log,
live: func(next config.Server, changed []string) bool {
if !onlyLive(changed) {
return false
}
current.Store(&next)
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
if err := writeFirewallState(next); err != nil {
log.Warn("could not record the firewall's settings for this machine", "err", err)
}
if agentSvc != nil {
agentSvc.BroadcastFirewall()
}
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "guard") }) {
if gate == nil {
// Built at start, and only when there are policies.
+8
View File
@@ -159,6 +159,14 @@ func (sa *serverAdd) firewall(ctx context.Context) error {
if err != nil {
return err
}
// What applies the cluster's firewall setting there when it changes.
if err := installFirewallSync(ctx, sa.remote); err != nil {
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
}
if !cfg.Firewall.Enabled {
sa.kept("firewall: off, as the cluster's settings say (firewall.enabled)")
return nil
}
self, err := sourceIPToward(sa.ip)
if err != nil {
return err
+15
View File
@@ -23,6 +23,15 @@ func (in *serverInit) ensureFirewall(ctx context.Context) error {
if err != nil {
return err
}
// What applies the cluster's setting here when it changes: the server
// writes it, as llmgw; this applies it, as root.
if err := in.ensureFirewallSync(ctx); err != nil {
return err
}
if !cfg.Firewall.Enabled {
in.kept("firewall: off, as the cluster's settings say (firewall.enabled)")
return nil
}
ports := controlPlanePorts(cfg)
if !in.noMirror && in.mirrorURL == "" {
ports = append(ports, firewall.MirrorFiles)
@@ -122,3 +131,9 @@ func loopbackListen(listen string) bool {
ip := net.ParseIP(host)
return host == "localhost" || (ip != nil && ip.IsLoopback())
}
// ensureFirewallSync installs the units that apply the cluster's firewall
// setting on this control plane.
func (in *serverInit) ensureFirewallSync(ctx context.Context) error {
return installFirewallSync(ctx, firewall.Local{})
}
+56 -5
View File
@@ -62,13 +62,14 @@ const (
cUpgrade
cRestartAll
cBackup
cWipe
cLeave
cCount
)
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
// consoleAction is what is done once the screen closes: each runs its own
// screen, and the console comes back after.
@@ -150,7 +151,7 @@ type consoleModel struct {
saveNote string
// A question open as a dialog.
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
dialog string // "", "target", "edit", "addname", "wipe", "saved", "note", "confirm-leave", "error"
dialogTitle string
dialogText string
prompt textField
@@ -407,7 +408,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m consoleModel) entryOK(i int) bool {
if m.node {
return i == cSettings || i == cManual || i == cLeave
return i == cSettings || i == cManual || i == cWipe || i == cLeave
}
return i != cReplicate || !m.storeReplicated()
}
@@ -493,6 +494,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m.ask("Replicate the store", "user@address of database machine", 2)
case cLeave:
return m.leave()
case cWipe:
return m.askWipe()
case cUpgrade, cRestartAll, cBackup:
return m, m.doEntry()
case cOverview:
@@ -744,6 +747,13 @@ func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.dialogText = floatingQuestion
m.prompt.Placeholder = "10.0.0.161/24"
}
case "wipe":
if host, _ := os.Hostname(); v != host {
m.dialogText = m.wipeQuestion() + "\n\n" + sBad.Render("that is not this machine's name; nothing is removed")
return m, nil
}
m.action = consoleAction{kind: "wipe"}
return m, tea.Quit
case "edit":
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
@@ -1055,7 +1065,7 @@ func (m consoleModel) View() string {
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true})
case cSettings:
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
case cAddNode, cLeave:
case cAddNode, cWipe:
if (i == cAddNode) != m.node {
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
}
@@ -1096,7 +1106,7 @@ func (m consoleModel) View() string {
f.DialogTitle = m.dialogTitle
f.Dialog = m.dialogText
switch m.dialog {
case "target", "edit", "addname":
case "target", "edit", "addname", "wipe":
q := m.dialogText
if m.dialog == "target" && m.editKind > 1 {
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
@@ -1298,6 +1308,9 @@ func (m consoleModel) panel() (title, body, keys string) {
case cBackup:
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
case cWipe:
item = "\n Removes llmgw from this machine: its services, firewall table,\n configuration, certificates, the authority, secrets, state and users,\n so it can be set up again from nothing. Model weights, engines, the\n store's database and backups stay. Asks for the machine's name first.\n\n " +
sDim.Render("On the command line, with --models, --store or --all for more:") + "\n sudo llmgw wipe"
case cLeave:
item = "\n Closes the console. The cluster keeps running."
}
@@ -1621,6 +1634,16 @@ func runConsole(ctx context.Context, out io.Writer, file string) error {
// Already set up: only the store changes.
in.noMirror, in.noNode, in.noModel = true, true, true
err = in.runInitTUI(ctx, out)
case "wipe":
c := Root()
c.SetArgs([]string{"wipe", "--yes"})
c.SetOut(out)
c.SetErr(out)
if err := c.ExecuteContext(ctx); err != nil {
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
}
// Nothing is left here for the console to show.
return nil
case "run":
c := Root()
c.SetArgs(act.targets)
@@ -1666,3 +1689,31 @@ func (m consoleModel) standbyNote() string {
}
return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n"
}
// askWipe asks for this machine's name before removing llmgw from it, saying
// what would go.
func (m consoleModel) askWipe() (tea.Model, tea.Cmd) {
if os.Geteuid() != 0 {
m.dialog, m.dialogTitle = "error", "Wipe this machine"
m.dialogText = "Removing services, users and files under /etc needs root:\nstart the console with sudo llmgw."
return m, nil
}
m.dialog, m.dialogTitle, m.dialogText = "wipe", "Wipe this machine", m.wipeQuestion()
m.prompt = newTextField()
m.prompt.Width = 40
m.prompt.Focus()
return m, nil
}
func (m consoleModel) wipeQuestion() string {
host, _ := os.Hostname()
w := &wiper{}
var b strings.Builder
b.WriteString("This removes, on " + host + ":\n\n")
for _, s := range w.plan() {
b.WriteString(" - " + s.what + "\n")
}
b.WriteString("\nModel weights, engines, the store's database and backups stay\n" +
"(sudo llmgw wipe --all takes them too).\n\nType the machine's name to remove it all:")
return b.String()
}
+24
View File
@@ -300,3 +300,27 @@ func TestTheConsoleEditsTheClustersSettingsInTheStore(t *testing.T) {
t.Errorf("the cluster's key went into the machine's file:\n%s", b)
}
}
func TestTheConsoleWipesOnlyOnTheMachinesName(t *testing.T) {
m := newConsoleModel("127.0.0.1:1", filepath.Join(t.TempDir(), "server.yaml"))
m.width, m.height, m.loaded = 120, 40, true
m.menu = cWipe
m.dialog, m.dialogTitle, m.dialogText = "wipe", "Wipe this machine", m.wipeQuestion()
m.prompt = newTextField()
m.prompt.Focus()
if v := m.View(); !strings.Contains(v, "Type the machine's name") {
t.Errorf("the dialog does not ask for the name:\n%s", v)
}
m.prompt.SetValue("not-this-one")
mm := press(m, "enter")
if a := mm.(consoleModel).action; a.kind != "" {
t.Fatalf("a wrong name wiped: %+v", a)
}
cm := mm.(consoleModel)
host, _ := os.Hostname()
cm.prompt.SetValue(host)
mm = press(cm, "enter")
if a := mm.(consoleModel).action; a.kind != "wipe" {
t.Errorf("the right name did not: %+v", a)
}
}
+277
View File
@@ -0,0 +1,277 @@
package cli
import (
"bufio"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"github.com/spf13/cobra"
)
// wipeUnits are the systemd units llmgw installs, stopped first so nothing
// writes while the rest goes.
var wipeUnits = []string{
"llmgw-server.service", "llmgw-agent.service", "llmgw-mirror.service",
firewallSyncPathUnit, "llmgw-firewall-sync.service", "llmgw-firewall.service",
"llmgw-backup.timer", "llmgw-backup.service",
}
// wiper removes llmgw from a machine. root prefixes every path, for tests;
// run runs a command.
type wiper struct {
root string
run func(name string, args ...string) error
store bool
models bool
all bool
}
// wipeStep is one thing to remove, said before it is done.
type wipeStep struct {
what string
do func() error
}
func (w *wiper) path(p string) string { return filepath.Join(w.root, p) }
func (w *wiper) exists(p string) bool {
_, err := os.Lstat(w.path(p))
return err == nil
}
func (w *wiper) glob(pattern string) []string {
m, _ := filepath.Glob(w.path(pattern))
out := make([]string, len(m))
for i, p := range m {
out[i] = strings.TrimPrefix(p, w.root)
}
return out
}
// mentions is whether a file is there and says llmgw: a file of a name
// another tool also uses (keepalived's, etcd's) is removed only when llmgw
// wrote it.
func (w *wiper) mentions(p string) bool {
b, err := os.ReadFile(w.path(p))
return err == nil && strings.Contains(string(b), "llmgw")
}
func (w *wiper) remove(paths ...string) func() error {
return func() error {
var errs []error
for _, p := range paths {
if err := os.RemoveAll(w.path(p)); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
}
// plan is every step, in order, for what is on this machine.
func (w *wiper) plan() []wipeStep {
var steps []wipeStep
add := func(what string, do func() error) { steps = append(steps, wipeStep{what, do}) }
files := func(what string, paths ...string) {
var there []string
for _, p := range paths {
if w.exists(p) {
there = append(there, p)
}
}
if len(there) > 0 {
add(what+": "+strings.Join(there, ", "), w.remove(there...))
}
}
// The services, stopped and gone.
var units []string
for _, u := range wipeUnits {
if w.exists("/etc/systemd/system/" + u) {
units = append(units, u)
}
}
if len(units) > 0 {
add("stop and remove "+strings.Join(units, ", "), func() error {
for _, u := range units {
_ = w.run("systemctl", "disable", "--now", u)
if err := os.Remove(w.path("/etc/systemd/system/" + u)); err != nil && !os.IsNotExist(err) {
return err
}
}
return w.run("systemctl", "daemon-reload")
})
}
// Engines run as their own user, kept alive across the agent's restarts:
// stopped by user, or they would keep their GPUs.
if w.userExists("llmgw-engine") {
add("stop the engines (every process of llmgw-engine)", func() error {
_ = w.run("pkill", "-KILL", "-u", "llmgw-engine")
return nil
})
}
add("take llmgw's table out of the firewall (nft inet llmgw), if loaded", func() error {
_ = w.run("nft", "delete", "table", "inet", "llmgw")
return nil
})
if w.mentions("/etc/keepalived/keepalived.conf") {
add("stop keepalived and remove llmgw's floating address (/etc/keepalived/keepalived.conf)", func() error {
_ = w.run("systemctl", "disable", "--now", "keepalived")
return w.remove("/etc/keepalived/keepalived.conf")()
})
}
files("configuration, certificates, the authority and secrets", "/etc/llmgw")
files("state: the admin token, a SQLite store, what the agent kept",
"/var/lib/llmgw", "/var/lib/llmgw-agent")
files("the sudo grant setup left", "/etc/sudoers.d/llmgw-setup")
if w.exists("/usr/local/share/ca-certificates/llmgw-ca.crt") {
add("the cluster's authority from the system's trust (llmgw-ca.crt)", func() error {
if err := w.remove("/usr/local/share/ca-certificates/llmgw-ca.crt")(); err != nil {
return err
}
_ = w.run("update-ca-certificates", "--fresh")
return nil
})
}
if w.exists("/etc/pki/ca-trust/source/anchors/llmgw-ca.pem") {
add("the cluster's authority from the system's trust (llmgw-ca.pem)", func() error {
if err := w.remove("/etc/pki/ca-trust/source/anchors/llmgw-ca.pem")(); err != nil {
return err
}
_ = w.run("update-ca-trust")
return nil
})
}
if w.store || w.all {
w.planStore(add, files)
}
if w.models || w.all {
files("model weights and the mirror", "/srv/models", "/var/lib/llmgw-weights", "/var/lib/llmgw-models")
files("installed engines", "/opt/llmgw")
}
if w.all {
files("backups", DefaultBackupDir)
}
for _, u := range []string{"llmgw-engine", "llmgw"} {
if w.userExists(u) {
add("the user "+u, func() error { return w.run("userdel", u) })
}
}
if w.all {
files("the llmgw binary", "/usr/local/bin/llmgw", "/usr/local/bin/llmgw.prev")
}
return steps
}
// planStore is the store, when it is on this machine: a member of the
// replicated one (Patroni, etcd), or llmgw's database in a local PostgreSQL.
func (w *wiper) planStore(add func(string, func() error), files func(string, ...string)) {
patroni := w.mentions("/etc/patroni/config.yml")
if patroni || w.mentions("/etc/default/etcd") {
add("stop Patroni and etcd", func() error {
_ = w.run("systemctl", "disable", "--now", "patroni")
_ = w.run("systemctl", "disable", "--now", "etcd")
return nil
})
}
if patroni {
data := append(w.glob("/var/lib/postgresql/*/llmgw"), w.glob("/var/lib/pgsql/*/llmgw")...)
files("the store's database on this member, and Patroni's configuration",
append([]string{"/etc/patroni", "/var/lib/postgresql/.pgpass_patroni", "/root/llmgw-pg"}, data...)...)
} else if w.exists("/usr/bin/psql") {
add("drop the llmgw database and role from the local PostgreSQL, if there", func() error {
_ = w.run("runuser", "-u", "postgres", "--", "psql", "-q",
"-c", "DROP DATABASE IF EXISTS llmgw", "-c", "DROP ROLE IF EXISTS llmgw")
return nil
})
}
if w.mentions("/etc/default/etcd") {
files("etcd's data and configuration", "/var/lib/etcd/default", "/etc/etcd/tls", "/etc/default/etcd")
}
}
func (w *wiper) userExists(name string) bool {
b, err := os.ReadFile(w.path("/etc/passwd"))
return err == nil && strings.Contains("\n"+string(b), "\n"+name+":")
}
func newWipeCmd() *cobra.Command {
var w wiper
var yes bool
c := &cobra.Command{
Use: "wipe",
Short: "Remove llmgw from this machine: its services, configuration and state",
Long: "Stops and removes llmgw's services, takes its firewall table out, and\n" +
"deletes its configuration, certificates, the authority, secrets, state\n" +
"and users — what server init, node add, server add and mirror init left —\n" +
"so the machine can be set up again from nothing. It says everything it\n" +
"will remove first, and asks for the machine's name.\n\n" +
"Kept unless asked: model weights, the mirror and engines (--models), the\n" +
"store's database when it is on this machine (--store), backups and the\n" +
"binary (--all, which is all of these). Packages llmgw installed —\n" +
"PostgreSQL, Patroni, etcd, nftables, the node exporter — stay.\n\n" +
"A node wiped is then forgotten on the control plane with\n" +
"'llmgw node remove <name>'.",
Example: " sudo llmgw wipe\n" +
" sudo llmgw wipe --models\n" +
" sudo llmgw wipe --all --yes",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if runtime.GOOS != "linux" {
return fmt.Errorf("wipe removes llmgw from a Linux machine; this is %s", runtime.GOOS)
}
if os.Geteuid() != 0 {
return errors.New("wipe removes services, users and files under /etc: run it with sudo")
}
out := cmd.OutOrStdout()
w.run = func(name string, args ...string) error {
return exec.CommandContext(cmd.Context(), name, args...).Run() //nolint:gosec // fixed commands
}
steps := w.plan()
host, _ := os.Hostname()
fmt.Fprintf(out, "On %s, this removes:\n", host)
for _, s := range steps {
fmt.Fprintf(out, " - %s\n", s.what)
}
if !yes {
if !isTerminal(out) {
return errors.New("nothing removed: give --yes to wipe without being asked")
}
fmt.Fprintf(out, "\nType this machine's name (%s) to remove all of it: ", host)
answer, _ := bufio.NewReader(cmd.InOrStdin()).ReadString('\n')
if strings.TrimSpace(answer) != host {
return errors.New("nothing removed")
}
}
var failed []string
for _, s := range steps {
if err := s.do(); err != nil {
failed = append(failed, s.what+": "+err.Error())
fmt.Fprintf(out, " !! %s: %v\n", s.what, err)
continue
}
fmt.Fprintf(out, " ok %s\n", s.what)
}
if len(failed) > 0 {
return fmt.Errorf("%d step(s) failed; running wipe again finishes them", len(failed))
}
fmt.Fprintln(out, "\nllmgw is gone from this machine. If it was a node, forget it on the control plane: "+
"llmgw node remove "+host)
return nil
},
}
f := c.Flags()
f.BoolVar(&yes, "yes", false, "do not ask")
f.BoolVar(&w.models, "models", false, "also remove model weights, the mirror's files and installed engines")
f.BoolVar(&w.store, "store", false, "also remove the store's database here: a Patroni member's data, etcd's, or llmgw's database in a local PostgreSQL")
f.BoolVar(&w.all, "all", false, "everything: --models, --store, backups and the binary")
return c
}
+106
View File
@@ -0,0 +1,106 @@
package cli
import (
"os"
"path/filepath"
"strings"
"testing"
)
func wipeFixture(t *testing.T) string {
t.Helper()
root := t.TempDir()
for p, body := range map[string]string{
"/etc/llmgw/server.yaml": "listen: :8080\n",
"/etc/llmgw/pki/ca-key.pem": "key",
"/var/lib/llmgw/admin-token": "t",
"/var/lib/llmgw-agent/control-planes.json": "[]",
"/etc/systemd/system/llmgw-server.service": "[Unit]",
"/etc/systemd/system/llmgw-agent.service": "[Unit]",
"/etc/sudoers.d/llmgw-setup": "x",
"/etc/keepalived/keepalived.conf": "# llmgw: the gateway's floating address\n",
"/etc/passwd": "root:x:0:0::/root:/bin/sh\nllmgw:x:999:999::/var/lib/llmgw:/usr/sbin/nologin\nllmgw-engine:x:998:998::/:/usr/sbin/nologin\n",
"/srv/models/qwen/v1/q.gguf": "w",
"/opt/llmgw/engines/llamacpp/b1/llama-server": "e",
"/etc/patroni/config.yml": "scope: llmgw\n",
"/var/lib/postgresql/16/llmgw/PG_VERSION": "16",
"/etc/default/etcd": "ETCD_INITIAL_CLUSTER_TOKEN=llmgw\n",
"/var/lib/etcd/default/member/snap": "s",
"/usr/local/bin/llmgw": "bin",
} {
full := filepath.Join(root, p)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
return root
}
func TestWipeRemovesTheConfigurationAndKeepsWhatWasNotAsked(t *testing.T) {
root := wipeFixture(t)
var ran []string
w := &wiper{root: root, run: func(name string, args ...string) error {
ran = append(ran, name+" "+strings.Join(args, " "))
return nil
}}
for _, s := range w.plan() {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.what, err)
}
}
for _, gone := range []string{"/etc/llmgw", "/var/lib/llmgw", "/var/lib/llmgw-agent",
"/etc/systemd/system/llmgw-server.service", "/etc/sudoers.d/llmgw-setup", "/etc/keepalived/keepalived.conf"} {
if _, err := os.Stat(filepath.Join(root, gone)); err == nil {
t.Errorf("%s is still there", gone)
}
}
for _, kept := range []string{"/srv/models/qwen/v1/q.gguf", "/opt/llmgw/engines", "/etc/patroni/config.yml",
"/var/lib/postgresql/16/llmgw", "/var/lib/etcd/default", "/usr/local/bin/llmgw"} {
if _, err := os.Stat(filepath.Join(root, kept)); err != nil {
t.Errorf("%s was removed without being asked", kept)
}
}
all := strings.Join(ran, "\n")
for _, want := range []string{"systemctl disable --now llmgw-server.service", "pkill -KILL -u llmgw-engine",
"nft delete table inet llmgw", "systemctl disable --now keepalived", "userdel llmgw-engine", "userdel llmgw"} {
if !strings.Contains(all, want) {
t.Errorf("did not run %q:\n%s", want, all)
}
}
if strings.Contains(all, "patroni") {
t.Error("the store was touched without --store")
}
}
func TestWipeAllTakesTheStoreAndTheModelsToo(t *testing.T) {
root := wipeFixture(t)
w := &wiper{root: root, all: true, run: func(string, ...string) error { return nil }}
for _, s := range w.plan() {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.what, err)
}
}
for _, gone := range []string{"/srv/models", "/opt/llmgw", "/etc/patroni", "/var/lib/postgresql/16/llmgw",
"/var/lib/etcd/default", "/etc/default/etcd", "/usr/local/bin/llmgw"} {
if _, err := os.Stat(filepath.Join(root, gone)); err == nil {
t.Errorf("%s is still there", gone)
}
}
}
func TestWipeLeavesAKeepalivedItDidNotWrite(t *testing.T) {
root := t.TempDir()
p := filepath.Join(root, "/etc/keepalived/keepalived.conf")
_ = os.MkdirAll(filepath.Dir(p), 0o755)
_ = os.WriteFile(p, []byte("vrrp_instance site_vip {}\n"), 0o600)
w := &wiper{root: root, run: func(string, ...string) error { return nil }}
for _, s := range w.plan() {
_ = s.do()
}
if _, err := os.Stat(p); err != nil {
t.Error("someone else's keepalived configuration was removed")
}
}