The firewall turns off and on with the cluster's settings; llmgw wipe
firewall.enabled (default true) is a cluster setting, applied live: the leader sends a Firewall message to every agent when it registers and when the setting or firewall_networks changes, and the agent (root) takes llmgw's nft table out or installs it. A control plane writes the state to /var/lib/llmgw/firewall.json, which llmgw-firewall-sync.path applies as root with `llmgw firewall sync`; server init and server add install the units, `llmgw firewall sync --install` on one set up before. node add and server add put no firewall on a machine while the setting is off. `llmgw wipe` removes llmgw from a machine after listing what goes and asking for its name: units, engines' processes, the nft table, /etc/llmgw, /var/lib/llmgw, the agent's state, the sudo grant, the CA from the trust store, keepalived's configuration when llmgw wrote it, and the users. --models, --store (Patroni member, etcd, or the local database) and --all take more. The console has it under Do. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0730d23b77
commit
32c3942040
29 files changed
+1446
-230
No files matched your search
@@ -237,8 +237,10 @@ func (w *settingsWatch) check(ctx context.Context) {
|
||||
// again: what it reads at every use, or not at all.
|
||||
var liveSettings = []string{
|
||||
"guard",
|
||||
// Sent to the nodes and this machine's firewall unit as they change.
|
||||
"firewall", "firewall_networks",
|
||||
// Read by server init, node add and backups, not by the server.
|
||||
"firewall_networks", "backup_recipients", "backup_copy_to",
|
||||
"backup_recipients", "backup_copy_to",
|
||||
}
|
||||
|
||||
// onlyLive is whether every changed key is one of liveSettings.
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
)
|
||||
|
||||
// firewallStateFile is where a control plane writes what the cluster's
|
||||
// settings say of the firewall, for llmgw-firewall-sync to apply as root:
|
||||
// the server itself runs as llmgw and cannot change the ruleset.
|
||||
const firewallStateFile = "/var/lib/llmgw/firewall.json"
|
||||
|
||||
// firewallState is what is written there.
|
||||
type firewallState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Networks []string `json:"networks,omitempty"`
|
||||
}
|
||||
|
||||
// writeFirewallState records the settings' firewall for this machine's sync
|
||||
// unit. Without the state directory — a server not run by systemd — there
|
||||
// is no unit to read it, and nothing is written.
|
||||
func writeFirewallState(cfg config.Server) error {
|
||||
if _, err := os.Stat("/var/lib/llmgw"); err != nil {
|
||||
return nil //nolint:nilerr // not a control plane set up by init
|
||||
}
|
||||
b, _ := json.Marshal(firewallState{Enabled: cfg.Firewall.Enabled, Networks: cfg.FirewallNetworks})
|
||||
if old, err := os.ReadFile(firewallStateFile); err == nil && string(old) == string(b) {
|
||||
return nil
|
||||
}
|
||||
tmp := firewallStateFile + ".new"
|
||||
if err := os.WriteFile(tmp, b, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, firewallStateFile)
|
||||
}
|
||||
|
||||
// The units that apply it: a path unit watching the file, and the oneshot it
|
||||
// starts.
|
||||
const (
|
||||
firewallSyncPathUnit = "llmgw-firewall-sync.path"
|
||||
firewallSyncUnitFile = `[Unit]
|
||||
Description=llmgw firewall: apply the cluster's settings on this control plane
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/llmgw firewall sync
|
||||
`
|
||||
firewallSyncPathFile = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this control plane
|
||||
|
||||
[Path]
|
||||
PathChanged=` + firewallStateFile + `
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
)
|
||||
|
||||
func newFirewallCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "firewall",
|
||||
Short: "llmgw's firewall on this machine",
|
||||
Long: "Whether llmgw keeps a firewall is the cluster's setting firewall.enabled\n" +
|
||||
"(llmgw ctl config set firewall.enabled false). Nodes are told by the\n" +
|
||||
"control plane; a control plane writes it to " + firewallStateFile + ",\n" +
|
||||
"which " + firewallSyncPathUnit + " applies with 'llmgw firewall sync'.",
|
||||
}
|
||||
var install bool
|
||||
sync := &cobra.Command{
|
||||
Use: "sync",
|
||||
Short: "Apply what the cluster's settings say of the firewall, here (root)",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("the firewall is changed as root: run it with sudo")
|
||||
}
|
||||
if install {
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), firewallSyncPathUnit+" follows the cluster's settings here")
|
||||
if _, err := os.Stat(firewallStateFile); err != nil {
|
||||
return nil // the server writes it when it starts
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(firewallStateFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var st firewallState
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return fmt.Errorf("%s: %w", firewallStateFile, err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
msg, err := syncFirewall(ctx, firewall.Local{}, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), msg)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
sync.Flags().BoolVar(&install, "install", false, "first install the units that run this whenever the settings change")
|
||||
c.AddCommand(sync)
|
||||
return c
|
||||
}
|
||||
|
||||
// installFirewallSync writes the sync units on a machine and has the path
|
||||
// unit follow the state file.
|
||||
func installFirewallSync(ctx context.Context, r firewall.Runner) error {
|
||||
body := firewallSyncUnitFile + "\x00" + firewallSyncPathFile
|
||||
out, err := r.Run(ctx, `set -e
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
cat > "$t/all"
|
||||
python3 - "$t" <<'PY'
|
||||
import sys
|
||||
d = sys.argv[1]
|
||||
a, b = open(d + "/all", "rb").read().split(b"\x00")
|
||||
open(d + "/service", "wb").write(a)
|
||||
open(d + "/path", "wb").write(b)
|
||||
PY
|
||||
sudo install -m 644 "$t/service" /etc/systemd/system/llmgw-firewall-sync.service
|
||||
sudo install -m 644 "$t/path" /etc/systemd/system/`+firewallSyncPathUnit+`
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now `+firewallSyncPathUnit+` >/dev/null 2>&1`, []byte(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing %s: %v: %s", firewallSyncPathUnit, err, string(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncFirewall puts a firewall state in force on this machine.
|
||||
func syncFirewall(ctx context.Context, r firewall.Runner, st firewallState) (string, error) {
|
||||
if !st.Enabled {
|
||||
had, err := firewall.Disable(ctx, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if had {
|
||||
return "llmgw's firewall is off here", nil
|
||||
}
|
||||
return "llmgw's firewall was already off here", nil
|
||||
}
|
||||
loaded, err := firewall.Enable(ctx, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !loaded {
|
||||
return "no llmgw rules on this machine to put back; 'llmgw server init' writes them", nil
|
||||
}
|
||||
var extra []netip.Prefix
|
||||
for _, n := range st.Networks {
|
||||
if p, err := netip.ParsePrefix(n); err == nil {
|
||||
extra = append(extra, p)
|
||||
} else if a, err := netip.ParseAddr(n); err == nil {
|
||||
extra = append(extra, netip.PrefixFrom(a, a.BitLen()))
|
||||
}
|
||||
}
|
||||
if len(extra) > 0 {
|
||||
if _, err := firewall.Install(ctx, r, firewall.Spec{Allow: extra}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "llmgw's firewall is on here", nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type scriptLog struct {
|
||||
scripts []string
|
||||
reply func(script string) string
|
||||
}
|
||||
|
||||
func (s *scriptLog) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
|
||||
s.scripts = append(s.scripts, script)
|
||||
if s.reply != nil {
|
||||
return []byte(s.reply(script)), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestAControlPlaneFollowsTheFirewallSetting(t *testing.T) {
|
||||
off := &scriptLog{}
|
||||
if _, err := syncFirewall(context.Background(), off, firewallState{Enabled: false}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(off.scripts, "\n"), "nft delete table inet llmgw") {
|
||||
t.Errorf("off did not take the table out: %q", off.scripts)
|
||||
}
|
||||
|
||||
none := &scriptLog{reply: func(s string) string {
|
||||
if strings.Contains(s, "echo loaded") {
|
||||
return "none\n"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
msg, err := syncFirewall(context.Background(), none, firewallState{Enabled: true})
|
||||
if err != nil || !strings.Contains(msg, "server init") {
|
||||
t.Errorf("on, with no rules here: %q %v", msg, err)
|
||||
}
|
||||
|
||||
on := &scriptLog{reply: func(s string) string {
|
||||
if strings.Contains(s, "echo loaded") {
|
||||
return "loaded\n"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
if msg, err := syncFirewall(context.Background(), on, firewallState{Enabled: true}); err != nil || msg != "llmgw's firewall is on here" {
|
||||
t.Errorf("on: %q %v", msg, err)
|
||||
}
|
||||
}
|
||||
@@ -174,8 +174,11 @@ func (na *nodeAdd) prepare() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The node's ports — its engines', its metrics — for the cluster only.
|
||||
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && !o.NoFirewall {
|
||||
// The node's ports — its engines', its metrics — for the cluster only,
|
||||
// unless the cluster's settings turned llmgw's firewall off.
|
||||
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && !cfg.Firewall.Enabled {
|
||||
o.NoFirewall = true
|
||||
} else if err == nil && !o.NoFirewall {
|
||||
from, to := cfg.EnginePorts()
|
||||
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
|
||||
o.FirewallAllow, _ = cfg.FirewallAllow()
|
||||
|
||||
@@ -59,6 +59,8 @@ func Root() *cobra.Command {
|
||||
newSelfUpgradeCmd(),
|
||||
newReleaseCmd(),
|
||||
newJoinCmd(),
|
||||
newFirewallCmd(),
|
||||
newWipeCmd(),
|
||||
newDocsCmd(),
|
||||
)
|
||||
return root
|
||||
|
||||
@@ -223,6 +223,10 @@ func newServerCmd() *cobra.Command {
|
||||
// The control plane replaces the static endpoint list: agents
|
||||
// register, the reconciler places replicas, and Ready ones are
|
||||
// published to the router (plan §6.4).
|
||||
// The settings in force, for what reads them at each use.
|
||||
var current atomic.Pointer[config.Server]
|
||||
current.Store(&cfg)
|
||||
var agentSvc *agentsvc.Service
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
@@ -247,6 +251,14 @@ func newServerCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
defer ctl.Close()
|
||||
agentSvc = svc
|
||||
// Whether nodes keep llmgw's firewall, as the settings say now.
|
||||
svc.SetFirewall(func() *llmgwv1.Firewall {
|
||||
c := current.Load()
|
||||
from, to := c.EnginePorts()
|
||||
return &llmgwv1.Firewall{Enabled: c.Firewall.Enabled, Networks: c.FirewallNetworks,
|
||||
EnginePortsFrom: int32(from), EnginePortsTo: int32(to)} //nolint:gosec // ports
|
||||
})
|
||||
ctl.Reconciler().SetNotifier(notifier)
|
||||
admin = ctl
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
@@ -653,11 +665,24 @@ func newServerCmd() *cobra.Command {
|
||||
// Every change to the settings in force: what can change live
|
||||
// does, the rest starts this process again, which systemd brings
|
||||
// back on them.
|
||||
// And this machine's own firewall, through its sync unit.
|
||||
if err := writeFirewallState(cfg); err != nil {
|
||||
log.Warn("could not record the firewall's settings for this machine", "err", err)
|
||||
}
|
||||
watch := &settingsWatch{path: configPath, st: rt.Store(), cur: boot, log: log,
|
||||
live: func(next config.Server, changed []string) bool {
|
||||
if !onlyLive(changed) {
|
||||
return false
|
||||
}
|
||||
current.Store(&next)
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
|
||||
if err := writeFirewallState(next); err != nil {
|
||||
log.Warn("could not record the firewall's settings for this machine", "err", err)
|
||||
}
|
||||
if agentSvc != nil {
|
||||
agentSvc.BroadcastFirewall()
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "guard") }) {
|
||||
if gate == nil {
|
||||
// Built at start, and only when there are policies.
|
||||
|
||||
@@ -159,6 +159,14 @@ func (sa *serverAdd) firewall(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// What applies the cluster's firewall setting there when it changes.
|
||||
if err := installFirewallSync(ctx, sa.remote); err != nil {
|
||||
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
|
||||
}
|
||||
if !cfg.Firewall.Enabled {
|
||||
sa.kept("firewall: off, as the cluster's settings say (firewall.enabled)")
|
||||
return nil
|
||||
}
|
||||
self, err := sourceIPToward(sa.ip)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -23,6 +23,15 @@ func (in *serverInit) ensureFirewall(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// What applies the cluster's setting here when it changes: the server
|
||||
// writes it, as llmgw; this applies it, as root.
|
||||
if err := in.ensureFirewallSync(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if !cfg.Firewall.Enabled {
|
||||
in.kept("firewall: off, as the cluster's settings say (firewall.enabled)")
|
||||
return nil
|
||||
}
|
||||
ports := controlPlanePorts(cfg)
|
||||
if !in.noMirror && in.mirrorURL == "" {
|
||||
ports = append(ports, firewall.MirrorFiles)
|
||||
@@ -122,3 +131,9 @@ func loopbackListen(listen string) bool {
|
||||
ip := net.ParseIP(host)
|
||||
return host == "localhost" || (ip != nil && ip.IsLoopback())
|
||||
}
|
||||
|
||||
// ensureFirewallSync installs the units that apply the cluster's firewall
|
||||
// setting on this control plane.
|
||||
func (in *serverInit) ensureFirewallSync(ctx context.Context) error {
|
||||
return installFirewallSync(ctx, firewall.Local{})
|
||||
}
|
||||
@@ -62,13 +62,14 @@ const (
|
||||
cUpgrade
|
||||
cRestartAll
|
||||
cBackup
|
||||
cWipe
|
||||
cLeave
|
||||
cCount
|
||||
)
|
||||
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||||
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||||
"Restart all replicas", "Back up now", "Leave"}
|
||||
"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
|
||||
|
||||
// consoleAction is what is done once the screen closes: each runs its own
|
||||
// screen, and the console comes back after.
|
||||
@@ -150,7 +151,7 @@ type consoleModel struct {
|
||||
saveNote string
|
||||
|
||||
// A question open as a dialog.
|
||||
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
|
||||
dialog string // "", "target", "edit", "addname", "wipe", "saved", "note", "confirm-leave", "error"
|
||||
dialogTitle string
|
||||
dialogText string
|
||||
prompt textField
|
||||
@@ -407,7 +408,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
|
||||
func (m consoleModel) entryOK(i int) bool {
|
||||
if m.node {
|
||||
return i == cSettings || i == cManual || i == cLeave
|
||||
return i == cSettings || i == cManual || i == cWipe || i == cLeave
|
||||
}
|
||||
return i != cReplicate || !m.storeReplicated()
|
||||
}
|
||||
@@ -493,6 +494,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
return m.ask("Replicate the store", "user@address of database machine", 2)
|
||||
case cLeave:
|
||||
return m.leave()
|
||||
case cWipe:
|
||||
return m.askWipe()
|
||||
case cUpgrade, cRestartAll, cBackup:
|
||||
return m, m.doEntry()
|
||||
case cOverview:
|
||||
@@ -744,6 +747,13 @@ func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.dialogText = floatingQuestion
|
||||
m.prompt.Placeholder = "10.0.0.161/24"
|
||||
}
|
||||
case "wipe":
|
||||
if host, _ := os.Hostname(); v != host {
|
||||
m.dialogText = m.wipeQuestion() + "\n\n" + sBad.Render("that is not this machine's name; nothing is removed")
|
||||
return m, nil
|
||||
}
|
||||
m.action = consoleAction{kind: "wipe"}
|
||||
return m, tea.Quit
|
||||
case "edit":
|
||||
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
|
||||
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
|
||||
@@ -1055,7 +1065,7 @@ func (m consoleModel) View() string {
|
||||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true})
|
||||
case cSettings:
|
||||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
|
||||
case cAddNode, cLeave:
|
||||
case cAddNode, cWipe:
|
||||
if (i == cAddNode) != m.node {
|
||||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
|
||||
}
|
||||
@@ -1096,7 +1106,7 @@ func (m consoleModel) View() string {
|
||||
f.DialogTitle = m.dialogTitle
|
||||
f.Dialog = m.dialogText
|
||||
switch m.dialog {
|
||||
case "target", "edit", "addname":
|
||||
case "target", "edit", "addname", "wipe":
|
||||
q := m.dialogText
|
||||
if m.dialog == "target" && m.editKind > 1 {
|
||||
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
|
||||
@@ -1298,6 +1308,9 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
case cBackup:
|
||||
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
|
||||
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
|
||||
case cWipe:
|
||||
item = "\n Removes llmgw from this machine: its services, firewall table,\n configuration, certificates, the authority, secrets, state and users,\n so it can be set up again from nothing. Model weights, engines, the\n store's database and backups stay. Asks for the machine's name first.\n\n " +
|
||||
sDim.Render("On the command line, with --models, --store or --all for more:") + "\n sudo llmgw wipe"
|
||||
case cLeave:
|
||||
item = "\n Closes the console. The cluster keeps running."
|
||||
}
|
||||
@@ -1621,6 +1634,16 @@ func runConsole(ctx context.Context, out io.Writer, file string) error {
|
||||
// Already set up: only the store changes.
|
||||
in.noMirror, in.noNode, in.noModel = true, true, true
|
||||
err = in.runInitTUI(ctx, out)
|
||||
case "wipe":
|
||||
c := Root()
|
||||
c.SetArgs([]string{"wipe", "--yes"})
|
||||
c.SetOut(out)
|
||||
c.SetErr(out)
|
||||
if err := c.ExecuteContext(ctx); err != nil {
|
||||
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
|
||||
}
|
||||
// Nothing is left here for the console to show.
|
||||
return nil
|
||||
case "run":
|
||||
c := Root()
|
||||
c.SetArgs(act.targets)
|
||||
@@ -1666,3 +1689,31 @@ func (m consoleModel) standbyNote() string {
|
||||
}
|
||||
return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n"
|
||||
}
|
||||
|
||||
// askWipe asks for this machine's name before removing llmgw from it, saying
|
||||
// what would go.
|
||||
func (m consoleModel) askWipe() (tea.Model, tea.Cmd) {
|
||||
if os.Geteuid() != 0 {
|
||||
m.dialog, m.dialogTitle = "error", "Wipe this machine"
|
||||
m.dialogText = "Removing services, users and files under /etc needs root:\nstart the console with sudo llmgw."
|
||||
return m, nil
|
||||
}
|
||||
m.dialog, m.dialogTitle, m.dialogText = "wipe", "Wipe this machine", m.wipeQuestion()
|
||||
m.prompt = newTextField()
|
||||
m.prompt.Width = 40
|
||||
m.prompt.Focus()
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m consoleModel) wipeQuestion() string {
|
||||
host, _ := os.Hostname()
|
||||
w := &wiper{}
|
||||
var b strings.Builder
|
||||
b.WriteString("This removes, on " + host + ":\n\n")
|
||||
for _, s := range w.plan() {
|
||||
b.WriteString(" - " + s.what + "\n")
|
||||
}
|
||||
b.WriteString("\nModel weights, engines, the store's database and backups stay\n" +
|
||||
"(sudo llmgw wipe --all takes them too).\n\nType the machine's name to remove it all:")
|
||||
return b.String()
|
||||
}
|
||||
@@ -300,3 +300,27 @@ func TestTheConsoleEditsTheClustersSettingsInTheStore(t *testing.T) {
|
||||
t.Errorf("the cluster's key went into the machine's file:\n%s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleWipesOnlyOnTheMachinesName(t *testing.T) {
|
||||
m := newConsoleModel("127.0.0.1:1", filepath.Join(t.TempDir(), "server.yaml"))
|
||||
m.width, m.height, m.loaded = 120, 40, true
|
||||
m.menu = cWipe
|
||||
m.dialog, m.dialogTitle, m.dialogText = "wipe", "Wipe this machine", m.wipeQuestion()
|
||||
m.prompt = newTextField()
|
||||
m.prompt.Focus()
|
||||
if v := m.View(); !strings.Contains(v, "Type the machine's name") {
|
||||
t.Errorf("the dialog does not ask for the name:\n%s", v)
|
||||
}
|
||||
m.prompt.SetValue("not-this-one")
|
||||
mm := press(m, "enter")
|
||||
if a := mm.(consoleModel).action; a.kind != "" {
|
||||
t.Fatalf("a wrong name wiped: %+v", a)
|
||||
}
|
||||
cm := mm.(consoleModel)
|
||||
host, _ := os.Hostname()
|
||||
cm.prompt.SetValue(host)
|
||||
mm = press(cm, "enter")
|
||||
if a := mm.(consoleModel).action; a.kind != "wipe" {
|
||||
t.Errorf("the right name did not: %+v", a)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// wipeUnits are the systemd units llmgw installs, stopped first so nothing
|
||||
// writes while the rest goes.
|
||||
var wipeUnits = []string{
|
||||
"llmgw-server.service", "llmgw-agent.service", "llmgw-mirror.service",
|
||||
firewallSyncPathUnit, "llmgw-firewall-sync.service", "llmgw-firewall.service",
|
||||
"llmgw-backup.timer", "llmgw-backup.service",
|
||||
}
|
||||
|
||||
// wiper removes llmgw from a machine. root prefixes every path, for tests;
|
||||
// run runs a command.
|
||||
type wiper struct {
|
||||
root string
|
||||
run func(name string, args ...string) error
|
||||
store bool
|
||||
models bool
|
||||
all bool
|
||||
}
|
||||
|
||||
// wipeStep is one thing to remove, said before it is done.
|
||||
type wipeStep struct {
|
||||
what string
|
||||
do func() error
|
||||
}
|
||||
|
||||
func (w *wiper) path(p string) string { return filepath.Join(w.root, p) }
|
||||
|
||||
func (w *wiper) exists(p string) bool {
|
||||
_, err := os.Lstat(w.path(p))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (w *wiper) glob(pattern string) []string {
|
||||
m, _ := filepath.Glob(w.path(pattern))
|
||||
out := make([]string, len(m))
|
||||
for i, p := range m {
|
||||
out[i] = strings.TrimPrefix(p, w.root)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mentions is whether a file is there and says llmgw: a file of a name
|
||||
// another tool also uses (keepalived's, etcd's) is removed only when llmgw
|
||||
// wrote it.
|
||||
func (w *wiper) mentions(p string) bool {
|
||||
b, err := os.ReadFile(w.path(p))
|
||||
return err == nil && strings.Contains(string(b), "llmgw")
|
||||
}
|
||||
|
||||
func (w *wiper) remove(paths ...string) func() error {
|
||||
return func() error {
|
||||
var errs []error
|
||||
for _, p := range paths {
|
||||
if err := os.RemoveAll(w.path(p)); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
|
||||
// plan is every step, in order, for what is on this machine.
|
||||
func (w *wiper) plan() []wipeStep {
|
||||
var steps []wipeStep
|
||||
add := func(what string, do func() error) { steps = append(steps, wipeStep{what, do}) }
|
||||
files := func(what string, paths ...string) {
|
||||
var there []string
|
||||
for _, p := range paths {
|
||||
if w.exists(p) {
|
||||
there = append(there, p)
|
||||
}
|
||||
}
|
||||
if len(there) > 0 {
|
||||
add(what+": "+strings.Join(there, ", "), w.remove(there...))
|
||||
}
|
||||
}
|
||||
|
||||
// The services, stopped and gone.
|
||||
var units []string
|
||||
for _, u := range wipeUnits {
|
||||
if w.exists("/etc/systemd/system/" + u) {
|
||||
units = append(units, u)
|
||||
}
|
||||
}
|
||||
if len(units) > 0 {
|
||||
add("stop and remove "+strings.Join(units, ", "), func() error {
|
||||
for _, u := range units {
|
||||
_ = w.run("systemctl", "disable", "--now", u)
|
||||
if err := os.Remove(w.path("/etc/systemd/system/" + u)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.run("systemctl", "daemon-reload")
|
||||
})
|
||||
}
|
||||
// Engines run as their own user, kept alive across the agent's restarts:
|
||||
// stopped by user, or they would keep their GPUs.
|
||||
if w.userExists("llmgw-engine") {
|
||||
add("stop the engines (every process of llmgw-engine)", func() error {
|
||||
_ = w.run("pkill", "-KILL", "-u", "llmgw-engine")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
add("take llmgw's table out of the firewall (nft inet llmgw), if loaded", func() error {
|
||||
_ = w.run("nft", "delete", "table", "inet", "llmgw")
|
||||
return nil
|
||||
})
|
||||
if w.mentions("/etc/keepalived/keepalived.conf") {
|
||||
add("stop keepalived and remove llmgw's floating address (/etc/keepalived/keepalived.conf)", func() error {
|
||||
_ = w.run("systemctl", "disable", "--now", "keepalived")
|
||||
return w.remove("/etc/keepalived/keepalived.conf")()
|
||||
})
|
||||
}
|
||||
|
||||
files("configuration, certificates, the authority and secrets", "/etc/llmgw")
|
||||
files("state: the admin token, a SQLite store, what the agent kept",
|
||||
"/var/lib/llmgw", "/var/lib/llmgw-agent")
|
||||
files("the sudo grant setup left", "/etc/sudoers.d/llmgw-setup")
|
||||
if w.exists("/usr/local/share/ca-certificates/llmgw-ca.crt") {
|
||||
add("the cluster's authority from the system's trust (llmgw-ca.crt)", func() error {
|
||||
if err := w.remove("/usr/local/share/ca-certificates/llmgw-ca.crt")(); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = w.run("update-ca-certificates", "--fresh")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if w.exists("/etc/pki/ca-trust/source/anchors/llmgw-ca.pem") {
|
||||
add("the cluster's authority from the system's trust (llmgw-ca.pem)", func() error {
|
||||
if err := w.remove("/etc/pki/ca-trust/source/anchors/llmgw-ca.pem")(); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = w.run("update-ca-trust")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if w.store || w.all {
|
||||
w.planStore(add, files)
|
||||
}
|
||||
if w.models || w.all {
|
||||
files("model weights and the mirror", "/srv/models", "/var/lib/llmgw-weights", "/var/lib/llmgw-models")
|
||||
files("installed engines", "/opt/llmgw")
|
||||
}
|
||||
if w.all {
|
||||
files("backups", DefaultBackupDir)
|
||||
}
|
||||
|
||||
for _, u := range []string{"llmgw-engine", "llmgw"} {
|
||||
if w.userExists(u) {
|
||||
add("the user "+u, func() error { return w.run("userdel", u) })
|
||||
}
|
||||
}
|
||||
if w.all {
|
||||
files("the llmgw binary", "/usr/local/bin/llmgw", "/usr/local/bin/llmgw.prev")
|
||||
}
|
||||
return steps
|
||||
}
|
||||
|
||||
// planStore is the store, when it is on this machine: a member of the
|
||||
// replicated one (Patroni, etcd), or llmgw's database in a local PostgreSQL.
|
||||
func (w *wiper) planStore(add func(string, func() error), files func(string, ...string)) {
|
||||
patroni := w.mentions("/etc/patroni/config.yml")
|
||||
if patroni || w.mentions("/etc/default/etcd") {
|
||||
add("stop Patroni and etcd", func() error {
|
||||
_ = w.run("systemctl", "disable", "--now", "patroni")
|
||||
_ = w.run("systemctl", "disable", "--now", "etcd")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if patroni {
|
||||
data := append(w.glob("/var/lib/postgresql/*/llmgw"), w.glob("/var/lib/pgsql/*/llmgw")...)
|
||||
files("the store's database on this member, and Patroni's configuration",
|
||||
append([]string{"/etc/patroni", "/var/lib/postgresql/.pgpass_patroni", "/root/llmgw-pg"}, data...)...)
|
||||
} else if w.exists("/usr/bin/psql") {
|
||||
add("drop the llmgw database and role from the local PostgreSQL, if there", func() error {
|
||||
_ = w.run("runuser", "-u", "postgres", "--", "psql", "-q",
|
||||
"-c", "DROP DATABASE IF EXISTS llmgw", "-c", "DROP ROLE IF EXISTS llmgw")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if w.mentions("/etc/default/etcd") {
|
||||
files("etcd's data and configuration", "/var/lib/etcd/default", "/etc/etcd/tls", "/etc/default/etcd")
|
||||
}
|
||||
}
|
||||
|
||||
func (w *wiper) userExists(name string) bool {
|
||||
b, err := os.ReadFile(w.path("/etc/passwd"))
|
||||
return err == nil && strings.Contains("\n"+string(b), "\n"+name+":")
|
||||
}
|
||||
|
||||
func newWipeCmd() *cobra.Command {
|
||||
var w wiper
|
||||
var yes bool
|
||||
c := &cobra.Command{
|
||||
Use: "wipe",
|
||||
Short: "Remove llmgw from this machine: its services, configuration and state",
|
||||
Long: "Stops and removes llmgw's services, takes its firewall table out, and\n" +
|
||||
"deletes its configuration, certificates, the authority, secrets, state\n" +
|
||||
"and users — what server init, node add, server add and mirror init left —\n" +
|
||||
"so the machine can be set up again from nothing. It says everything it\n" +
|
||||
"will remove first, and asks for the machine's name.\n\n" +
|
||||
"Kept unless asked: model weights, the mirror and engines (--models), the\n" +
|
||||
"store's database when it is on this machine (--store), backups and the\n" +
|
||||
"binary (--all, which is all of these). Packages llmgw installed —\n" +
|
||||
"PostgreSQL, Patroni, etcd, nftables, the node exporter — stay.\n\n" +
|
||||
"A node wiped is then forgotten on the control plane with\n" +
|
||||
"'llmgw node remove <name>'.",
|
||||
Example: " sudo llmgw wipe\n" +
|
||||
" sudo llmgw wipe --models\n" +
|
||||
" sudo llmgw wipe --all --yes",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if runtime.GOOS != "linux" {
|
||||
return fmt.Errorf("wipe removes llmgw from a Linux machine; this is %s", runtime.GOOS)
|
||||
}
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("wipe removes services, users and files under /etc: run it with sudo")
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
w.run = func(name string, args ...string) error {
|
||||
return exec.CommandContext(cmd.Context(), name, args...).Run() //nolint:gosec // fixed commands
|
||||
}
|
||||
steps := w.plan()
|
||||
host, _ := os.Hostname()
|
||||
fmt.Fprintf(out, "On %s, this removes:\n", host)
|
||||
for _, s := range steps {
|
||||
fmt.Fprintf(out, " - %s\n", s.what)
|
||||
}
|
||||
if !yes {
|
||||
if !isTerminal(out) {
|
||||
return errors.New("nothing removed: give --yes to wipe without being asked")
|
||||
}
|
||||
fmt.Fprintf(out, "\nType this machine's name (%s) to remove all of it: ", host)
|
||||
answer, _ := bufio.NewReader(cmd.InOrStdin()).ReadString('\n')
|
||||
if strings.TrimSpace(answer) != host {
|
||||
return errors.New("nothing removed")
|
||||
}
|
||||
}
|
||||
var failed []string
|
||||
for _, s := range steps {
|
||||
if err := s.do(); err != nil {
|
||||
failed = append(failed, s.what+": "+err.Error())
|
||||
fmt.Fprintf(out, " !! %s: %v\n", s.what, err)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, " ok %s\n", s.what)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d step(s) failed; running wipe again finishes them", len(failed))
|
||||
}
|
||||
fmt.Fprintln(out, "\nllmgw is gone from this machine. If it was a node, forget it on the control plane: "+
|
||||
"llmgw node remove "+host)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.BoolVar(&yes, "yes", false, "do not ask")
|
||||
f.BoolVar(&w.models, "models", false, "also remove model weights, the mirror's files and installed engines")
|
||||
f.BoolVar(&w.store, "store", false, "also remove the store's database here: a Patroni member's data, etcd's, or llmgw's database in a local PostgreSQL")
|
||||
f.BoolVar(&w.all, "all", false, "everything: --models, --store, backups and the binary")
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func wipeFixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for p, body := range map[string]string{
|
||||
"/etc/llmgw/server.yaml": "listen: :8080\n",
|
||||
"/etc/llmgw/pki/ca-key.pem": "key",
|
||||
"/var/lib/llmgw/admin-token": "t",
|
||||
"/var/lib/llmgw-agent/control-planes.json": "[]",
|
||||
"/etc/systemd/system/llmgw-server.service": "[Unit]",
|
||||
"/etc/systemd/system/llmgw-agent.service": "[Unit]",
|
||||
"/etc/sudoers.d/llmgw-setup": "x",
|
||||
"/etc/keepalived/keepalived.conf": "# llmgw: the gateway's floating address\n",
|
||||
"/etc/passwd": "root:x:0:0::/root:/bin/sh\nllmgw:x:999:999::/var/lib/llmgw:/usr/sbin/nologin\nllmgw-engine:x:998:998::/:/usr/sbin/nologin\n",
|
||||
"/srv/models/qwen/v1/q.gguf": "w",
|
||||
"/opt/llmgw/engines/llamacpp/b1/llama-server": "e",
|
||||
"/etc/patroni/config.yml": "scope: llmgw\n",
|
||||
"/var/lib/postgresql/16/llmgw/PG_VERSION": "16",
|
||||
"/etc/default/etcd": "ETCD_INITIAL_CLUSTER_TOKEN=llmgw\n",
|
||||
"/var/lib/etcd/default/member/snap": "s",
|
||||
"/usr/local/bin/llmgw": "bin",
|
||||
} {
|
||||
full := filepath.Join(root, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestWipeRemovesTheConfigurationAndKeepsWhatWasNotAsked(t *testing.T) {
|
||||
root := wipeFixture(t)
|
||||
var ran []string
|
||||
w := &wiper{root: root, run: func(name string, args ...string) error {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
return nil
|
||||
}}
|
||||
for _, s := range w.plan() {
|
||||
if err := s.do(); err != nil {
|
||||
t.Fatalf("%s: %v", s.what, err)
|
||||
}
|
||||
}
|
||||
for _, gone := range []string{"/etc/llmgw", "/var/lib/llmgw", "/var/lib/llmgw-agent",
|
||||
"/etc/systemd/system/llmgw-server.service", "/etc/sudoers.d/llmgw-setup", "/etc/keepalived/keepalived.conf"} {
|
||||
if _, err := os.Stat(filepath.Join(root, gone)); err == nil {
|
||||
t.Errorf("%s is still there", gone)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{"/srv/models/qwen/v1/q.gguf", "/opt/llmgw/engines", "/etc/patroni/config.yml",
|
||||
"/var/lib/postgresql/16/llmgw", "/var/lib/etcd/default", "/usr/local/bin/llmgw"} {
|
||||
if _, err := os.Stat(filepath.Join(root, kept)); err != nil {
|
||||
t.Errorf("%s was removed without being asked", kept)
|
||||
}
|
||||
}
|
||||
all := strings.Join(ran, "\n")
|
||||
for _, want := range []string{"systemctl disable --now llmgw-server.service", "pkill -KILL -u llmgw-engine",
|
||||
"nft delete table inet llmgw", "systemctl disable --now keepalived", "userdel llmgw-engine", "userdel llmgw"} {
|
||||
if !strings.Contains(all, want) {
|
||||
t.Errorf("did not run %q:\n%s", want, all)
|
||||
}
|
||||
}
|
||||
if strings.Contains(all, "patroni") {
|
||||
t.Error("the store was touched without --store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWipeAllTakesTheStoreAndTheModelsToo(t *testing.T) {
|
||||
root := wipeFixture(t)
|
||||
w := &wiper{root: root, all: true, run: func(string, ...string) error { return nil }}
|
||||
for _, s := range w.plan() {
|
||||
if err := s.do(); err != nil {
|
||||
t.Fatalf("%s: %v", s.what, err)
|
||||
}
|
||||
}
|
||||
for _, gone := range []string{"/srv/models", "/opt/llmgw", "/etc/patroni", "/var/lib/postgresql/16/llmgw",
|
||||
"/var/lib/etcd/default", "/etc/default/etcd", "/usr/local/bin/llmgw"} {
|
||||
if _, err := os.Stat(filepath.Join(root, gone)); err == nil {
|
||||
t.Errorf("%s is still there", gone)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWipeLeavesAKeepalivedItDidNotWrite(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := filepath.Join(root, "/etc/keepalived/keepalived.conf")
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
_ = os.WriteFile(p, []byte("vrrp_instance site_vip {}\n"), 0o600)
|
||||
w := &wiper{root: root, run: func(string, ...string) error { return nil }}
|
||||
for _, s := range w.plan() {
|
||||
_ = s.do()
|
||||
}
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Error("someone else's keepalived configuration was removed")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user