From cbbfce235e3a5912e540835e2dfed5e02d38c203 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 13:27:57 +0200 Subject: [PATCH] node add lets the node through the control plane's firewall The control plane's firewall answered only the networks it was installed with, so a node on another subnet was set up, could not reach 7443 and showed as lost. node add now admits the node's network (or its address, when routed) into the local ruleset, keeping everything already there. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 ++++ docs/install.md | 9 ++++-- internal/cli/node.go | 37 +++++++++++++++++++++ internal/firewall/firewall.go | 36 +++++++++++++++++++++ internal/firewall/firewall_test.go | 52 ++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa1258..3069acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ APIs change between minors without a deprecation period. ### Fixed +- **A node on another network joins**: `node add` set up the node's firewall + but left the control plane's to the networks it was installed with, so a node + on another subnet was installed and then never heard from (lost). It now lets + the node in on the control plane it runs on — its subnet, or its address + when a router is between them — and says so; a network already let in + changes nothing. + - **An upgraded control plane gets the guardrails**: `ctl upgrade` replaces the binary but leaves server.yaml as it is, and one written before guardrails has no `guard:` section, so nothing was checked. `llmgw server init`, run again on such diff --git a/docs/install.md b/docs/install.md index 2264c2f..72c03fa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2101,9 +2101,12 @@ loaded at boot by `llmgw-firewall.service`): "The cluster's networks" are, on each machine, the network of the interface it reaches the control planes (or the other store members) on — or just -their address, when the route goes through a gateway. Anything else that -must reach those ports, a Prometheus on another subnet or nodes behind a -router, goes in `server.yaml` before the machines are set up: +their address, when the route goes through a gateway. `node add` opens +the control plane's to each node it adds when the node is on a network not +already let in: its subnet when the control plane is on it, its address alone +when a router is between them. Anything else that must reach those ports, a +Prometheus on another subnet, a second control plane's or a separate mirror's +for nodes behind a router, goes in `server.yaml`: ```yaml firewall_networks: [10.0.5.0/24, 192.168.1.20] diff --git a/internal/cli/node.go b/internal/cli/node.go index e00f9f3..8b41dfe 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "io" + "net" + "net/netip" "os" "slices" "strings" @@ -17,6 +19,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/firewall" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -266,6 +269,9 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error if len(addrs) > 1 { title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs)) } + if !o.NoFirewall { + na.letIn(ctx, o.Addr, out) + } var node string var err error if tui { @@ -297,6 +303,37 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error return nil } +// letIn opens this control plane's firewall to the node, when it has one +// and the node is on a network it does not already let in: otherwise a node +// on another subnet is set up and then never heard from. Said, not fatal. +func (na *nodeAdd) letIn(ctx context.Context, host string, out io.Writer) { + var addrs []string + if a, err := netip.ParseAddr(host); err == nil { + addrs = []string{a.String()} + } else { + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip4", host) + if err != nil { + fmt.Fprintf(out, " !! firewall: cannot resolve %s (%v); if it is on another network, "+ + "list it in firewall_networks and run 'llmgw server init'\n", host, err) + return + } + for _, ip := range ips { + addrs = append(addrs, ip.Unmap().String()) + } + } + added, err := firewall.Admit(ctx, pgha.Local{}, addrs) + switch { + case err != nil: + fmt.Fprintf(out, " !! firewall: could not let %s in: %v\n", host, err) + case len(added) > 0: + shown := make([]string, len(added)) + for i, p := range added { + shown[i] = p.String() + } + fmt.Fprintf(out, " ok firewall: this control plane now lets in %s\n", strings.Join(shown, ", ")) + } +} + func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) { p := &plainProgress{out: out} fmt.Fprintf(out, "%s\n", title) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 9d55d98..e70468d 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -317,3 +317,39 @@ func Present(ctx context.Context, r Runner) bool { _, err := r.Run(ctx, "sudo -n /usr/sbin/nft list table inet "+Table+" >/dev/null 2>&1", nil) return err == nil } + +// Admit lets addresses through a machine's ruleset when it has one: each +// one's network, as Networks finds it, unless a network already there holds +// it. A machine without llmgw's rules is left without them. What was added +// is returned, nothing when every address was already let in. +func Admit(ctx context.Context, r Runner, addrs []string) ([]netip.Prefix, error) { + old, err := r.Run(ctx, "sudo cat "+RulesFile+" 2>/dev/null || true", nil) + if err != nil { + return nil, err + } + prev := parse(string(old)) + if len(prev.Ports) == 0 { + return nil, nil + } + var missing []string + for _, s := range addrs { + a, err := netip.ParseAddr(s) + if err != nil { + continue + } + if !slices.ContainsFunc(prev.Allow, func(p netip.Prefix) bool { return p.Contains(a) }) { + missing = append(missing, s) + } + } + if len(missing) == 0 { + return nil, nil + } + nets, err := Networks(ctx, r, missing) + if err != nil { + return nil, err + } + if _, err := Install(ctx, r, Spec{Allow: nets}); err != nil { + return nil, err + } + return nets, nil +} diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 247f3b9..5470661 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -1,6 +1,7 @@ package firewall import ( + "context" "net/netip" "strings" "testing" @@ -58,3 +59,54 @@ func TestARulesetReadsBackAsItsSpec(t *testing.T) { t.Errorf("read back as\n%s\nfrom\n%s", Script(got), Script(in)) } } + +// fakeMachine answers the scripts Admit runs: the ruleset file, the route, +// and an install, which it records. +type fakeMachine struct { + rules string + route string + installed []byte +} + +func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byte, error) { + switch { + case strings.HasPrefix(script, "sudo cat "): + return []byte(f.rules), nil + case strings.HasPrefix(script, "ip -o route get"): + return []byte(f.route), nil + default: + f.installed = stdin + return []byte("changed\n"), nil + } +} + +func TestANodeOnAnotherNetworkIsLetIn(t *testing.T) { + rules := Script(Spec{Ports: []Port{AgentPort, MirrorFiles}, + Allow: []netip.Prefix{netip.MustParsePrefix("10.104.130.128/25")}}) + m := &fakeMachine{rules: rules, + route: "10.20.0.5 via 10.104.130.129 dev eth0 src 10.104.130.140 uid 0\n" + + "2: eth0 inet 10.104.130.140/25 brd 10.104.130.255 scope global eth0\n"} + + added, err := Admit(context.Background(), m, []string{"10.104.130.200"}) + if err != nil || len(added) != 0 || m.installed != nil { + t.Fatalf("an address the rules hold: added %v, err %v, installed %v", added, err, m.installed != nil) + } + + added, err = Admit(context.Background(), m, []string{"10.20.0.5", "gpu01"}) + if err != nil { + t.Fatal(err) + } + if len(added) != 1 || added[0].String() != "10.20.0.5/32" { + t.Errorf("added %v, want the routed node's address alone", added) + } + got := parse(strings.Split(string(m.installed), "\x00")[0]) + if len(got.Ports) != 2 || len(got.Allow) != 2 { + t.Errorf("installed %+v, want both ports kept and both networks", got) + } + + // No llmgw rules on the machine: none are made. + bare := &fakeMachine{} + if added, err := Admit(context.Background(), bare, []string{"10.20.0.5"}); err != nil || added != nil || bare.installed != nil { + t.Errorf("a machine without rules: added %v, err %v", added, err) + } +}