diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa1258..3069acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ APIs change between minors without a deprecation period. ### Fixed +- **A node on another network joins**: `node add` set up the node's firewall + but left the control plane's to the networks it was installed with, so a node + on another subnet was installed and then never heard from (lost). It now lets + the node in on the control plane it runs on — its subnet, or its address + when a router is between them — and says so; a network already let in + changes nothing. + - **An upgraded control plane gets the guardrails**: `ctl upgrade` replaces the binary but leaves server.yaml as it is, and one written before guardrails has no `guard:` section, so nothing was checked. `llmgw server init`, run again on such diff --git a/docs/install.md b/docs/install.md index 2264c2f..72c03fa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2101,9 +2101,12 @@ loaded at boot by `llmgw-firewall.service`): "The cluster's networks" are, on each machine, the network of the interface it reaches the control planes (or the other store members) on — or just -their address, when the route goes through a gateway. Anything else that -must reach those ports, a Prometheus on another subnet or nodes behind a -router, goes in `server.yaml` before the machines are set up: +their address, when the route goes through a gateway. `node add` opens +the control plane's to each node it adds when the node is on a network not +already let in: its subnet when the control plane is on it, its address alone +when a router is between them. Anything else that must reach those ports, a +Prometheus on another subnet, a second control plane's or a separate mirror's +for nodes behind a router, goes in `server.yaml`: ```yaml firewall_networks: [10.0.5.0/24, 192.168.1.20] diff --git a/internal/cli/node.go b/internal/cli/node.go index e00f9f3..8b41dfe 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "io" + "net" + "net/netip" "os" "slices" "strings" @@ -17,6 +19,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/firewall" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -266,6 +269,9 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error if len(addrs) > 1 { title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs)) } + if !o.NoFirewall { + na.letIn(ctx, o.Addr, out) + } var node string var err error if tui { @@ -297,6 +303,37 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error return nil } +// letIn opens this control plane's firewall to the node, when it has one +// and the node is on a network it does not already let in: otherwise a node +// on another subnet is set up and then never heard from. Said, not fatal. +func (na *nodeAdd) letIn(ctx context.Context, host string, out io.Writer) { + var addrs []string + if a, err := netip.ParseAddr(host); err == nil { + addrs = []string{a.String()} + } else { + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip4", host) + if err != nil { + fmt.Fprintf(out, " !! firewall: cannot resolve %s (%v); if it is on another network, "+ + "list it in firewall_networks and run 'llmgw server init'\n", host, err) + return + } + for _, ip := range ips { + addrs = append(addrs, ip.Unmap().String()) + } + } + added, err := firewall.Admit(ctx, pgha.Local{}, addrs) + switch { + case err != nil: + fmt.Fprintf(out, " !! firewall: could not let %s in: %v\n", host, err) + case len(added) > 0: + shown := make([]string, len(added)) + for i, p := range added { + shown[i] = p.String() + } + fmt.Fprintf(out, " ok firewall: this control plane now lets in %s\n", strings.Join(shown, ", ")) + } +} + func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) { p := &plainProgress{out: out} fmt.Fprintf(out, "%s\n", title) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 9d55d98..e70468d 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -317,3 +317,39 @@ func Present(ctx context.Context, r Runner) bool { _, err := r.Run(ctx, "sudo -n /usr/sbin/nft list table inet "+Table+" >/dev/null 2>&1", nil) return err == nil } + +// Admit lets addresses through a machine's ruleset when it has one: each +// one's network, as Networks finds it, unless a network already there holds +// it. A machine without llmgw's rules is left without them. What was added +// is returned, nothing when every address was already let in. +func Admit(ctx context.Context, r Runner, addrs []string) ([]netip.Prefix, error) { + old, err := r.Run(ctx, "sudo cat "+RulesFile+" 2>/dev/null || true", nil) + if err != nil { + return nil, err + } + prev := parse(string(old)) + if len(prev.Ports) == 0 { + return nil, nil + } + var missing []string + for _, s := range addrs { + a, err := netip.ParseAddr(s) + if err != nil { + continue + } + if !slices.ContainsFunc(prev.Allow, func(p netip.Prefix) bool { return p.Contains(a) }) { + missing = append(missing, s) + } + } + if len(missing) == 0 { + return nil, nil + } + nets, err := Networks(ctx, r, missing) + if err != nil { + return nil, err + } + if _, err := Install(ctx, r, Spec{Allow: nets}); err != nil { + return nil, err + } + return nets, nil +} diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 247f3b9..5470661 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -1,6 +1,7 @@ package firewall import ( + "context" "net/netip" "strings" "testing" @@ -58,3 +59,54 @@ func TestARulesetReadsBackAsItsSpec(t *testing.T) { t.Errorf("read back as\n%s\nfrom\n%s", Script(got), Script(in)) } } + +// fakeMachine answers the scripts Admit runs: the ruleset file, the route, +// and an install, which it records. +type fakeMachine struct { + rules string + route string + installed []byte +} + +func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byte, error) { + switch { + case strings.HasPrefix(script, "sudo cat "): + return []byte(f.rules), nil + case strings.HasPrefix(script, "ip -o route get"): + return []byte(f.route), nil + default: + f.installed = stdin + return []byte("changed\n"), nil + } +} + +func TestANodeOnAnotherNetworkIsLetIn(t *testing.T) { + rules := Script(Spec{Ports: []Port{AgentPort, MirrorFiles}, + Allow: []netip.Prefix{netip.MustParsePrefix("10.104.130.128/25")}}) + m := &fakeMachine{rules: rules, + route: "10.20.0.5 via 10.104.130.129 dev eth0 src 10.104.130.140 uid 0\n" + + "2: eth0 inet 10.104.130.140/25 brd 10.104.130.255 scope global eth0\n"} + + added, err := Admit(context.Background(), m, []string{"10.104.130.200"}) + if err != nil || len(added) != 0 || m.installed != nil { + t.Fatalf("an address the rules hold: added %v, err %v, installed %v", added, err, m.installed != nil) + } + + added, err = Admit(context.Background(), m, []string{"10.20.0.5", "gpu01"}) + if err != nil { + t.Fatal(err) + } + if len(added) != 1 || added[0].String() != "10.20.0.5/32" { + t.Errorf("added %v, want the routed node's address alone", added) + } + got := parse(strings.Split(string(m.installed), "\x00")[0]) + if len(got.Ports) != 2 || len(got.Allow) != 2 { + t.Errorf("installed %+v, want both ports kept and both networks", got) + } + + // No llmgw rules on the machine: none are made. + bare := &fakeMachine{} + if added, err := Admit(context.Background(), bare, []string{"10.20.0.5"}); err != nil || added != nil || bare.installed != nil { + t.Errorf("a machine without rules: added %v, err %v", added, err) + } +}