node add lets the node through the control plane's firewall

The control plane's firewall answered only the networks it was installed
with, so a node on another subnet was set up, could not reach 7443 and
showed as lost. node add now admits the node's network (or its address,
when routed) into the local ruleset, keeping everything already there.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 13:27:57 +02:00
1 parent 4ff62c5dd2
commit cbbfce235e
5 files changed
+138 -3

No files matched your search

+7
View File
@@ -25,6 +25,13 @@ APIs change between minors without a deprecation period.
### Fixed
- **A node on another network joins**: `node add` set up the node's firewall
but left the control plane's to the networks it was installed with, so a node
on another subnet was installed and then never heard from (lost). It now lets
the node in on the control plane it runs on — its subnet, or its address
when a router is between them — and says so; a network already let in
changes nothing.
- **An upgraded control plane gets the guardrails**: `ctl upgrade` replaces the
binary but leaves server.yaml as it is, and one written before guardrails has no
`guard:` section, so nothing was checked. `llmgw server init`, run again on such
+6 -3
View File
@@ -2101,9 +2101,12 @@ loaded at boot by `llmgw-firewall.service`):
"The cluster's networks" are, on each machine, the network of the interface
it reaches the control planes (or the other store members) on — or just
their address, when the route goes through a gateway. Anything else that
must reach those ports, a Prometheus on another subnet or nodes behind a
router, goes in `server.yaml` before the machines are set up:
their address, when the route goes through a gateway. `node add` opens
the control plane's to each node it adds when the node is on a network not
already let in: its subnet when the control plane is on it, its address alone
when a router is between them. Anything else that must reach those ports, a
Prometheus on another subnet, a second control plane's or a separate mirror's
for nodes behind a router, goes in `server.yaml`:
```yaml
firewall_networks: [10.0.5.0/24, 192.168.1.20]
+37
View File
@@ -5,6 +5,8 @@ import (
"errors"
"fmt"
"io"
"net"
"net/netip"
"os"
"slices"
"strings"
@@ -17,6 +19,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
@@ -266,6 +269,9 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error
if len(addrs) > 1 {
title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs))
}
if !o.NoFirewall {
na.letIn(ctx, o.Addr, out)
}
var node string
var err error
if tui {
@@ -297,6 +303,37 @@ func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error
return nil
}
// letIn opens this control plane's firewall to the node, when it has one
// and the node is on a network it does not already let in: otherwise a node
// on another subnet is set up and then never heard from. Said, not fatal.
func (na *nodeAdd) letIn(ctx context.Context, host string, out io.Writer) {
var addrs []string
if a, err := netip.ParseAddr(host); err == nil {
addrs = []string{a.String()}
} else {
ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip4", host)
if err != nil {
fmt.Fprintf(out, " !! firewall: cannot resolve %s (%v); if it is on another network, "+
"list it in firewall_networks and run 'llmgw server init'\n", host, err)
return
}
for _, ip := range ips {
addrs = append(addrs, ip.Unmap().String())
}
}
added, err := firewall.Admit(ctx, pgha.Local{}, addrs)
switch {
case err != nil:
fmt.Fprintf(out, " !! firewall: could not let %s in: %v\n", host, err)
case len(added) > 0:
shown := make([]string, len(added))
for i, p := range added {
shown[i] = p.String()
}
fmt.Fprintf(out, " ok firewall: this control plane now lets in %s\n", strings.Join(shown, ", "))
}
}
func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) {
p := &plainProgress{out: out}
fmt.Fprintf(out, "%s\n", title)
+36
View File
@@ -317,3 +317,39 @@ func Present(ctx context.Context, r Runner) bool {
_, err := r.Run(ctx, "sudo -n /usr/sbin/nft list table inet "+Table+" >/dev/null 2>&1", nil)
return err == nil
}
// Admit lets addresses through a machine's ruleset when it has one: each
// one's network, as Networks finds it, unless a network already there holds
// it. A machine without llmgw's rules is left without them. What was added
// is returned, nothing when every address was already let in.
func Admit(ctx context.Context, r Runner, addrs []string) ([]netip.Prefix, error) {
old, err := r.Run(ctx, "sudo cat "+RulesFile+" 2>/dev/null || true", nil)
if err != nil {
return nil, err
}
prev := parse(string(old))
if len(prev.Ports) == 0 {
return nil, nil
}
var missing []string
for _, s := range addrs {
a, err := netip.ParseAddr(s)
if err != nil {
continue
}
if !slices.ContainsFunc(prev.Allow, func(p netip.Prefix) bool { return p.Contains(a) }) {
missing = append(missing, s)
}
}
if len(missing) == 0 {
return nil, nil
}
nets, err := Networks(ctx, r, missing)
if err != nil {
return nil, err
}
if _, err := Install(ctx, r, Spec{Allow: nets}); err != nil {
return nil, err
}
return nets, nil
}
+52
View File
@@ -1,6 +1,7 @@
package firewall
import (
"context"
"net/netip"
"strings"
"testing"
@@ -58,3 +59,54 @@ func TestARulesetReadsBackAsItsSpec(t *testing.T) {
t.Errorf("read back as\n%s\nfrom\n%s", Script(got), Script(in))
}
}
// fakeMachine answers the scripts Admit runs: the ruleset file, the route,
// and an install, which it records.
type fakeMachine struct {
rules string
route string
installed []byte
}
func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byte, error) {
switch {
case strings.HasPrefix(script, "sudo cat "):
return []byte(f.rules), nil
case strings.HasPrefix(script, "ip -o route get"):
return []byte(f.route), nil
default:
f.installed = stdin
return []byte("changed\n"), nil
}
}
func TestANodeOnAnotherNetworkIsLetIn(t *testing.T) {
rules := Script(Spec{Ports: []Port{AgentPort, MirrorFiles},
Allow: []netip.Prefix{netip.MustParsePrefix("10.104.130.128/25")}})
m := &fakeMachine{rules: rules,
route: "10.20.0.5 via 10.104.130.129 dev eth0 src 10.104.130.140 uid 0\n" +
"2: eth0 inet 10.104.130.140/25 brd 10.104.130.255 scope global eth0\n"}
added, err := Admit(context.Background(), m, []string{"10.104.130.200"})
if err != nil || len(added) != 0 || m.installed != nil {
t.Fatalf("an address the rules hold: added %v, err %v, installed %v", added, err, m.installed != nil)
}
added, err = Admit(context.Background(), m, []string{"10.20.0.5", "gpu01"})
if err != nil {
t.Fatal(err)
}
if len(added) != 1 || added[0].String() != "10.20.0.5/32" {
t.Errorf("added %v, want the routed node's address alone", added)
}
got := parse(strings.Split(string(m.installed), "\x00")[0])
if len(got.Ports) != 2 || len(got.Allow) != 2 {
t.Errorf("installed %+v, want both ports kept and both networks", got)
}
// No llmgw rules on the machine: none are made.
bare := &fakeMachine{}
if added, err := Admit(context.Background(), bare, []string{"10.20.0.5"}); err != nil || added != nil || bare.installed != nil {
t.Errorf("a machine without rules: added %v, err %v", added, err)
}
}