network-config: policy routing for addresses on secondary subnets with a gateway

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -66,6 +66,8 @@ publish a tag that has no section here.
   download over mutual TLS, installed by a root path unit with automatic rollback;
   `rack_binaries` for other architectures, `auto_update` to opt out; versions in
   `rollcall controllers` and the web UI.
+- Addresses on secondary subnets with a gateway get their own routing table and a source rule in
+  `network-config`, so replies from a storage or tenant VLAN leave through that VLAN's gateway.
 - `rollcall version`; CD-built binary carries `git describe`.
 - Release notes from `CHANGELOG.md`.
 

cmd/rollcall/network.go

@@ -958,12 +958,17 @@ func (p *NetPlan) primaryIPs() (mgmt, ib string) {
 // (netplan syntax; cloud-init converts it for NetworkManager, networkd or ENI).
 func networkConfig(p *NetPlan) map[string]any {
 	eth, bonds, vlans := map[string]any{}, map[string]any{}, map[string]any{}
+	// Addresses on subnets other than the default route's answer through their own
+	// subnet's gateway: one routing table per such address and a rule "from it".
+	// Without this a reply to an off-subnet host leaves through the default
+	// interface with the wrong source and is dropped (reverse-path filtering).
+	nextTable := 100
 	fill := func(e map[string]any, pi PlanIface) {
 		if pi.MTU > 0 {
 			e["mtu"] = pi.MTU
 		}
 		var addrs, dns, search []string
-		var routes []map[string]any
+		var routes, policy []map[string]any
 		dhcp4, dhcp6 := false, false
 		for _, pl := range pi.Links {
 			switch pl.Mode {
@@ -984,6 +989,17 @@ func networkConfig(p *NetPlan) map[string]any {
 						to = "::/0"
 					}
 					routes = append(routes, map[string]any{"to": to, "via": pl.Gateway})
+				} else if pl.Gateway != "" {
+					to, host := "0.0.0.0/0", "/32"
+					if pl.Family == 6 {
+						to, host = "::/0", "/128"
+					}
+					t := nextTable
+					nextTable++
+					routes = append(routes,
+						map[string]any{"to": pl.Subnet, "scope": "link", "table": t},
+						map[string]any{"to": to, "via": pl.Gateway, "table": t})
+					policy = append(policy, map[string]any{"from": pl.IP + host, "table": t})
 				}
 				for _, d := range pl.DNS {
 					if !hasTag(dns, d) {
@@ -1009,6 +1025,9 @@ func networkConfig(p *NetPlan) map[string]any {
 		if len(routes) > 0 {
 			e["routes"] = routes
 		}
+		if len(policy) > 0 {
+			e["routing-policy"] = policy
+		}
 		if len(dns) > 0 || len(search) > 0 {
 			ns := map[string]any{}
 			if len(dns) > 0 {

cmd/rollcall/network_test.go

@@ -243,3 +243,35 @@ func TestDnsmasqUpstream(t *testing.T) {
 		t.Errorf("resolved stub: %q", got)
 	}
 }
+
+func TestSecondarySubnetPolicyRouting(t *testing.T) {
+	nics := []invNIC{
+		{Name: "eno1", MAC: "aa:00:00:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}},
+		{Name: "eno2", MAC: "aa:00:00:00:00:02", Link: true},
+	}
+	storage := Subnet{CIDR: "10.20.0.0/24", Gateway: "10.20.0.1", Fabric: "f0", VID: 200, Purpose: "storage"}
+	l := &NetLayout{Name: "two", Interfaces: []LayoutIface{
+		{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
+		{Type: "physical", Select: "name:eno2", Links: []LayoutLink{{Subnet: "10.20.0.0/24", Mode: "auto"}}},
+	}}
+	p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage}, nil, allocFrom("10.0.0.21", "10.20.0.5"))
+	b, _ := json.Marshal(networkConfig(p))
+	s := string(b)
+	for _, want := range []string{
+		`{"to":"0.0.0.0/0","via":"10.0.0.1"}`, // main table: the default link only
+		`{"scope":"link","table":100,"to":"10.20.0.0/24"}`,
+		`{"table":100,"to":"0.0.0.0/0","via":"10.20.0.1"}`,
+		`"routing-policy":[{"from":"10.20.0.5/32","table":100}]`,
+	} {
+		if !strings.Contains(s, want) {
+			t.Errorf("network-config lacks %s:\n%s", want, s)
+		}
+	}
+	// a secondary subnet without a gateway stays on-link: no table, no rule
+	storage.Gateway = ""
+	p = resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage}, nil, allocFrom("10.0.0.21", "10.20.0.5"))
+	b, _ = json.Marshal(networkConfig(p))
+	if strings.Contains(string(b), "routing-policy") || strings.Contains(string(b), `"table"`) {
+		t.Errorf("gateway-less subnet got policy routing:\n%s", b)
+	}
+}