Keep llmgw's internal ports to the cluster's networks with an nftables table of its own

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 11:03:46 +02:00
1 parent 7e57863edd
commit 7b6276901a
17 files changed
+800 -10

No files matched your search

+44 -1
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net"
"net/netip"
"os"
"runtime"
"strings"
@@ -13,7 +14,9 @@ import (
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -26,6 +29,7 @@ func newMirrorInitCmd(dir *string) *cobra.Command {
var (
controlPlanes []string
base string
noFirewall bool
)
cmd := &cobra.Command{
Use: "init",
@@ -56,11 +60,19 @@ Safe to run again: it changes only what differs.`,
if _, err := parseAllow(controlPlanes); err != nil {
return err
}
return mirrorInit(cmd.Context(), cmd.OutOrStdout(), *dir, base, controlPlanes)
if err := mirrorInit(cmd.Context(), cmd.OutOrStdout(), *dir, base, controlPlanes); err != nil {
return err
}
if !noFirewall {
mirrorFirewall(cmd.Context(), cmd.OutOrStdout(), controlPlanes)
}
return nil
},
}
cmd.Flags().StringArrayVar(&controlPlanes, "control-plane", nil,
"address of the control plane that manages this mirror; repeatable")
cmd.Flags().BoolVar(&noFirewall, "no-firewall", false,
"leave the firewall alone, instead of keeping the mirror's ports to the cluster's networks")
cmd.Flags().StringVar(&base, "base", "",
"the address nodes fetch from, http://host:8081 (default: this machine's first address)")
return cmd
@@ -123,3 +135,34 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
fmt.Fprintf(out, "\nThe mirror is up. On the control plane:\n\n sudo llmgw server init --mirror %s\n\n", api)
return nil
}
// mirrorFirewall keeps the mirror's ports to the networks it reaches the
// control planes on: its files are for the cluster's nodes, its API for the
// control planes, which the API checks again by address.
func mirrorFirewall(ctx context.Context, out io.Writer, controlPlanes []string) {
allowed, _ := parseAllow(controlPlanes)
var toward []string
var nets []netip.Prefix
for _, p := range allowed {
if p.IsSingleIP() {
toward = append(toward, p.Addr().String())
} else {
nets = append(nets, p)
}
}
found, err := firewall.Networks(ctx, pgha.Local{}, toward)
if err == nil {
var changed bool
changed, err = firewall.Install(ctx, pgha.Local{}, firewall.Spec{
Ports: []firewall.Port{firewall.MirrorFiles, firewall.MirrorAPI, firewall.NodeExporter},
Allow: append(found, nets...),
})
if err == nil {
if changed {
fmt.Fprintln(out, " ok firewall: the mirror's ports answer the cluster's networks only")
}
return
}
}
fmt.Fprintf(out, " !! firewall: %v; the mirror's ports answer every network\n", err)
}
+8
View File
@@ -14,6 +14,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
@@ -122,6 +123,7 @@ engines are kept.`,
f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
f.BoolVar(&na.opts.NoNodeExporter, "no-node-exporter", false, "leave Prometheus' node exporter off the machine")
f.BoolVar(&na.opts.NoFirewall, "no-firewall", false, "leave the machine's firewall alone, instead of keeping llmgw's ports to the cluster's networks")
f.StringArrayVar(&na.opts.DisableEngines, "disable-engine", nil, "an engine this node never installs, e.g. vllm on GPUs it does not run on; repeatable")
f.StringVar(&na.admin, "admin", "", "the control plane's admin API, to watch the node join (default: from server.yaml, else 127.0.0.1:9090)")
f.StringVar(&na.serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, for the admin API's address")
@@ -168,6 +170,12 @@ func (na *nodeAdd) prepare() error {
}
}
}
// The node's ports — its engines', its metrics — for the cluster only.
if cfg, err := config.Load(na.serverCfg); err == nil && !o.NoFirewall {
from, to := cfg.EnginePorts()
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
}
o.Version = version.Version
if o.Binary == "" {
exe, err := os.Executable()
+41
View File
@@ -20,6 +20,7 @@ import (
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -38,6 +39,8 @@ type serverAdd struct {
vip string
remote onboard.Remote
// noFirewall leaves the new control plane's firewall as it is.
noFirewall bool
// addr is the machine's address as given; ip what the others reach it at.
addr, ip, name string
out io.Writer
@@ -78,6 +81,8 @@ func newServerAddCmd() *cobra.Command {
f.StringArrayVar(&sa.hosts, "host", nil, "another name for the new control plane's certificate; repeatable")
f.StringVar(&sa.vip, "vip", "", "a floating address for whichever control plane leads, as 10.0.0.160/24")
f.StringVar(&sa.pkiDir, "pki", sa.pkiDir, "directory holding the certificate authority")
f.BoolVar(&sa.noFirewall, "no-firewall", false,
"leave its firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
return cmd
}
@@ -121,6 +126,12 @@ func (sa *serverAdd) run(ctx context.Context) error {
{"store", sa.storeAccess},
{"service", sa.service},
}
if !sa.noFirewall {
steps = append(steps, struct {
name string
fn func(context.Context) error
}{"firewall", sa.firewall})
}
if sa.vip != "" {
steps = append(steps, struct {
name string
@@ -136,6 +147,36 @@ func (sa *serverAdd) run(ctx context.Context) error {
return nil
}
// firewall keeps the new control plane's internal ports to the networks it
// reaches this one on. Not fatal: it serves without one.
func (sa *serverAdd) firewall(ctx context.Context) error {
cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml"))
if err != nil {
return err
}
self, err := sourceIPToward(sa.ip)
if err != nil {
return err
}
nets, err := firewall.Networks(ctx, sa.remote, []string{self})
if err == nil {
extra, _ := cfg.FirewallAllow()
nets = append(nets, extra...)
var changed bool
if changed, err = firewall.Install(ctx, sa.remote,
firewall.Spec{Ports: controlPlanePorts(cfg), Allow: nets}); err == nil {
if changed {
sa.done("firewall: llmgw's internal ports answer the cluster's networks only")
} else {
sa.kept("firewall")
}
return nil
}
}
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
return nil
}
func (sa *serverAdd) connect(ctx context.Context) error {
out, err := sa.sh(ctx, "hostname; sudo -n true && echo sudo-ok", nil)
if err != nil {
+11
View File
@@ -81,6 +81,9 @@ type serverInit struct {
// make it serve something: a mirror, this machine as a node, and a small
// model on it.
noMirror, noNode, noModel bool
// noFirewall leaves this machine's firewall, and the store members',
// as they are.
noFirewall bool
// mirrorURL is a mirror on another machine to use instead of running one.
mirrorURL string
// pgSetup prepares PostgreSQL and returns the store address; a seam for
@@ -161,6 +164,8 @@ func newServerInitCmd() *cobra.Command {
"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
f.BoolVar(&in.noFirewall, "no-firewall", false,
"leave the firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
f.BoolVar(&in.postgres, "postgres", false,
"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
f.StringArrayVar(&in.patroni, "patroni", nil,
@@ -221,6 +226,12 @@ func (in *serverInit) run(ctx context.Context) error {
return fmt.Errorf("starter model: %w", err)
}
}
if hasSystemd && !in.noFirewall {
if err := in.ensureFirewall(ctx); err != nil {
// The cluster works without it; said, not fatal.
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
}
}
in.nextSteps()
return nil
}
+124
View File
@@ -0,0 +1,124 @@
package cli
import (
"context"
"fmt"
"net"
"net/netip"
"os"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
)
// ensureFirewall keeps this control plane's internal ports to the networks
// the cluster is on: the agents' stream, the metrics, the mirror's files,
// the engines and etcd when they are here. The client API is not among
// them; it is meant to be reached.
func (in *serverInit) ensureFirewall(ctx context.Context) error {
cfg, err := config.Load(in.configPath())
if err != nil {
return err
}
ports := controlPlanePorts(cfg)
if !in.noMirror && in.mirrorURL == "" {
ports = append(ports, firewall.MirrorFiles)
}
if !in.noNode {
from, to := cfg.EnginePorts()
ports = append(ports, firewall.Engines(from, to), firewall.NodeExporter)
}
if _, err := os.Stat("/etc/default/etcd"); err == nil {
// The store's third etcd vote.
ports = append(ports, firewall.EtcdClient, firewall.EtcdPeer)
}
self, err := ipv4Of(in.reachableAddr())
if err != nil {
return err
}
nets, err := firewall.Networks(ctx, pgha.Local{}, []string{self})
if err != nil {
return err
}
extra, _ := cfg.FirewallAllow()
nets = append(nets, extra...)
changed, err := firewall.Install(ctx, pgha.Local{}, firewall.Spec{Ports: ports, Allow: nets})
if err != nil {
return err
}
shown := make([]string, len(nets))
for i, n := range nets {
shown[i] = n.String()
}
if changed {
in.done("firewall: llmgw's internal ports answer %s only (%s)", strings.Join(shown, ", "), firewall.RulesFile)
} else {
in.kept("firewall: llmgw's internal ports answer %s only", strings.Join(shown, ", "))
}
return nil
}
// storeFirewall keeps each store member's database, Patroni and etcd ports to
// the networks it reaches this control plane and the other members on. A
// member it cannot be set on is said, and left as it was.
func (in *serverInit) storeFirewall(ctx context.Context, members []pgha.Node, self string, extra []netip.Prefix) {
ports := []firewall.Port{firewall.Postgres, firewall.PatroniAPI, firewall.EtcdClient, firewall.EtcdPeer,
firewall.NodeExporter}
for _, m := range members {
toward := []string{self}
for _, o := range members {
if o.IP != m.IP {
toward = append(toward, o.IP)
}
}
nets, err := firewall.Networks(ctx, m.Remote, toward)
if err == nil {
nets = append(nets, extra...)
var changed bool
if changed, err = firewall.Install(ctx, m.Remote, firewall.Spec{Ports: ports, Allow: nets}); err == nil {
if changed {
in.done("firewall on %s: the store's ports answer the cluster's networks only", m.Name)
} else {
in.kept("firewall on %s", m.Name)
}
continue
}
}
fmt.Fprintf(in.out, " !! firewall on %s: %v\n", m.Name, err)
}
}
// controlPlanePorts are a control plane's internal ports, from its
// configuration: the agents' stream, and the metrics when they listen
// beyond this machine.
func controlPlanePorts(cfg config.Server) []firewall.Port {
var ports []firewall.Port
if p := portOf(cfg.AgentListen); p > 0 {
ports = append(ports, firewall.Single(p, "control plane: the agents' stream"))
}
if p := portOf(cfg.MetricsListen); p > 0 && !loopbackListen(cfg.MetricsListen) {
ports = append(ports, firewall.Single(p, "control plane: metrics, for Prometheus"))
}
return ports
}
func portOf(listen string) int {
_, port, err := net.SplitHostPort(listen)
if err != nil {
return 0
}
p, _ := strconv.Atoi(port)
return p
}
func loopbackListen(listen string) bool {
host, _, err := net.SplitHostPort(listen)
if err != nil {
return false
}
ip := net.ParseIP(host)
return host == "localhost" || (ip != nil && ip.IsLoopback())
}
+4
View File
@@ -81,6 +81,10 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) {
if err := in.writeSecret(patroniRestFile(in.etcDir), setup.RestPassword); err != nil {
return err
}
if !in.noFirewall {
extra, _ := cfg.FirewallAllow()
in.storeFirewall(ctx, setup.DB, self, extra)
}
current := cfg.Auth.Store
if current == "" {
current = in.storeAddress()