Keep llmgw's internal ports to the cluster's networks with an nftables table of its own
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7e57863edd
commit
7b6276901a
17 files changed
+800
-10
No files matched your search
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -13,7 +14,9 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -26,6 +29,7 @@ func newMirrorInitCmd(dir *string) *cobra.Command {
|
||||
var (
|
||||
controlPlanes []string
|
||||
base string
|
||||
noFirewall bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "init",
|
||||
@@ -56,11 +60,19 @@ Safe to run again: it changes only what differs.`,
|
||||
if _, err := parseAllow(controlPlanes); err != nil {
|
||||
return err
|
||||
}
|
||||
return mirrorInit(cmd.Context(), cmd.OutOrStdout(), *dir, base, controlPlanes)
|
||||
if err := mirrorInit(cmd.Context(), cmd.OutOrStdout(), *dir, base, controlPlanes); err != nil {
|
||||
return err
|
||||
}
|
||||
if !noFirewall {
|
||||
mirrorFirewall(cmd.Context(), cmd.OutOrStdout(), controlPlanes)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringArrayVar(&controlPlanes, "control-plane", nil,
|
||||
"address of the control plane that manages this mirror; repeatable")
|
||||
cmd.Flags().BoolVar(&noFirewall, "no-firewall", false,
|
||||
"leave the firewall alone, instead of keeping the mirror's ports to the cluster's networks")
|
||||
cmd.Flags().StringVar(&base, "base", "",
|
||||
"the address nodes fetch from, http://host:8081 (default: this machine's first address)")
|
||||
return cmd
|
||||
@@ -123,3 +135,34 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
|
||||
fmt.Fprintf(out, "\nThe mirror is up. On the control plane:\n\n sudo llmgw server init --mirror %s\n\n", api)
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirrorFirewall keeps the mirror's ports to the networks it reaches the
|
||||
// control planes on: its files are for the cluster's nodes, its API for the
|
||||
// control planes, which the API checks again by address.
|
||||
func mirrorFirewall(ctx context.Context, out io.Writer, controlPlanes []string) {
|
||||
allowed, _ := parseAllow(controlPlanes)
|
||||
var toward []string
|
||||
var nets []netip.Prefix
|
||||
for _, p := range allowed {
|
||||
if p.IsSingleIP() {
|
||||
toward = append(toward, p.Addr().String())
|
||||
} else {
|
||||
nets = append(nets, p)
|
||||
}
|
||||
}
|
||||
found, err := firewall.Networks(ctx, pgha.Local{}, toward)
|
||||
if err == nil {
|
||||
var changed bool
|
||||
changed, err = firewall.Install(ctx, pgha.Local{}, firewall.Spec{
|
||||
Ports: []firewall.Port{firewall.MirrorFiles, firewall.MirrorAPI, firewall.NodeExporter},
|
||||
Allow: append(found, nets...),
|
||||
})
|
||||
if err == nil {
|
||||
if changed {
|
||||
fmt.Fprintln(out, " ok firewall: the mirror's ports answer the cluster's networks only")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, " !! firewall: %v; the mirror's ports answer every network\n", err)
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
@@ -122,6 +123,7 @@ engines are kept.`,
|
||||
f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
|
||||
f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
|
||||
f.BoolVar(&na.opts.NoNodeExporter, "no-node-exporter", false, "leave Prometheus' node exporter off the machine")
|
||||
f.BoolVar(&na.opts.NoFirewall, "no-firewall", false, "leave the machine's firewall alone, instead of keeping llmgw's ports to the cluster's networks")
|
||||
f.StringArrayVar(&na.opts.DisableEngines, "disable-engine", nil, "an engine this node never installs, e.g. vllm on GPUs it does not run on; repeatable")
|
||||
f.StringVar(&na.admin, "admin", "", "the control plane's admin API, to watch the node join (default: from server.yaml, else 127.0.0.1:9090)")
|
||||
f.StringVar(&na.serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, for the admin API's address")
|
||||
@@ -168,6 +170,12 @@ func (na *nodeAdd) prepare() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The node's ports — its engines', its metrics — for the cluster only.
|
||||
if cfg, err := config.Load(na.serverCfg); err == nil && !o.NoFirewall {
|
||||
from, to := cfg.EnginePorts()
|
||||
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
|
||||
o.FirewallAllow, _ = cfg.FirewallAllow()
|
||||
}
|
||||
o.Version = version.Version
|
||||
if o.Binary == "" {
|
||||
exe, err := os.Executable()
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -38,6 +39,8 @@ type serverAdd struct {
|
||||
vip string
|
||||
|
||||
remote onboard.Remote
|
||||
// noFirewall leaves the new control plane's firewall as it is.
|
||||
noFirewall bool
|
||||
// addr is the machine's address as given; ip what the others reach it at.
|
||||
addr, ip, name string
|
||||
out io.Writer
|
||||
@@ -78,6 +81,8 @@ func newServerAddCmd() *cobra.Command {
|
||||
f.StringArrayVar(&sa.hosts, "host", nil, "another name for the new control plane's certificate; repeatable")
|
||||
f.StringVar(&sa.vip, "vip", "", "a floating address for whichever control plane leads, as 10.0.0.160/24")
|
||||
f.StringVar(&sa.pkiDir, "pki", sa.pkiDir, "directory holding the certificate authority")
|
||||
f.BoolVar(&sa.noFirewall, "no-firewall", false,
|
||||
"leave its firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -121,6 +126,12 @@ func (sa *serverAdd) run(ctx context.Context) error {
|
||||
{"store", sa.storeAccess},
|
||||
{"service", sa.service},
|
||||
}
|
||||
if !sa.noFirewall {
|
||||
steps = append(steps, struct {
|
||||
name string
|
||||
fn func(context.Context) error
|
||||
}{"firewall", sa.firewall})
|
||||
}
|
||||
if sa.vip != "" {
|
||||
steps = append(steps, struct {
|
||||
name string
|
||||
@@ -136,6 +147,36 @@ func (sa *serverAdd) run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// firewall keeps the new control plane's internal ports to the networks it
|
||||
// reaches this one on. Not fatal: it serves without one.
|
||||
func (sa *serverAdd) firewall(ctx context.Context) error {
|
||||
cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
self, err := sourceIPToward(sa.ip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nets, err := firewall.Networks(ctx, sa.remote, []string{self})
|
||||
if err == nil {
|
||||
extra, _ := cfg.FirewallAllow()
|
||||
nets = append(nets, extra...)
|
||||
var changed bool
|
||||
if changed, err = firewall.Install(ctx, sa.remote,
|
||||
firewall.Spec{Ports: controlPlanePorts(cfg), Allow: nets}); err == nil {
|
||||
if changed {
|
||||
sa.done("firewall: llmgw's internal ports answer the cluster's networks only")
|
||||
} else {
|
||||
sa.kept("firewall")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sa *serverAdd) connect(ctx context.Context) error {
|
||||
out, err := sa.sh(ctx, "hostname; sudo -n true && echo sudo-ok", nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -81,6 +81,9 @@ type serverInit struct {
|
||||
// make it serve something: a mirror, this machine as a node, and a small
|
||||
// model on it.
|
||||
noMirror, noNode, noModel bool
|
||||
// noFirewall leaves this machine's firewall, and the store members',
|
||||
// as they are.
|
||||
noFirewall bool
|
||||
// mirrorURL is a mirror on another machine to use instead of running one.
|
||||
mirrorURL string
|
||||
// pgSetup prepares PostgreSQL and returns the store address; a seam for
|
||||
@@ -161,6 +164,8 @@ func newServerInitCmd() *cobra.Command {
|
||||
"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
|
||||
f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
|
||||
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
|
||||
f.BoolVar(&in.noFirewall, "no-firewall", false,
|
||||
"leave the firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
|
||||
f.BoolVar(&in.postgres, "postgres", false,
|
||||
"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
|
||||
f.StringArrayVar(&in.patroni, "patroni", nil,
|
||||
@@ -221,6 +226,12 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
return fmt.Errorf("starter model: %w", err)
|
||||
}
|
||||
}
|
||||
if hasSystemd && !in.noFirewall {
|
||||
if err := in.ensureFirewall(ctx); err != nil {
|
||||
// The cluster works without it; said, not fatal.
|
||||
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
|
||||
}
|
||||
}
|
||||
in.nextSteps()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
)
|
||||
|
||||
// ensureFirewall keeps this control plane's internal ports to the networks
|
||||
// the cluster is on: the agents' stream, the metrics, the mirror's files,
|
||||
// the engines and etcd when they are here. The client API is not among
|
||||
// them; it is meant to be reached.
|
||||
func (in *serverInit) ensureFirewall(ctx context.Context) error {
|
||||
cfg, err := config.Load(in.configPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ports := controlPlanePorts(cfg)
|
||||
if !in.noMirror && in.mirrorURL == "" {
|
||||
ports = append(ports, firewall.MirrorFiles)
|
||||
}
|
||||
if !in.noNode {
|
||||
from, to := cfg.EnginePorts()
|
||||
ports = append(ports, firewall.Engines(from, to), firewall.NodeExporter)
|
||||
}
|
||||
if _, err := os.Stat("/etc/default/etcd"); err == nil {
|
||||
// The store's third etcd vote.
|
||||
ports = append(ports, firewall.EtcdClient, firewall.EtcdPeer)
|
||||
}
|
||||
self, err := ipv4Of(in.reachableAddr())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nets, err := firewall.Networks(ctx, pgha.Local{}, []string{self})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extra, _ := cfg.FirewallAllow()
|
||||
nets = append(nets, extra...)
|
||||
changed, err := firewall.Install(ctx, pgha.Local{}, firewall.Spec{Ports: ports, Allow: nets})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shown := make([]string, len(nets))
|
||||
for i, n := range nets {
|
||||
shown[i] = n.String()
|
||||
}
|
||||
if changed {
|
||||
in.done("firewall: llmgw's internal ports answer %s only (%s)", strings.Join(shown, ", "), firewall.RulesFile)
|
||||
} else {
|
||||
in.kept("firewall: llmgw's internal ports answer %s only", strings.Join(shown, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// storeFirewall keeps each store member's database, Patroni and etcd ports to
|
||||
// the networks it reaches this control plane and the other members on. A
|
||||
// member it cannot be set on is said, and left as it was.
|
||||
func (in *serverInit) storeFirewall(ctx context.Context, members []pgha.Node, self string, extra []netip.Prefix) {
|
||||
ports := []firewall.Port{firewall.Postgres, firewall.PatroniAPI, firewall.EtcdClient, firewall.EtcdPeer,
|
||||
firewall.NodeExporter}
|
||||
for _, m := range members {
|
||||
toward := []string{self}
|
||||
for _, o := range members {
|
||||
if o.IP != m.IP {
|
||||
toward = append(toward, o.IP)
|
||||
}
|
||||
}
|
||||
nets, err := firewall.Networks(ctx, m.Remote, toward)
|
||||
if err == nil {
|
||||
nets = append(nets, extra...)
|
||||
var changed bool
|
||||
if changed, err = firewall.Install(ctx, m.Remote, firewall.Spec{Ports: ports, Allow: nets}); err == nil {
|
||||
if changed {
|
||||
in.done("firewall on %s: the store's ports answer the cluster's networks only", m.Name)
|
||||
} else {
|
||||
in.kept("firewall on %s", m.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(in.out, " !! firewall on %s: %v\n", m.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// controlPlanePorts are a control plane's internal ports, from its
|
||||
// configuration: the agents' stream, and the metrics when they listen
|
||||
// beyond this machine.
|
||||
func controlPlanePorts(cfg config.Server) []firewall.Port {
|
||||
var ports []firewall.Port
|
||||
if p := portOf(cfg.AgentListen); p > 0 {
|
||||
ports = append(ports, firewall.Single(p, "control plane: the agents' stream"))
|
||||
}
|
||||
if p := portOf(cfg.MetricsListen); p > 0 && !loopbackListen(cfg.MetricsListen) {
|
||||
ports = append(ports, firewall.Single(p, "control plane: metrics, for Prometheus"))
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
func portOf(listen string) int {
|
||||
_, port, err := net.SplitHostPort(listen)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
p, _ := strconv.Atoi(port)
|
||||
return p
|
||||
}
|
||||
|
||||
func loopbackListen(listen string) bool {
|
||||
host, _, err := net.SplitHostPort(listen)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return host == "localhost" || (ip != nil && ip.IsLoopback())
|
||||
}
|
||||
@@ -81,6 +81,10 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) {
|
||||
if err := in.writeSecret(patroniRestFile(in.etcDir), setup.RestPassword); err != nil {
|
||||
return err
|
||||
}
|
||||
if !in.noFirewall {
|
||||
extra, _ := cfg.FirewallAllow()
|
||||
in.storeFirewall(ctx, setup.DB, self, extra)
|
||||
}
|
||||
current := cfg.Auth.Store
|
||||
if current == "" {
|
||||
current = in.storeAddress()
|
||||
|
||||
Reference in new issue
Block a user