The firewall setting reaches the mirror machine and the store's members
The leading control plane tells the mirror over its API (PUT /v1/firewall, kept as .firewall.json beside the models and applied by the same path unit control planes use), and keeps the state in the store's cluster_state for members that run no llmgw, which a root timer reads from the member's own copy. The mirror also lets in the network of every node and control plane the leader knows. Tried on the PoC: off, on and a network added, on every kind of machine. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5be3da505b
commit
8df78a4d41
20 files changed
+772
-34
No files matched your search
+49
-21
@@ -7,12 +7,14 @@ import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// firewallStateFile is where a control plane writes what the cluster's
|
||||
@@ -20,11 +22,9 @@ import (
|
||||
// the server itself runs as llmgw and cannot change the ruleset.
|
||||
const firewallStateFile = "/var/lib/llmgw/firewall.json"
|
||||
|
||||
// firewallState is what is written there.
|
||||
type firewallState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Networks []string `json:"networks,omitempty"`
|
||||
}
|
||||
// firewallState is what is written there: the same as the mirror is told
|
||||
// and keeps beside its models.
|
||||
type firewallState = mirror.FirewallState
|
||||
|
||||
// writeFirewallState records the settings' firewall for this machine's sync
|
||||
// unit. Without the state directory — a server not run by systemd — there
|
||||
@@ -46,25 +46,34 @@ func writeFirewallState(cfg config.Server) error {
|
||||
|
||||
// The units that apply it: a path unit watching the file, and the oneshot it
|
||||
// starts.
|
||||
const (
|
||||
firewallSyncPathUnit = "llmgw-firewall-sync.path"
|
||||
firewallSyncUnitFile = `[Unit]
|
||||
Description=llmgw firewall: apply the cluster's settings on this control plane
|
||||
const firewallSyncPathUnit = "llmgw-firewall-sync.path"
|
||||
|
||||
// firewallSyncUnits are the two units for a state file: on a control plane
|
||||
// the one its server writes, on a mirror the one the control plane has the
|
||||
// mirror write.
|
||||
func firewallSyncUnits(state string) (service, pathUnit string) {
|
||||
arg := ""
|
||||
if state != firewallStateFile {
|
||||
arg = " --state " + state
|
||||
}
|
||||
service = `[Unit]
|
||||
Description=llmgw firewall: apply the cluster's settings on this machine
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/llmgw firewall sync
|
||||
ExecStart=/usr/local/bin/llmgw firewall sync` + arg + `
|
||||
`
|
||||
firewallSyncPathFile = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this control plane
|
||||
pathUnit = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this machine
|
||||
|
||||
[Path]
|
||||
PathChanged=` + firewallStateFile + `
|
||||
PathChanged=` + state + `
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
)
|
||||
return service, pathUnit
|
||||
}
|
||||
|
||||
func newFirewallCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
@@ -76,6 +85,7 @@ func newFirewallCmd() *cobra.Command {
|
||||
"which " + firewallSyncPathUnit + " applies with 'llmgw firewall sync'.",
|
||||
}
|
||||
var install bool
|
||||
state := firewallStateFile
|
||||
sync := &cobra.Command{
|
||||
Use: "sync",
|
||||
Short: "Apply what the cluster's settings say of the firewall, here (root)",
|
||||
@@ -85,21 +95,21 @@ func newFirewallCmd() *cobra.Command {
|
||||
return errors.New("the firewall is changed as root: run it with sudo")
|
||||
}
|
||||
if install {
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}); err != nil {
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}, state); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), firewallSyncPathUnit+" follows the cluster's settings here")
|
||||
if _, err := os.Stat(firewallStateFile); err != nil {
|
||||
return nil // the server writes it when it starts
|
||||
if _, err := os.Stat(state); err != nil {
|
||||
return nil // written when the settings are first told
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(firewallStateFile)
|
||||
b, err := os.ReadFile(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var st firewallState
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return fmt.Errorf("%s: %w", firewallStateFile, err)
|
||||
return fmt.Errorf("%s: %w", state, err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
@@ -112,14 +122,17 @@ func newFirewallCmd() *cobra.Command {
|
||||
},
|
||||
}
|
||||
sync.Flags().BoolVar(&install, "install", false, "first install the units that run this whenever the settings change")
|
||||
sync.Flags().StringVar(&state, "state", firewallStateFile,
|
||||
"the file the settings are written to: a control plane's, or "+mirror.FirewallFile+" in a mirror's directory")
|
||||
c.AddCommand(sync)
|
||||
return c
|
||||
}
|
||||
|
||||
// installFirewallSync writes the sync units on a machine and has the path
|
||||
// unit follow the state file.
|
||||
func installFirewallSync(ctx context.Context, r firewall.Runner) error {
|
||||
body := firewallSyncUnitFile + "\x00" + firewallSyncPathFile
|
||||
func installFirewallSync(ctx context.Context, r firewall.Runner, state string) error {
|
||||
service, pathUnit := firewallSyncUnits(state)
|
||||
body := service + "\x00" + pathUnit
|
||||
out, err := r.Run(ctx, `set -e
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
cat > "$t/all"
|
||||
@@ -172,5 +185,20 @@ func syncFirewall(ctx context.Context, r firewall.Runner, st firewallState) (str
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
// The cluster's machines: a node on a network this one has not let in
|
||||
// yet, which the mirror's files are for.
|
||||
if len(st.Toward) > 0 {
|
||||
added, err := firewall.Admit(ctx, r, st.Toward)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(added) > 0 {
|
||||
shown := make([]string, len(added))
|
||||
for i, p := range added {
|
||||
shown[i] = p.String()
|
||||
}
|
||||
return "llmgw's firewall is on here, and now lets in " + strings.Join(shown, ", "), nil
|
||||
}
|
||||
}
|
||||
return "llmgw's firewall is on here", nil
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
)
|
||||
|
||||
type scriptLog struct {
|
||||
@@ -49,3 +52,32 @@ func TestAControlPlaneFollowsTheFirewallSetting(t *testing.T) {
|
||||
t.Errorf("on: %q %v", msg, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineLetsInTheClusterMachinesItIsTold(t *testing.T) {
|
||||
// The rules hold 10.0.0.0/24; a node on 192.168.7.0/24 is let in, as
|
||||
// the route to it says, and one already in is not asked about.
|
||||
r := &scriptLog{reply: func(s string) string {
|
||||
switch {
|
||||
case strings.Contains(s, "echo loaded"):
|
||||
return "loaded\n"
|
||||
case strings.Contains(s, "cat /etc/llmgw/firewall.nft"):
|
||||
return firewallRulesFor("10.0.0.0/24")
|
||||
case strings.Contains(s, "ip -o route get 192.168.7.20"):
|
||||
return "192.168.7.20 dev eth1 src 192.168.7.1 uid 0\n2: eth1 inet 192.168.7.1/24 brd 192.168.7.255 scope global eth1\n"
|
||||
case strings.Contains(s, "nft -f"):
|
||||
return "changed\n"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
msg, err := syncFirewall(context.Background(), r, firewallState{Enabled: true, Toward: []string{"10.0.0.147", "192.168.7.20"}})
|
||||
if err != nil || !strings.Contains(msg, "192.168.7.0/24") {
|
||||
t.Fatalf("%q %v", msg, err)
|
||||
}
|
||||
if strings.Contains(strings.Join(r.scripts, "\n"), "route get 10.0.0.147") {
|
||||
t.Error("asked the route to a machine already let in")
|
||||
}
|
||||
}
|
||||
|
||||
func firewallRulesFor(nets string) string {
|
||||
return firewall.Script(firewall.Spec{Ports: []firewall.Port{firewall.MirrorFiles}, Allow: []netip.Prefix{netip.MustParsePrefix(nets)}})
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// firewallStateKey is where, in the store, the firewall's state is kept
|
||||
// for the store's members to read.
|
||||
const firewallStateKey = "firewall"
|
||||
|
||||
// firewallPublisher tells the machines that cannot ask a control plane
|
||||
// what the cluster's settings say of their firewall: the mirror, over its
|
||||
// API, and the store's members, through the store they hold a copy of.
|
||||
// Nodes are told over their agents' streams and control planes by their
|
||||
// own servers. On the leader only, at start, at every change to the
|
||||
// firewall's settings, and every few minutes for nodes come and gone.
|
||||
type firewallPublisher struct {
|
||||
settings func() config.Server
|
||||
st *store.DB
|
||||
nodes func() []string
|
||||
leader func() bool
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
mirror *mirror.Client
|
||||
kick chan struct{}
|
||||
}
|
||||
|
||||
func newFirewallPublisher(settings func() config.Server, st *store.DB, nodes func() []string,
|
||||
leader func() bool, log *slog.Logger) *firewallPublisher {
|
||||
return &firewallPublisher{settings: settings, st: st, nodes: nodes, leader: leader, log: log,
|
||||
kick: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// setMirror has the mirror told too.
|
||||
func (p *firewallPublisher) setMirror(m *mirror.Client) {
|
||||
p.mu.Lock()
|
||||
p.mirror = m
|
||||
p.mu.Unlock()
|
||||
p.Kick()
|
||||
}
|
||||
|
||||
// Kick has the state worked out and told again now.
|
||||
func (p *firewallPublisher) Kick() {
|
||||
select {
|
||||
case p.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (p *firewallPublisher) run(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
p.once(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
case <-p.kick:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// state is what the settings say now, with the cluster's machines.
|
||||
func (p *firewallPublisher) state(ctx context.Context) mirror.FirewallState {
|
||||
cfg := p.settings()
|
||||
st := mirror.FirewallState{Enabled: cfg.Firewall.Enabled, Networks: slices.Clone(cfg.FirewallNetworks)}
|
||||
var toward []string
|
||||
if p.nodes != nil {
|
||||
toward = append(toward, p.nodes()...)
|
||||
}
|
||||
if p.st != nil {
|
||||
if cps, err := p.st.ControlPlanes(ctx); err == nil {
|
||||
for _, c := range cps {
|
||||
toward = append(toward, c.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, t := range toward {
|
||||
if a, ok := hostAddr(t); ok {
|
||||
st.Toward = append(st.Toward, a)
|
||||
}
|
||||
}
|
||||
slices.Sort(st.Toward)
|
||||
st.Toward = slices.Compact(st.Toward)
|
||||
return st
|
||||
}
|
||||
|
||||
// hostAddr is the address in an address, host:port or URL; names are not
|
||||
// looked up here.
|
||||
func hostAddr(s string) (string, bool) {
|
||||
if u, err := url.Parse(s); err == nil && u.Host != "" {
|
||||
s = u.Hostname()
|
||||
} else if h, _, err := net.SplitHostPort(s); err == nil {
|
||||
s = h
|
||||
}
|
||||
a, err := netip.ParseAddr(s)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return a.Unmap().String(), true
|
||||
}
|
||||
|
||||
func (p *firewallPublisher) once(ctx context.Context) {
|
||||
if p.leader != nil && !p.leader() {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
st := p.state(ctx)
|
||||
p.mu.Lock()
|
||||
m := p.mirror
|
||||
p.mu.Unlock()
|
||||
if m != nil {
|
||||
changed, err := m.SetFirewall(ctx, st)
|
||||
switch {
|
||||
case err != nil:
|
||||
p.log.Warn("could not tell the mirror what the settings say of its firewall", "err", err)
|
||||
case changed:
|
||||
p.log.Info("the mirror's firewall follows the cluster's settings", "enabled", st.Enabled)
|
||||
}
|
||||
}
|
||||
if p.st != nil {
|
||||
b, _ := json.Marshal(st)
|
||||
changed, err := p.st.PutClusterState(ctx, firewallStateKey, string(b))
|
||||
switch {
|
||||
case err != nil:
|
||||
p.log.Warn("could not record the firewall's state for the store's members", "err", err)
|
||||
case changed:
|
||||
p.log.Info("the store's members' firewall follows the cluster's settings", "enabled", st.Enabled)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
func TestTheLeaderTellsTheMirrorAndTheStoreWhatTheFirewallSettingsSay(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
if err := st.PutControlPlane(ctx, store.ControlPlane{URL: "https://10.0.0.150:7443", Name: "ctl02"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
srv := httptest.NewServer(mirror.NewAPI(mirror.NewManager(root, slog.New(slog.DiscardHandler)), "http://m:8081").Handler())
|
||||
defer srv.Close()
|
||||
mc, _ := mirror.NewClient(srv.URL, tlsconf.Config{}, 0)
|
||||
|
||||
cfg := config.Server{Firewall: config.FirewallSettings{Enabled: false}, FirewallNetworks: []string{"192.168.5.0/24"}}
|
||||
leader := false
|
||||
p := newFirewallPublisher(func() config.Server { return cfg }, st,
|
||||
func() []string { return []string{"10.0.0.147", "", "10.0.0.147", "node-by-name"} },
|
||||
func() bool { return leader }, slog.New(slog.DiscardHandler))
|
||||
p.setMirror(mc)
|
||||
|
||||
p.once(ctx)
|
||||
if _, err := st.ClusterState(ctx, firewallStateKey); err == nil {
|
||||
t.Error("a standby told the store")
|
||||
}
|
||||
leader = true
|
||||
p.once(ctx)
|
||||
var got mirror.FirewallState
|
||||
v, err := st.ClusterState(ctx, firewallStateKey)
|
||||
if err != nil || json.Unmarshal([]byte(v), &got) != nil {
|
||||
t.Fatalf("the store holds %q %v", v, err)
|
||||
}
|
||||
if got.Enabled || got.Networks[0] != "192.168.5.0/24" || len(got.Toward) != 2 ||
|
||||
got.Toward[0] != "10.0.0.147" || got.Toward[1] != "10.0.0.150" {
|
||||
t.Errorf("state %+v", got)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(root, mirror.FirewallFile))
|
||||
if err != nil || string(b) != v {
|
||||
t.Errorf("the mirror holds %s, the store %s", b, v)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -65,6 +66,14 @@ Safe to run again: it changes only what differs.`,
|
||||
}
|
||||
if !noFirewall {
|
||||
mirrorFirewall(cmd.Context(), cmd.OutOrStdout(), controlPlanes)
|
||||
// And after, as the cluster's settings say: the control plane
|
||||
// tells the mirror, which writes it beside its models.
|
||||
state := filepath.Join(*dir, mirror.FirewallFile)
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}, state); err != nil {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " !! firewall: %v; the cluster's settings will not change it here\n", err)
|
||||
} else {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), " ok firewall: follows the cluster's settings ("+firewallSyncPathUnit+")")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
|
||||
@@ -233,6 +233,9 @@ func newServerCmd() *cobra.Command {
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
// What the settings say of the firewall, for the mirror and the
|
||||
// store's members; nil without a control plane.
|
||||
var fwPub *firewallPublisher
|
||||
updateMetrics := &modelserve.UpdateCollector{}
|
||||
// The store's cluster, when it is one; read by the metrics and
|
||||
// by `ctl status`.
|
||||
@@ -276,6 +279,19 @@ func newServerCmd() *cobra.Command {
|
||||
ctl.Reconciler().SetNotifier(notifier)
|
||||
admin = ctl
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
fwPub = newFirewallPublisher(func() config.Server { return *current.Load() }, rt.Store(),
|
||||
func() []string {
|
||||
var out []string
|
||||
for _, n := range ctl.Inventory().Nodes() {
|
||||
out = append(out, n.MgmtAddr)
|
||||
}
|
||||
return out
|
||||
},
|
||||
func() bool { return ctl.Leadership() == nil || ctl.Leadership().IsLeader() }, log)
|
||||
go fwPub.run(ctx, 5*time.Minute)
|
||||
if l := ctl.Leadership(); l != nil {
|
||||
l.OnLead(fwPub.Kick)
|
||||
}
|
||||
|
||||
// Before AdminHandler, which is where the registry is built —
|
||||
// registering after it silently publishes nothing, which is
|
||||
@@ -519,6 +535,9 @@ func newServerCmd() *cobra.Command {
|
||||
return fmt.Errorf("the model mirror: %w", err)
|
||||
}
|
||||
rt.SetMirror(mc)
|
||||
if fwPub != nil {
|
||||
fwPub.setMirror(mc)
|
||||
}
|
||||
// Adapters trained on the nodes are kept there too; sending one
|
||||
// takes longer than the mirror's usual answers.
|
||||
if adminAPI != nil {
|
||||
@@ -743,6 +762,9 @@ func newServerCmd() *cobra.Command {
|
||||
if agentSvc != nil {
|
||||
agentSvc.BroadcastFirewall()
|
||||
}
|
||||
if fwPub != nil {
|
||||
fwPub.Kick()
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "agents") }) && agentSvc != nil {
|
||||
// A moment apart per node, so off this goroutine.
|
||||
|
||||
@@ -160,7 +160,7 @@ func (sa *serverAdd) firewall(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
// What applies the cluster's firewall setting there when it changes.
|
||||
if err := installFirewallSync(ctx, sa.remote); err != nil {
|
||||
if err := installFirewallSync(ctx, sa.remote, firewallStateFile); err != nil {
|
||||
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
|
||||
}
|
||||
if !cfg.Firewall.Enabled {
|
||||
|
||||
@@ -93,6 +93,11 @@ func (in *serverInit) storeFirewall(ctx context.Context, members []pgha.Node, se
|
||||
} else {
|
||||
in.kept("firewall on %s", m.Name)
|
||||
}
|
||||
// And after, as the cluster's settings say, read from the
|
||||
// member's own copy of the store.
|
||||
if err := firewall.InstallFollower(ctx, m.Remote); err != nil {
|
||||
fmt.Fprintf(in.out, " !! firewall on %s: %v; the cluster's settings will not change it\n", m.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -135,5 +140,5 @@ func loopbackListen(listen string) bool {
|
||||
// ensureFirewallSync installs the units that apply the cluster's firewall
|
||||
// setting on this control plane.
|
||||
func (in *serverInit) ensureFirewallSync(ctx context.Context) error {
|
||||
return installFirewallSync(ctx, firewall.Local{})
|
||||
return installFirewallSync(ctx, firewall.Local{}, firewallStateFile)
|
||||
}
|
||||
Reference in new issue
Block a user