The firewall setting reaches the mirror machine and the store's members
The leading control plane tells the mirror over its API (PUT /v1/firewall, kept as .firewall.json beside the models and applied by the same path unit control planes use), and keeps the state in the store's cluster_state for members that run no llmgw, which a root timer reads from the member's own copy. The mirror also lets in the network of every node and control plane the leader knows. Tried on the PoC: off, on and a network added, on every kind of machine. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5be3da505b
commit
8df78a4d41
20 files changed
+772
-34
No files matched your search
@@ -0,0 +1,154 @@
|
||||
package firewall
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A store member that is nothing else runs no llmgw process: Patroni, etcd
|
||||
// and PostgreSQL only. What the cluster's settings say of its firewall it
|
||||
// reads from its own copy of the store, where the leading control plane
|
||||
// keeps it (the cluster_state table, key firewall), once a minute, by a
|
||||
// root timer running FollowScript. It turns llmgw's table off and on, and
|
||||
// lets in the networks the settings add; what `server init` let in stays.
|
||||
|
||||
// FollowScript is the script, as python3 runs it. LLMGW_FW_STATE and
|
||||
// LLMGW_FW_RULES replace the store and the rules file, for its tests.
|
||||
const FollowScript = `#!/usr/bin/python3
|
||||
# Written by llmgw: llmgw's firewall on this store member, as the cluster's
|
||||
# settings say. Run by llmgw-firewall-follow.timer.
|
||||
import ipaddress, json, os, subprocess, sys, tempfile
|
||||
|
||||
RULES = os.environ.get("LLMGW_FW_RULES", "` + RulesFile + `")
|
||||
|
||||
def run(*a):
|
||||
return subprocess.run(a, capture_output=True, text=True)
|
||||
|
||||
raw = os.environ.get("LLMGW_FW_STATE")
|
||||
if raw is None:
|
||||
r = run("runuser", "-u", "postgres", "--", "psql", "-d", "llmgw", "-tAqc",
|
||||
"SELECT value FROM cluster_state WHERE key = 'firewall'")
|
||||
# No table yet, or nothing told: a control plane older than this, or
|
||||
# none leading. Left as it is.
|
||||
if r.returncode != 0:
|
||||
sys.exit(0)
|
||||
raw = r.stdout
|
||||
if not raw.strip():
|
||||
sys.exit(0)
|
||||
st = json.loads(raw)
|
||||
present = run("nft", "list", "table", "inet", "` + Table + `").returncode == 0
|
||||
|
||||
if not st.get("enabled", True):
|
||||
if present:
|
||||
run("systemctl", "disable", "--now", "` + Unit + `")
|
||||
run("nft", "delete", "table", "inet", "` + Table + `")
|
||||
print("llmgw's firewall is off here")
|
||||
sys.exit(0)
|
||||
|
||||
if not os.path.exists(RULES):
|
||||
sys.exit(0)
|
||||
lines = open(RULES).read().split("\n")
|
||||
want = []
|
||||
for n in st.get("networks") or []:
|
||||
try:
|
||||
want.append(ipaddress.ip_network(n, strict=False))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def saddr(l):
|
||||
s = l.strip()
|
||||
return s.startswith("ip saddr {") or s.startswith("ip6 saddr {")
|
||||
|
||||
changed = False
|
||||
for fam, kw in ((4, "ip saddr {"), (6, "ip6 saddr {")):
|
||||
add = [str(n) for n in want if n.version == fam]
|
||||
if not add:
|
||||
continue
|
||||
same = [i for i, l in enumerate(lines) if l.strip().startswith(kw)]
|
||||
if same:
|
||||
i = same[0]
|
||||
l = lines[i]
|
||||
a, b = l.index("{") + 1, l.index("}")
|
||||
have = [x.strip() for x in l[a:b].split(",") if x.strip()]
|
||||
new = [x for x in add if x not in have]
|
||||
if new:
|
||||
lines[i] = l[:a] + " " + ", ".join(have + new) + " " + l[b:]
|
||||
changed = True
|
||||
continue
|
||||
other = [i for i, l in enumerate(lines) if saddr(l)]
|
||||
if not other:
|
||||
continue
|
||||
l = lines[other[0]]
|
||||
indent = l[:len(l) - len(l.lstrip())]
|
||||
lines.insert(other[0] + 1, indent + kw + " " + ", ".join(add) + " }" + l[l.index("}") + 1:])
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(RULES))
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write("\n".join(lines))
|
||||
if run("nft", "-c", "-f", tmp).returncode != 0:
|
||||
os.unlink(tmp)
|
||||
print("the settings' networks make a ruleset nft refuses; left as it was", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, RULES)
|
||||
if changed or not present:
|
||||
run("systemctl", "enable", "` + Unit + `")
|
||||
if run("nft", "-f", RULES).returncode != 0:
|
||||
sys.exit(1)
|
||||
print("llmgw's firewall is on here")
|
||||
`
|
||||
|
||||
// FollowerPath is where FollowScript is installed.
|
||||
const FollowerPath = "/usr/local/lib/llmgw/firewall-follow"
|
||||
|
||||
// FollowerTimer is the timer that runs it.
|
||||
const FollowerTimer = "llmgw-firewall-follow.timer"
|
||||
|
||||
const followService = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this store member
|
||||
After=postgresql.service patroni.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 ` + FollowerPath + `
|
||||
`
|
||||
|
||||
const followTimer = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this store member
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=1min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
`
|
||||
|
||||
// InstallFollower puts FollowScript and its timer on a store member.
|
||||
func InstallFollower(ctx context.Context, r Runner) error {
|
||||
body := FollowScript + "\x00" + followService + "\x00" + followTimer
|
||||
out, err := r.Run(ctx, `set -e
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
cat > "$t/all"
|
||||
python3 - "$t" <<'PY'
|
||||
import sys
|
||||
d = sys.argv[1]
|
||||
s, svc, tim = open(d + "/all", "rb").read().split(b"\x00")
|
||||
open(d + "/script", "wb").write(s)
|
||||
open(d + "/service", "wb").write(svc)
|
||||
open(d + "/timer", "wb").write(tim)
|
||||
PY
|
||||
sudo install -d -m 755 /usr/local/lib/llmgw
|
||||
sudo install -m 755 "$t/script" `+FollowerPath+`
|
||||
sudo install -m 644 "$t/service" /etc/systemd/system/llmgw-firewall-follow.service
|
||||
sudo install -m 644 "$t/timer" /etc/systemd/system/`+FollowerTimer+`
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now `+FollowerTimer+` >/dev/null 2>&1`, []byte(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing %s: %v: %s", FollowerTimer, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package firewall
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// follow runs FollowScript against a rules file and a state, with nft and
|
||||
// systemctl replaced by scripts that log what they were asked and answer as
|
||||
// a machine whose table is loaded or not.
|
||||
func follow(t *testing.T, rules, state string, loaded bool) (string, string, string) {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("python3"); err != nil {
|
||||
t.Skip("no python3")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "bin")
|
||||
_ = os.Mkdir(bin, 0o755)
|
||||
log := filepath.Join(dir, "log")
|
||||
list := "exit 1"
|
||||
if loaded {
|
||||
list = "exit 0"
|
||||
}
|
||||
for name, body := range map[string]string{
|
||||
"nft": `echo "nft $*" >> ` + log + "\n" + `[ "$1" = list ] && ` + list + "\nexit 0\n",
|
||||
"systemctl": `echo "systemctl $*" >> ` + log + "\nexit 0\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\n"+body), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rf := filepath.Join(dir, "firewall.nft")
|
||||
if rules != "" {
|
||||
_ = os.WriteFile(rf, []byte(rules), 0o644)
|
||||
}
|
||||
script := filepath.Join(dir, "follow")
|
||||
_ = os.WriteFile(script, []byte(FollowScript), 0o755)
|
||||
cmd := exec.Command("python3", script)
|
||||
cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "LLMGW_FW_STATE="+state, "LLMGW_FW_RULES="+rf)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %s", err, out)
|
||||
}
|
||||
got, _ := os.ReadFile(rf)
|
||||
calls, _ := os.ReadFile(log)
|
||||
return string(out), string(got), string(calls)
|
||||
}
|
||||
|
||||
var memberRules = Script(Spec{Ports: []Port{Postgres, EtcdClient}, Allow: mustPrefixes("10.0.0.0/24")})
|
||||
|
||||
func TestAStoreMemberFollowsTheSettings(t *testing.T) {
|
||||
// Off: the table goes, the rules stay for later.
|
||||
out, rules, calls := follow(t, memberRules, `{"enabled":false}`, true)
|
||||
if !strings.Contains(out, "off") || !strings.Contains(calls, "nft delete table inet llmgw") ||
|
||||
!strings.Contains(calls, "systemctl disable --now "+Unit) || rules != memberRules {
|
||||
t.Errorf("off: %q %q", out, calls)
|
||||
}
|
||||
// On again: loaded from the rules kept.
|
||||
out, _, calls = follow(t, memberRules, `{"enabled":true}`, false)
|
||||
if !strings.Contains(out, "on") || !strings.Contains(calls, "nft -f") {
|
||||
t.Errorf("on: %q %q", out, calls)
|
||||
}
|
||||
// On and loaded, nothing new: nothing done.
|
||||
out, _, calls = follow(t, memberRules, `{"enabled":true,"networks":["10.0.0.0/24"]}`, true)
|
||||
if out != "" || strings.Contains(calls, "nft -f") {
|
||||
t.Errorf("unchanged: %q %q", out, calls)
|
||||
}
|
||||
// Networks the settings add, of both families, are let in beside what
|
||||
// init let in; the result reads back as the same ports.
|
||||
_, rules, calls = follow(t, memberRules, `{"enabled":true,"networks":["192.168.5.0/24","fd00::/64","nonsense"]}`, true)
|
||||
if !strings.Contains(calls, "nft -c -f") || !strings.Contains(calls, "nft -f") {
|
||||
t.Errorf("not checked and loaded: %q", calls)
|
||||
}
|
||||
back := parse(rules)
|
||||
var nets []string
|
||||
for _, p := range back.Allow {
|
||||
nets = append(nets, p.String())
|
||||
}
|
||||
if strings.Join(nets, ",") != "10.0.0.0/24,192.168.5.0/24,fd00::/64" || len(back.Ports) != 2 {
|
||||
t.Errorf("rules now:\n%s", rules)
|
||||
}
|
||||
// Nothing told yet, or no rules: left alone.
|
||||
if out, _, calls := follow(t, memberRules, "", true); out != "" || calls != "" {
|
||||
t.Errorf("nothing told: %q %q", out, calls)
|
||||
}
|
||||
if out, _, calls := follow(t, "", `{"enabled":true}`, false); out != "" || strings.Contains(calls, "nft -f") {
|
||||
t.Errorf("no rules: %q %q", out, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func mustPrefixes(ss ...string) []netip.Prefix {
|
||||
var out []netip.Prefix
|
||||
for _, s := range ss {
|
||||
out = append(out, netip.MustParsePrefix(s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user