Files
llmgw/internal/agent/firewall.go
T
Ilker ManapandClaude Opus 5.5 748f777a56
CI / test (push) Successful in 10m57s
Slurm's accounting: slurmdbd and a MariaDB on the controller
Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 07:59:08 +01:00

120 lines
3.6 KiB
Go

package agent
import (
"context"
"net"
"net/netip"
"net/url"
"strings"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// slurmPorts are Slurm's on a node that runs any of it: slurmctld's and
// slurmd's, and srun's range for the steps it starts.
var slurmPorts = []firewall.Port{
{From: slurm.CtldPort, To: slurm.DbdPort, What: "slurm: slurmctld, slurmd and slurmdbd"},
{From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"},
}
// applyFirewall keeps llmgw's firewall on this node as the cluster's
// settings say: the engines' ports and the node exporter's for the networks
// the node reaches its control planes on, and those named, or no table at
// all. Off the receive loop, one at a time.
func (a *Agent) applyFirewall(m *llmgwv1.Firewall) {
a.fwMu.Lock()
defer a.fwMu.Unlock()
a.fwLast = m
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
r := a.fwRunner
if r == nil {
r = firewall.Local{}
}
if !m.GetEnabled() {
had, err := firewall.Disable(ctx, r)
switch {
case err != nil:
a.log.Warn("could not take llmgw's firewall off this node", "err", err)
case had:
a.log.Info("llmgw's firewall is off on this node, as the cluster's settings say")
}
return
}
a.mu.Lock()
servers := append([]string(nil), a.serverList...)
a.mu.Unlock()
nets, err := firewall.Networks(ctx, r, controlPlaneIPs(ctx, servers))
if err != nil {
a.log.Warn("could not tell which networks reach the control planes; the firewall is left as it is", "err", err)
return
}
for _, n := range m.GetNetworks() {
if p, err := netip.ParsePrefix(n); err == nil {
nets = append(nets, p)
} else if ad, err := netip.ParseAddr(n); err == nil {
nets = append(nets, netip.PrefixFrom(ad, ad.BitLen()))
}
}
from, to := int(m.GetEnginePortsFrom()), int(m.GetEnginePortsTo())
ports := []firewall.Port{firewall.NodeExporter}
if from > 0 && to >= from {
ports = append(ports, firewall.Engines(from, to))
}
// Once there, they stay: the ruleset keeps every port it held, so a
// node that leaves Slurm keeps them closed to everyone but the cluster.
if a.slurm != nil && a.slurm.HasPart() {
ports = append(ports, slurmPorts...)
}
changed, err := firewall.Install(ctx, r, firewall.Spec{Ports: ports, Allow: nets})
switch {
case err != nil:
a.log.Warn("could not put llmgw's firewall on this node", "err", err)
case changed:
a.log.Info("llmgw's firewall is on this node", "networks", len(nets))
}
}
// slurmMessage is the leader's word on this node's part in Slurm. A node
// that gets one has the firewall applied again, with Slurm's ports.
func (a *Agent) slurmMessage(m *llmgwv1.Slurm) {
had := a.slurm.HasPart()
a.slurm.Apply(m)
if had || !a.slurm.HasPart() {
return
}
a.fwMu.Lock()
last := a.fwLast
a.fwMu.Unlock()
if last != nil && last.GetEnabled() {
go a.applyFirewall(last)
}
}
// controlPlaneIPs are the addresses of the control planes an agent dials.
func controlPlaneIPs(ctx context.Context, servers []string) []string {
var out []string
for _, s := range servers {
host := s
if u, err := url.Parse(s); err == nil && u.Host != "" {
host = u.Hostname()
}
host = strings.Trim(host, "[]")
if a, err := netip.ParseAddr(host); err == nil {
out = append(out, a.String())
continue
}
ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
if err != nil {
continue
}
for _, ip := range ips {
out = append(out, ip.Unmap().String())
}
}
return out
}