Stateless (Warewulf-style) provisioning: profiles, node pools, overlays, dnsmasq export, compute image
Profiles bind a RAM-booted root image to overlay templates; assigning a machine allocates a node name and management IP, writes the per-MAC PXE entry with a per-node token and exports static leases for dnsmasq. Nodes pull rendered overlays (hosts, slurm.conf from inventory, gres.conf, SSH host key, munge key) through rollcall-overlay.service; the runtime refresh timer is the heartbeat. Adds the compute-debian13 image definition and a generic live-build root image script, CLI and web UI for profiles/assign. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
1430421a3f
commit
7fe7c26e22
38 files changed
+1543
-16
No files matched your search
@@ -8,6 +8,19 @@
|
||||
"agent_token": "<AGENT-TOKEN: rollcall keygen>",
|
||||
"commission_append": "boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 rollcall.token=<AGENT-TOKEN> console=tty0 console=ttyS0,115200n8",
|
||||
"ipmitool": "ipmitool",
|
||||
"server_url": "http://10.213.77.1",
|
||||
"images_dir": "/srv/rollcall/images",
|
||||
"overlays_dir": "/etc/rollcall/overlays",
|
||||
"dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
|
||||
"node_pools": {"mgmt": "10.213.77.50-10.213.77.99"},
|
||||
"cluster_vars": {
|
||||
"domain": "pxe.local",
|
||||
"cluster_name": "rollcall-lab",
|
||||
"slurm_controller": "pxe-server",
|
||||
"slurm_controller_ip": "10.213.77.1",
|
||||
"partition": "compute",
|
||||
"ssh_keys": ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDSAJ4owXlJreHP3sa4HstasxA5pk1ZAPi6vnzOpA5R1 ilker@t440s"]
|
||||
},
|
||||
"bmc_hints": [
|
||||
{"mac": "bc:24:11:8e:1e:6c", "address": "10.213.77.11", "port": 623, "user": "admin", "password": "password", "comment": "pxe-node01 (VM 211, pve-vbmc)"},
|
||||
{"mac": "bc:24:11:36:90:3f", "address": "10.213.77.12", "port": 623, "user": "admin", "password": "password", "comment": "pxe-node02 (VM 212, pve-vbmc)"}
|
||||
|
||||
@@ -19,7 +19,20 @@ install -m 0644 /usr/lib/syslinux/modules/bios/poweroff.c32 /srv/tftp/poweroff.c
|
||||
sed "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
|
||||
[ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall
|
||||
[ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall
|
||||
# stateless images: <images_dir>/<name>/..., reachable by pxelinux as images/<name>/
|
||||
install -d /srv/rollcall/images
|
||||
[ -e /srv/tftp/images ] || ln -s /srv/rollcall/images /srv/tftp/images
|
||||
[ -d /srv/tftp/efi64 ] && ln -sfn ../images /srv/tftp/efi64/images
|
||||
# overlay templates (server-only secrets such as munge.key live next to them and are kept)
|
||||
install -d -m 0750 -o root -g rollcall /etc/rollcall/overlays
|
||||
cp -a overlays/. /etc/rollcall/overlays/
|
||||
chgrp -R rollcall /etc/rollcall/overlays
|
||||
# dnsmasq host export + reload trigger
|
||||
install -d -m 0755 /var/lib/rollcall
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now rollcall-dnsmasq.path >/dev/null 2>&1 || true
|
||||
systemctl enable --now rollcall.service
|
||||
systemctl restart rollcall.service
|
||||
sleep 1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../system/etc/hosts.tmpl
|
||||
@@ -0,0 +1 @@
|
||||
../../../system/etc/slurm/slurm.conf.tmpl
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
# runs when the runtime overlay changed (new nodes, partition changes)
|
||||
systemctl try-restart slurmd.service
|
||||
exit 0
|
||||
@@ -0,0 +1,11 @@
|
||||
127.0.0.1 localhost
|
||||
::1 localhost ip6-localhost ip6-loopback
|
||||
{{- if .Vars.slurm_controller_ip }}
|
||||
{{ .Vars.slurm_controller_ip }} {{ .Vars.slurm_controller }}{{ if .Vars.domain }} {{ .Vars.slurm_controller }}.{{ .Vars.domain }}{{ end }}
|
||||
{{- end }}
|
||||
{{- range .Nodes }}
|
||||
{{ .MgmtIP }} {{ .Name }}{{ if $.Vars.domain }} {{ .Name }}.{{ $.Vars.domain }}{{ end }}
|
||||
{{- if .IBIP }}
|
||||
{{ .IBIP }} {{ .Name }}-ib
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,5 @@
|
||||
# rendered by rollcall for {{ .Node.Name }}
|
||||
{{- if gt .Node.GPUs 0 }}
|
||||
AutoDetect=off
|
||||
NodeName={{ .Node.Name }} Name=gpu File=/dev/nvidia[0-{{ sub .Node.GPUs 1 }}]
|
||||
{{- end }}
|
||||
@@ -0,0 +1,27 @@
|
||||
# rendered by rollcall {{ .Now }} - do not edit on the node
|
||||
ClusterName={{ default "rollcall" .Vars.cluster_name }}
|
||||
SlurmctldHost={{ .Vars.slurm_controller }}({{ .Vars.slurm_controller_ip }})
|
||||
SlurmUser=slurm
|
||||
AuthType=auth/munge
|
||||
StateSaveLocation=/var/spool/slurmctld
|
||||
SlurmdSpoolDir=/var/spool/slurmd
|
||||
SlurmctldPidFile=/run/slurmctld.pid
|
||||
SlurmdPidFile=/run/slurmd.pid
|
||||
SlurmctldLogFile=/var/log/slurm/slurmctld.log
|
||||
SlurmdLogFile=/var/log/slurm/slurmd.log
|
||||
ProctrackType=proctrack/linuxproc
|
||||
TaskPlugin=task/none
|
||||
ReturnToService=2
|
||||
SlurmctldTimeout=120
|
||||
SlurmdTimeout=300
|
||||
SchedulerType=sched/backfill
|
||||
SelectType=select/cons_tres
|
||||
SelectTypeParameters=CR_Core_Memory
|
||||
GresTypes=gpu
|
||||
MpiDefault=none
|
||||
|
||||
# nodes: generated from the commissioning inventory
|
||||
{{- range .Nodes }}
|
||||
NodeName={{ .Name }} NodeAddr={{ .MgmtIP }} Sockets={{ .Sockets }} CoresPerSocket={{ .CoresPerSocket }} ThreadsPerCore={{ .ThreadsPerCore }} RealMemory={{ pct .MemMB 90 }}{{ if gt .GPUs 0 }} Gres=gpu:{{ .GPUs }}{{ end }} State=UNKNOWN
|
||||
{{- end }}
|
||||
PartitionName={{ default "compute" .Vars.partition }} Nodes=ALL Default=YES MaxTime=INFINITE State=UP
|
||||
@@ -0,0 +1 @@
|
||||
{{ .Node.HostKeyPub }} {{ .Node.Name }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ .Node.HostKey }}
|
||||
@@ -0,0 +1,2 @@
|
||||
{{ range .Vars.ssh_keys }}{{ . }}
|
||||
{{ end }}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# runs once after the system overlay is applied (node {{ .Node.Name }})
|
||||
hostnamectl set-hostname {{ .Node.Name }} 2>/dev/null || hostname {{ .Node.Name }}
|
||||
chown munge:munge /etc/munge/munge.key && chmod 0400 /etc/munge/munge.key
|
||||
chmod 0600 /etc/ssh/ssh_host_ed25519_key; chmod 0644 /etc/ssh/ssh_host_ed25519_key.pub
|
||||
chmod 0700 /root/.ssh; chmod 0600 /root/.ssh/authorized_keys
|
||||
install -d -m 0755 /var/spool/slurmd /var/log/slurm
|
||||
systemctl daemon-reload
|
||||
exit 0
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=rollcall: reload dnsmasq when node host files change
|
||||
|
||||
[Path]
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts.dhcp
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=rollcall: HUP dnsmasq to re-read dhcp-hostsfile / addn-hosts
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl kill -s HUP dnsmasq.service
|
||||
@@ -16,7 +16,7 @@ NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg
|
||||
ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg /var/lib/rollcall
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in new issue
Block a user