ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
635 lines
22 KiB
Go
635 lines
22 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
// A simulated Proxmox: Proxmox VE in a QEMU/KVM virtual machine on the
|
|
// control plane host itself, for someone who wants to try hpcod and has no
|
|
// Proxmox to point it at.
|
|
//
|
|
// It sits on a bridge of its own with NAT to the host's uplink, so it
|
|
// never touches the network the host is on: its head nodes cannot collide
|
|
// with anything there, and nothing there can reach them except through
|
|
// the portal. Clusters run inside it, so the host needs nested
|
|
// virtualization, and everything is slower than on a real hypervisor -
|
|
// it is for trying, not for tenants.
|
|
const (
|
|
SimDir = VarDir + "/pvesim"
|
|
SimUnit = "hpcod-pvesim"
|
|
SimNetUnit = "hpcod-simnet"
|
|
SimBridge = "hpcodbr0"
|
|
SimTap = "hpcodsim0"
|
|
SimNet = "10.99.0"
|
|
SimGateway = SimNet + ".1"
|
|
SimAddress = SimNet + ".2"
|
|
SimNode = "pve-sim"
|
|
SimPassFile = EtcDir + "/pvesim.env"
|
|
simImageURL = "https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2"
|
|
)
|
|
|
|
// Sim reports whether the site uses a simulated Proxmox.
|
|
func (s Site) Sim() bool { return s["PVE_MODE"] == "sim" }
|
|
|
|
// ApplySim fills in every Proxmox and network value of a simulated site:
|
|
// none of them is a choice when the hypervisor is ours.
|
|
func (s Site) ApplySim() {
|
|
if !s.Sim() {
|
|
return
|
|
}
|
|
for k, v := range map[string]string{
|
|
"PVE_ENDPOINT": "https://" + SimAddress + ":8006", "PVE_INSECURE_TLS": "true",
|
|
"PVE_HOST": SimAddress, "PVE_NODE": SimNode, "PVE_NODES": "",
|
|
"PVE_SITE_ID": "pvesim", "PVE_SITE_NAME": "Simulated Proxmox",
|
|
"PVE_DATASTORE": "local", "PVE_SNIPPET_DATASTORE": "local",
|
|
"PVE_IMAGE": "local:import/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2",
|
|
"PVE_SDN_ZONE": "hpcod", "MGMT_BRIDGE": "vmbr0", "MGMT_GATEWAY": SimGateway,
|
|
"MGMT_FIRST": SimNet + ".100", "MGMT_COUNT": "20", "BUILD_IP": SimNet + ".99",
|
|
} {
|
|
s[k] = v
|
|
}
|
|
}
|
|
|
|
// ClearSim undoes ApplySim on a site that has been switched to a real
|
|
// Proxmox: a value that is still the simulation's own is reset to the
|
|
// default, so the form does not offer the simulation's private address as
|
|
// the real Proxmox's.
|
|
func (s Site) ClearSim() {
|
|
if s.Sim() {
|
|
return
|
|
}
|
|
sim := Site{"PVE_MODE": "sim"}
|
|
sim.ApplySim()
|
|
def := Defaults()
|
|
for k, v := range sim {
|
|
if k != "PVE_MODE" && s[k] == v {
|
|
s[k] = def[k]
|
|
}
|
|
}
|
|
}
|
|
|
|
// SimDefaults sizes the simulated Proxmox from what the host has: most of
|
|
// its cores, half its memory up to 64 GiB.
|
|
func SimDefaults() (cores, memGB int) {
|
|
cores = 4
|
|
if out, err := exec.Command("nproc").Output(); err == nil {
|
|
if n, _ := strconv.Atoi(strings.TrimSpace(string(out))); n > 2 {
|
|
cores = n - 1
|
|
}
|
|
}
|
|
memGB = 16
|
|
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
|
|
var kb int
|
|
for _, l := range strings.Split(string(raw), "\n") {
|
|
if strings.HasPrefix(l, "MemTotal:") {
|
|
fmt.Sscanf(strings.TrimPrefix(l, "MemTotal:"), "%d", &kb)
|
|
}
|
|
}
|
|
if g := kb / 1024 / 1024 / 2; g > 0 {
|
|
memGB = min(g, 64)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// SimPreflight says what the host lacks for a simulated Proxmox, before
|
|
// anything is installed or asked.
|
|
func SimPreflight() []string {
|
|
var miss []string
|
|
if _, err := os.Stat("/dev/kvm"); err != nil {
|
|
miss = append(miss, "/dev/kvm: this host cannot run KVM guests. On a VM, give it CPU type \"host\" and enable nested virtualization on its hypervisor.")
|
|
}
|
|
nested := ""
|
|
for _, f := range []string{"/sys/module/kvm_intel/parameters/nested", "/sys/module/kvm_amd/parameters/nested"} {
|
|
if b, err := os.ReadFile(f); err == nil {
|
|
nested = strings.TrimSpace(string(b))
|
|
}
|
|
}
|
|
if nested != "Y" && nested != "1" {
|
|
miss = append(miss, "nested virtualization is off: the clusters run inside the simulated Proxmox and need it (kvm_intel/kvm_amd nested=1).")
|
|
}
|
|
cores, mem := SimDefaults()
|
|
if cores < 4 {
|
|
miss = append(miss, fmt.Sprintf("%d cores: a head node and one compute node need at least 4.", cores+1))
|
|
}
|
|
if mem < 12 {
|
|
miss = append(miss, fmt.Sprintf("about %d GiB of memory to give it: a head and a compute node need at least 12.", mem))
|
|
}
|
|
return miss
|
|
}
|
|
|
|
// stepPVESim brings up the simulated Proxmox, if the site has one, and
|
|
// returns once its API answers.
|
|
func stepPVESim(ctx context.Context, e *Env) error {
|
|
s := e.Site
|
|
if !s.Sim() {
|
|
e.Logf("clusters are built on %s; no simulated Proxmox", s["PVE_ENDPOINT"])
|
|
return nil
|
|
}
|
|
if miss := SimPreflight(); len(miss) > 0 {
|
|
return fmt.Errorf("this host cannot run a simulated Proxmox:\n - %s", strings.Join(miss, "\n - "))
|
|
}
|
|
if err := e.mkdir(SimDir, 0o750, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
|
|
// The password setup signs in with, kept for the next run of setup and
|
|
// for the operator: it is root on a hypervisor nothing else can reach.
|
|
pw := envValue(e.P(SimPassFile), "PVESIM_ROOT_PASSWORD")
|
|
if pw == "" {
|
|
pw = randomWord(24)
|
|
if err := e.writeFile(SimPassFile, []byte("PVESIM_ROOT_PASSWORD="+pw+"\n"), 0o600, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
e.Creds.Password = pw
|
|
|
|
if err := e.simNetwork(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
disk := SimDir + "/pve.qcow2"
|
|
if !exists(e.P(disk)) {
|
|
base := SimDir + "/debian-13.qcow2"
|
|
if !exists(e.P(base)) {
|
|
if err := e.download(ctx, simImageURL, e.P(base)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := e.run(ctx, "", "qemu-img", "convert", "-O", "qcow2", base, disk); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "qemu-img", "resize", disk, s["SIM_DISK_GB"]+"G"); err != nil {
|
|
return err
|
|
}
|
|
if err := e.simSeed(ctx, pw); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("disk %s (%s GiB) prepared", disk, s["SIM_DISK_GB"])
|
|
}
|
|
|
|
if err := e.writeFile("/etc/systemd/system/"+SimUnit+".service", []byte(simUnit(s)), 0o644, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "daemon-reload"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "enable", "--now", SimUnit); err != nil {
|
|
return err
|
|
}
|
|
|
|
// The first boot installs Proxmox and reboots into it: ten to twenty
|
|
// minutes, all of it visible on the serial console.
|
|
e.Logf("waiting for Proxmox VE inside it (10-20 minutes on a first install); its console follows")
|
|
console := e.P(SimDir + "/console.log")
|
|
var off int64
|
|
deadline := time.Now().Add(40 * time.Minute)
|
|
ready := func() bool {
|
|
if !tokenOrTicket(ctx, s, pw) {
|
|
return false
|
|
}
|
|
// The API answers part-way through the install, before the reboot
|
|
// into Proxmox's kernel; that is not ready.
|
|
c, err := dialSSHPassword(ctx, SimAddress, pw)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
defer c.Close()
|
|
_, err = sshRun(c, "test -e /root/.pve-installed && uname -r | grep -q -- -pve")
|
|
return err == nil
|
|
}
|
|
for !ready() {
|
|
if f, err := os.Open(console); err == nil {
|
|
if st, _ := f.Stat(); st != nil && st.Size() > off {
|
|
buf := make([]byte, st.Size()-off)
|
|
f.ReadAt(buf, off)
|
|
off = st.Size()
|
|
for _, l := range strings.Split(string(buf), "\n") {
|
|
l = strings.TrimSpace(strings.Map(printable, l))
|
|
if i := strings.Index(l, "hpcod-sim:"); i >= 0 {
|
|
e.Logf(" %s", l[i:])
|
|
if strings.Contains(l, "FAILED") {
|
|
return fmt.Errorf("installing Proxmox inside the simulation failed:\n%s", e.simLogTail(ctx))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
f.Close()
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "is-active", "--quiet", SimUnit); err != nil {
|
|
j, _ := e.run(ctx, "", "journalctl", "-u", SimUnit, "-n", "15", "--no-pager")
|
|
return fmt.Errorf("the simulated Proxmox stopped:\n%s", j)
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("Proxmox did not answer at %s within 40 minutes; see %s", s["PVE_ENDPOINT"], console)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(10 * time.Second):
|
|
}
|
|
}
|
|
|
|
// Its guests run under KVM too. Without nested virtualization reaching
|
|
// it, the template build fails at qm start with a message about kvm
|
|
// that does not mention this host.
|
|
c, err := dialSSHPassword(ctx, SimAddress, pw)
|
|
if err != nil {
|
|
return fmt.Errorf("Proxmox answers but root SSH with its password does not: %w", err)
|
|
}
|
|
defer c.Close()
|
|
// Its DNS, set the way Proxmox keeps it: through its own API, which
|
|
// writes resolv.conf as a file it owns. Then checked from inside, by
|
|
// resolving the name its first download needs.
|
|
if api, err := PVEWithPassword(ctx, s["PVE_ENDPOINT"], true, "root@pam", pw); err == nil {
|
|
ns := strings.Fields(s["NAMESERVERS"])
|
|
form := url.Values{"search": {"hpcod.sim"}}
|
|
for i, n := range ns {
|
|
if i < 3 {
|
|
form.Set(fmt.Sprintf("dns%d", i+1), n)
|
|
}
|
|
}
|
|
if err := api.Put(ctx, "/nodes/"+SimNode+"/dns", form); err != nil {
|
|
return fmt.Errorf("setting the simulated Proxmox's DNS: %w", err)
|
|
}
|
|
}
|
|
if _, err := sshRun(c, "getent hosts dl.rockylinux.org"); err != nil {
|
|
return fmt.Errorf("the simulated Proxmox cannot resolve names, so it cannot download anything: %w", err)
|
|
}
|
|
if _, err := sshRun(c, "test -e /dev/kvm"); err != nil {
|
|
return fmt.Errorf("the simulated Proxmox has no /dev/kvm, so it cannot run cluster VMs: enable nested virtualization on this host")
|
|
}
|
|
e.Logf("Proxmox VE answers at %s, with KVM for its guests", s["PVE_ENDPOINT"])
|
|
return nil
|
|
}
|
|
|
|
// simLogTail is the end of the install log inside the simulation, read
|
|
// over SSH with root's password.
|
|
func (e *Env) simLogTail(ctx context.Context) string {
|
|
c, err := dialSSHPassword(ctx, SimAddress, e.Creds.Password)
|
|
if err != nil {
|
|
return "(cannot reach it to read /var/log/install-pve.log: " + err.Error() + ")"
|
|
}
|
|
defer c.Close()
|
|
out, _ := sshRun(c, "grep -nE '^E:|error' /var/log/install-pve.log | tail -8; echo ...; tail -12 /var/log/install-pve.log")
|
|
return out
|
|
}
|
|
|
|
// download fetches a URL to a file, saying how far it has got every tenth
|
|
// of the way: a large file over a slow mirror is otherwise forty minutes
|
|
// of nothing, which reads as a hang.
|
|
func (e *Env) download(ctx context.Context, url, dst string) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("%s: %s", url, resp.Status)
|
|
}
|
|
f, err := os.Create(dst + ".part")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
total := resp.ContentLength
|
|
e.Logf("downloading %s (%d MB)", path.Base(url), total>>20)
|
|
start, next := time.Now(), int64(10)
|
|
var got int64
|
|
buf := make([]byte, 1<<20)
|
|
for {
|
|
n, rerr := resp.Body.Read(buf)
|
|
if n > 0 {
|
|
if _, err := f.Write(buf[:n]); err != nil {
|
|
return err
|
|
}
|
|
got += int64(n)
|
|
if total > 0 && got*100/total >= next {
|
|
rate := float64(got) / 1e6 / time.Since(start).Seconds()
|
|
e.Logf(" %d%% %.1f MB/s", got*100/total, rate)
|
|
next += 10
|
|
}
|
|
}
|
|
if rerr == io.EOF {
|
|
break
|
|
}
|
|
if rerr != nil {
|
|
return rerr
|
|
}
|
|
}
|
|
if total > 0 && got != total {
|
|
return fmt.Errorf("%s: got %d of %d bytes", url, got, total)
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(dst+".part", dst)
|
|
}
|
|
|
|
// SimPowerdown presses the simulated Proxmox's power button and waits for
|
|
// QEMU to exit. It is the unit's ExecStop.
|
|
func SimPowerdown(wait time.Duration) error {
|
|
sock := SimDir + "/monitor.sock"
|
|
c, err := net.DialTimeout("unix", sock, 5*time.Second)
|
|
if err != nil {
|
|
return nil // not running, or already gone
|
|
}
|
|
if _, err := c.Write([]byte("system_powerdown\n")); err != nil {
|
|
c.Close()
|
|
return err
|
|
}
|
|
// The monitor closes when QEMU exits; reading until then is the wait.
|
|
_ = c.SetReadDeadline(time.Now().Add(wait))
|
|
_, _ = io.Copy(io.Discard, c)
|
|
c.Close()
|
|
return nil
|
|
}
|
|
|
|
// tokenOrTicket reports whether the API signs root in.
|
|
func tokenOrTicket(ctx context.Context, s Site, pw string) bool {
|
|
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
_, err := PVEWithPassword(cctx, s["PVE_ENDPOINT"], true, "root@pam", pw)
|
|
return err == nil
|
|
}
|
|
|
|
func printable(r rune) rune {
|
|
if r == '\t' || (r >= 0x20 && r < 0x7f) {
|
|
return r
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// simNetwork makes the bridge, gives the host its gateway address, and
|
|
// lets guests out through NAT. A oneshot unit, so it comes back on boot
|
|
// before the VM does.
|
|
func (e *Env) simNetwork(ctx context.Context) error {
|
|
// Refuse a host that already routes the simulated network somewhere.
|
|
if out, _ := e.run(ctx, "", "ip", "-4", "route", "show", SimNet+".0/24"); out != "" && !strings.Contains(out, SimBridge) {
|
|
return fmt.Errorf("this host already routes %s.0/24 (%s); the simulated network would collide", SimNet, out)
|
|
}
|
|
script := fmt.Sprintf(`#!/bin/sh
|
|
# Generated by hpcod setup: the simulated Proxmox's network.
|
|
set -e
|
|
ip link show %[1]s >/dev/null 2>&1 || ip link add %[1]s type bridge
|
|
ip addr replace %[2]s/24 dev %[1]s
|
|
ip link set %[1]s up
|
|
ip tuntap add %[3]s mode tap 2>/dev/null || true
|
|
ip link set %[3]s master %[1]s
|
|
ip link set %[3]s up
|
|
sysctl -qw net.ipv4.ip_forward=1
|
|
# Chain names are plain words on purpose: "fwd" is an nft keyword, and a
|
|
# chain called that is a syntax error.
|
|
nft -f - <<'EOF'
|
|
table ip hpcod_sim
|
|
delete table ip hpcod_sim
|
|
table ip hpcod_sim {
|
|
chain nat_out { type nat hook postrouting priority 100; ip saddr %[4]s.0/24 oifname != "%[1]s" masquerade; }
|
|
chain pass_through { type filter hook forward priority 0; iifname "%[1]s" accept; oifname "%[1]s" ct state established,related accept; }
|
|
}
|
|
EOF
|
|
# A DROP policy in the legacy FORWARD chain (Docker sets one) is final
|
|
# whatever nftables accepts, so the guests would reach nothing.
|
|
if command -v iptables >/dev/null && iptables -S FORWARD 2>/dev/null | grep -q '^-P FORWARD DROP'; then
|
|
iptables -C FORWARD -i %[1]s -j ACCEPT 2>/dev/null || iptables -I FORWARD -i %[1]s -j ACCEPT
|
|
iptables -C FORWARD -o %[1]s -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || \
|
|
iptables -I FORWARD -o %[1]s -m state --state RELATED,ESTABLISHED -j ACCEPT
|
|
fi
|
|
`, SimBridge, SimGateway, SimTap, SimNet)
|
|
if err := e.writeFile(SimDir+"/network.sh", []byte(script), 0o755, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
unit := fmt.Sprintf(`# Generated by hpcod setup.
|
|
[Unit]
|
|
Description=hpcod simulated Proxmox: bridge and NAT
|
|
Before=%s.service
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=yes
|
|
ExecStart=%s/network.sh
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
`, SimUnit, SimDir)
|
|
if err := e.writeFile("/etc/systemd/system/"+SimNetUnit+".service", []byte(unit), 0o644, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "daemon-reload"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "enable", SimNetUnit); err != nil {
|
|
return err
|
|
}
|
|
// Applied by running the script, which is safe to repeat, rather than
|
|
// by restarting the unit: a running simulation is attached to the tap
|
|
// and must not notice.
|
|
if out, err := e.run(ctx, "", e.P(SimDir+"/network.sh")); err != nil {
|
|
return fmt.Errorf("the simulated network did not come up: %w\n%s", err, out)
|
|
}
|
|
// Read back: the host has the gateway address on the bridge.
|
|
out, _ := e.run(ctx, "", "ip", "-4", "-o", "addr", "show", "dev", SimBridge)
|
|
if !strings.Contains(out, SimGateway+"/24") {
|
|
return fmt.Errorf("%s does not carry %s after setup: %s", SimBridge, SimGateway, out)
|
|
}
|
|
e.Logf("bridge %s at %s/24, NAT to the host's uplink", SimBridge, SimGateway)
|
|
return nil
|
|
}
|
|
|
|
// simMAC is a fixed, locally administered MAC: the guest's network config
|
|
// matches on it, and a new one per boot would leave it unconfigured.
|
|
const simMAC = "52:54:00:99:00:02"
|
|
|
|
// simSeed writes the cloud-init seed that turns the Debian image into
|
|
// Proxmox VE on first boot.
|
|
func (e *Env) simSeed(ctx context.Context, pw string) error {
|
|
dir := SimDir + "/seed"
|
|
if err := e.mkdir(dir, 0o700, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
pub, err := e.PublicKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
iid := make([]byte, 6)
|
|
_, _ = rand.Read(iid)
|
|
files := map[string]string{
|
|
"meta-data": fmt.Sprintf("instance-id: hpcod-pvesim-%x\nlocal-hostname: %s\n", iid, SimNode),
|
|
"network-config": fmt.Sprintf(`version: 2
|
|
ethernets:
|
|
uplink:
|
|
match: {macaddress: "%s"}
|
|
addresses: [%s/24]
|
|
routes: [{to: default, via: %s}]
|
|
nameservers: {addresses: [%s]}
|
|
`, simMAC, SimAddress, SimGateway, strings.Join(strings.Fields(e.Site["NAMESERVERS"]), ", ")),
|
|
"user-data": simUserData(pw, pub, e.Site["NAMESERVERS"]),
|
|
}
|
|
for name, body := range files {
|
|
if err := e.writeFile(dir+"/"+name, []byte(body), 0o600, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
iso := SimDir + "/seed.iso"
|
|
if _, err := e.run(ctx, "", "genisoimage", "-quiet", "-output", iso, "-volid", "cidata", "-joliet", "-rock",
|
|
filepath.Join(dir, "user-data"), filepath.Join(dir, "meta-data"), filepath.Join(dir, "network-config")); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// simUserData installs Proxmox VE on Debian 13 the way Proxmox documents
|
|
// it, then replaces the cloud image's networking with the bridge Proxmox
|
|
// expects, and reboots into its kernel. Progress lines start with
|
|
// "hpcod-sim:" on the console, which setup follows.
|
|
func simUserData(pw, pub, nameservers string) string {
|
|
return fmt.Sprintf(`#cloud-config
|
|
hostname: %[4]s
|
|
manage_etc_hosts: false
|
|
disable_root: false
|
|
ssh_pwauth: true
|
|
chpasswd:
|
|
expire: false
|
|
users:
|
|
- {name: root, password: "%[1]s", type: text}
|
|
# Root signs in with its password, as on a Proxmox installed from its ISO.
|
|
# No key is put here: setup installs its own over that password, which is
|
|
# the path a real first install takes.
|
|
write_files:
|
|
- path: /etc/ssh/sshd_config.d/10-hpcod-root.conf
|
|
content: |
|
|
PermitRootLogin yes
|
|
PasswordAuthentication yes
|
|
- path: /root/install-pve.sh
|
|
permissions: "0755"
|
|
content: |
|
|
#!/bin/bash
|
|
set -euxo pipefail
|
|
say() { echo "hpcod-sim: $*" > /dev/ttyS0; }
|
|
exec > /var/log/install-pve.log 2>&1
|
|
trap 'say "FAILED at line $LINENO; see /var/log/install-pve.log"' ERR
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
nic=$(ip -o link | awk -F': ' '/link\/ether %[5]s/{print $2; exit}')
|
|
printf '127.0.0.1 localhost\n%[3]s %[4]s.hpcod.sim %[4]s\n' > /etc/hosts
|
|
say "adding the Proxmox VE repository"
|
|
curl -fsSL https://enterprise.proxmox.com/debian/proxmox-archive-keyring-trixie.gpg \
|
|
-o /usr/share/keyrings/proxmox-archive-keyring.gpg
|
|
printf 'Types: deb\nURIs: http://download.proxmox.com/debian/pve\nSuites: trixie\nComponents: pve-no-subscription\nSigned-By: /usr/share/keyrings/proxmox-archive-keyring.gpg\n' \
|
|
> /etc/apt/sources.list.d/pve-install-repo.sources
|
|
echo "postfix postfix/main_mailer_type select No configuration" | debconf-set-selections
|
|
# Proxmox replaces Debian's grub-pc, whose postinst asks which disk to
|
|
# install to; unanswered, the install fails at the kernel. Answered
|
|
# with the disk the root filesystem is on.
|
|
rootdisk=/dev/$(lsblk -no PKNAME "$(findmnt -no SOURCE /)" | head -1)
|
|
echo "grub-pc grub-pc/install_devices multiselect $rootdisk" | debconf-set-selections
|
|
apt-get -o Acquire::Retries=3 update -q
|
|
say "upgrading Debian"
|
|
apt-get -o Acquire::Retries=3 full-upgrade -y -q
|
|
say "installing the Proxmox kernel and Proxmox VE (the long part)"
|
|
apt-get -o Acquire::Retries=3 install -y -q proxmox-default-kernel
|
|
apt-get -o Acquire::Retries=3 install -y -q proxmox-ve postfix open-iscsi chrony ifupdown2
|
|
apt-get remove -y -q os-prober || true
|
|
apt-get remove -y -q 'linux-image-*cloud*' || true
|
|
update-grub
|
|
say "configuring vmbr0 on $nic"
|
|
rm -f /etc/netplan/*.yaml
|
|
echo 'network: {config: disabled}' > /etc/cloud/cloud.cfg.d/99-hpcod-no-network.cfg
|
|
systemctl disable systemd-networkd.service systemd-networkd.socket 2>/dev/null || true
|
|
cat > /etc/network/interfaces <<EOF
|
|
auto lo
|
|
iface lo inet loopback
|
|
|
|
iface $nic inet manual
|
|
|
|
auto vmbr0
|
|
iface vmbr0 inet static
|
|
address %[3]s/24
|
|
gateway %[6]s
|
|
bridge-ports $nic
|
|
bridge-stp off
|
|
bridge-fd 0
|
|
|
|
source /etc/network/interfaces.d/*
|
|
EOF
|
|
sed -i 's/^ //' /etc/network/interfaces
|
|
# Proxmox leaves a pending interfaces.new from its install, and applies
|
|
# it over this file at boot: the bridge would be gone, and the host
|
|
# with it. Removed, then checked.
|
|
rm -f /etc/network/interfaces.new
|
|
grep -q '^iface vmbr0 inet static' /etc/network/interfaces || { say "FAILED: vmbr0 is not in /etc/network/interfaces"; exit 1; }
|
|
# A file of its own, not the cloud image's link into systemd-resolved:
|
|
# written through that link it lands in /run and is gone after the
|
|
# reboot, and Proxmox then cannot resolve anything it downloads.
|
|
systemctl disable --now systemd-resolved 2>/dev/null || true
|
|
rm -f /etc/resolv.conf
|
|
printf 'nameserver %%s\n' %[7]s > /etc/resolv.conf
|
|
grep -q '^nameserver' /etc/resolv.conf && [ ! -L /etc/resolv.conf ] || { say "FAILED: /etc/resolv.conf"; exit 1; }
|
|
say "Proxmox VE installed; rebooting into it"
|
|
touch /root/.pve-installed
|
|
systemctl reboot
|
|
runcmd:
|
|
- [systemctl, restart, ssh]
|
|
- [bash, -c, "test -e /root/.pve-installed || /root/install-pve.sh"]
|
|
`, pw, pub, SimAddress, SimNode, simMAC, SimGateway, nameservers)
|
|
}
|
|
|
|
// simUnit is the QEMU process, as a service.
|
|
func simUnit(s Site) string {
|
|
return fmt.Sprintf(`# Generated by hpcod setup: the simulated Proxmox.
|
|
[Unit]
|
|
Description=hpcod simulated Proxmox VE (QEMU)
|
|
# Wants, not Requires: with Requires, restarting the network unit restarts
|
|
# this one too - which, for a VM, is pulling its plug.
|
|
Wants=%[1]s.service
|
|
After=%[1]s.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStart=/usr/bin/qemu-system-x86_64 -name %[2]s -machine q35,accel=kvm -cpu host \
|
|
-smp %[3]s -m %[4]sG \
|
|
-drive file=%[5]s/pve.qcow2,if=virtio,cache=none,discard=unmap \
|
|
-drive file=%[5]s/seed.iso,media=cdrom,readonly=on \
|
|
-netdev tap,id=n0,ifname=%[6]s,script=no,downscript=no \
|
|
-device virtio-net-pci,netdev=n0,mac=%[7]s \
|
|
-display none -serial file:%[5]s/console.log \
|
|
-monitor unix:%[5]s/monitor.sock,server=on,wait=off
|
|
# Stopped the way a machine is shut down: an ACPI power button press, then
|
|
# a wait for the guest to finish. SIGTERM to QEMU is a power cut, and a file
|
|
# the guest had written but not yet flushed comes back empty - which is how
|
|
# resolv.conf was lost.
|
|
ExecStop=%[8]s pvesim powerdown
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
TimeoutStopSec=200
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
`, SimNetUnit, SimNode, s["SIM_CORES"], s["SIM_MEMORY_GB"], SimDir, SimTap, simMAC, BinPath)
|
|
}
|
|
|
|
// dialSSHPassword reaches a host as root with a password.
|
|
func dialSSHPassword(ctx context.Context, host, pw string) (*ssh.Client, error) {
|
|
return dialSSH(ctx, host, "root", sshPassword(pw)...)
|
|
}
|