Files
rollcall/cmd/rollcall/osimages.go
T
Ilker ManapandClaude Fable 5.1 0333c1e486
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Successful in 27s
Web UI redesigned for datacenter fleets
Sidebar navigation (Overview, Machines, Jobs, Tags, Profiles, OS images); machine list with
state/tag chips, search, sort, paging and multi-select bulk actions (power, deploy, assign,
tag, re-commission, release, boot device); machine detail with tabs (overview, hardware,
network, BMC, jobs, events, raw) and dialog-driven actions; fleet-wide /api/v1/events and
/api/v1/jobs; ?token= sign-in; GPU count in summaries counts real accelerators only;
OS catalog ignores dot-directories.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-27 14:48:29 +01:00

695 lines
20 KiB
Go

package main
// OS image catalog for disk-based deployment.
//
// <os_dir>/<name>/<version>/disk.raw.zst zstd-compressed raw disk image
// <os_dir>/<name>/<version>/meta.json OSMeta
// <os_dir>/<name>/current -> <version> symlink flipped atomically by sync/promote
//
// Sources (upstream "latest" URLs + vendor checksum files) live in
// os_sources (default /etc/rollcall/os-sources.json). `rollcall os sync`
// downloads only when the vendor checksum changed, verifies it, converts
// to raw with qemu-img, compresses with zstd and prunes old versions.
import (
"bufio"
"context"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"time"
"github.com/klauspost/compress/zstd"
)
type OSSource struct {
Name string `json:"name"`
URL string `json:"url"`
ChecksumURL string `json:"checksum_url"`
Checksum string `json:"checksum"` // sha256 | sha512 (type of the vendor checksum file)
Family string `json:"family"` // debian | rhel (informational)
Description string `json:"description"`
Track string `json:"track"` // latest (default): new versions become current; pinned: only on promote
Keep int `json:"keep"` // versions to keep (default 2)
}
type OSMeta struct {
Name string `json:"name"`
Version string `json:"version"`
Family string `json:"family"`
Description string `json:"description"`
SourceURL string `json:"source_url"`
SourceSHA256 string `json:"source_sha256"`
SourceSHA512 string `json:"source_sha512"`
SourceModified string `json:"source_modified"`
SourceFormat string `json:"source_format"`
SHA256 string `json:"sha256"` // of disk.raw.zst
Size int64 `json:"size"` // compressed bytes
RawSize int64 `json:"raw_size"` // virtual disk size the image needs
Imported time.Time `json:"imported"`
Current bool `json:"current,omitempty"`
}
type OSImage struct {
Name string `json:"name"`
Current *OSMeta `json:"current"`
Versions []OSMeta `json:"versions"`
Source *OSSource `json:"source"`
Warning string `json:"warning,omitempty"`
}
func loadSources(p string) ([]OSSource, error) {
b, err := os.ReadFile(p)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, err
}
var src []OSSource
if err := json.Unmarshal(b, &src); err != nil {
return nil, fmt.Errorf("%s: %w", p, err)
}
for i := range src {
if src[i].Track == "" {
src[i].Track = "latest"
}
if src[i].Keep <= 0 {
src[i].Keep = 2
}
if src[i].Checksum == "" {
src[i].Checksum = "sha256"
}
if strings.ContainsAny(src[i].Name, "/ \t") || src[i].Name == "" {
return nil, fmt.Errorf("%s: bad source name %q", p, src[i].Name)
}
}
return src, nil
}
var versionRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
func osVersions(dir, name string) ([]OSMeta, string) {
ents, err := os.ReadDir(filepath.Join(dir, name))
if err != nil {
return nil, ""
}
cur, _ := os.Readlink(filepath.Join(dir, name, "current"))
cur = path.Base(cur)
var out []OSMeta
for _, e := range ents {
if !e.IsDir() || !versionRe.MatchString(e.Name()) || strings.HasSuffix(e.Name(), ".tmp") {
continue
}
b, err := os.ReadFile(filepath.Join(dir, name, e.Name(), "meta.json"))
if err != nil {
continue
}
var m OSMeta
if json.Unmarshal(b, &m) != nil {
continue
}
m.Name, m.Version = name, e.Name()
m.Current = e.Name() == cur
out = append(out, m)
}
sort.Slice(out, func(i, j int) bool { return out[i].Imported.After(out[j].Imported) })
return out, cur
}
func osCatalog(dir, sourcesFile string) ([]OSImage, error) {
src, err := loadSources(sourcesFile)
if err != nil {
return nil, err
}
byName := map[string]*OSSource{}
for i := range src {
byName[src[i].Name] = &src[i]
}
names := map[string]bool{}
for n := range byName {
names[n] = true
}
if ents, err := os.ReadDir(dir); err == nil {
for _, e := range ents {
if e.IsDir() && !strings.HasPrefix(e.Name(), ".") {
names[e.Name()] = true
}
}
}
out := []OSImage{}
for n := range names {
img := OSImage{Name: n, Source: byName[n]}
vs, cur := osVersions(dir, n)
img.Versions = vs
for i := range vs {
if vs[i].Version == cur {
img.Current = &vs[i]
}
}
switch {
case img.Current == nil && len(vs) == 0:
img.Warning = "not imported"
case img.Current == nil:
img.Warning = "no current version (promote one)"
case time.Since(img.Current.Imported) > 45*24*time.Hour:
img.Warning = fmt.Sprintf("current version is %d days old", int(time.Since(img.Current.Imported).Hours()/24))
}
out = append(out, img)
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out, nil
}
// osResolve returns the metadata of name/version ("" or "current" = current).
func osResolve(dir, name, version string) (*OSMeta, error) {
if strings.ContainsAny(name, "/ \t") || name == "" {
return nil, fmt.Errorf("bad os name %q", name)
}
vs, cur := osVersions(dir, name)
if version == "" || version == "current" {
version = cur
if version == "" {
return nil, fmt.Errorf("os %s has no current version", name)
}
}
for i := range vs {
if vs[i].Version == version {
if _, err := os.Stat(filepath.Join(dir, name, version, "disk.raw.zst")); err != nil {
return nil, err
}
return &vs[i], nil
}
}
return nil, fmt.Errorf("os %s version %s: %w", name, version, errNotFound)
}
func osSetCurrent(dir, name, version string) error {
if _, err := os.Stat(filepath.Join(dir, name, version, "disk.raw.zst")); err != nil {
return err
}
link := filepath.Join(dir, name, "current")
tmp := link + ".tmp"
_ = os.Remove(tmp)
if err := os.Symlink(version, tmp); err != nil {
return err
}
return os.Rename(tmp, link)
}
func osRemove(dir, name, version string) error {
if name == "" || version == "" || !versionRe.MatchString(version) {
return fmt.Errorf("name and version required")
}
_, cur := osVersions(dir, name)
if cur == version {
return fmt.Errorf("%s/%s is the current version; promote another one first", name, version)
}
return os.RemoveAll(filepath.Join(dir, name, version))
}
// ---------------------------------------------------------------- import pipeline
type importOpts struct {
Dir string
Name string
Version string // "" = derive
Family string
Description string
SourceURL string
Modified string
SHA256 string // of the source file, if already known
SHA512 string
Log io.Writer
}
func logf(w io.Writer, f string, a ...any) {
if w != nil {
fmt.Fprintf(w, f+"\n", a...)
}
}
// osImportFile converts src (qcow2/raw/vmdk...) into <dir>/<name>/<version>/ and returns the meta.
func osImportFile(ctx context.Context, src string, o importOpts) (*OSMeta, error) {
if !versionRe.MatchString(o.Name) {
return nil, fmt.Errorf("bad name %q", o.Name)
}
info, err := qemuInfo(ctx, src)
if err != nil {
return nil, err
}
if o.SHA256 == "" || o.SHA512 == "" {
logf(o.Log, "hashing %s", src)
if o.SHA256, o.SHA512, err = hashFile(src); err != nil {
return nil, err
}
}
if o.Version == "" {
d := o.Modified
if d == "" {
st, _ := os.Stat(src)
d = st.ModTime().UTC().Format("20060102")
}
o.Version = d + "-" + o.SHA256[:6]
}
if !versionRe.MatchString(o.Version) {
return nil, fmt.Errorf("bad version %q", o.Version)
}
final := filepath.Join(o.Dir, o.Name, o.Version)
if _, err := os.Stat(filepath.Join(final, "meta.json")); err == nil {
return nil, fmt.Errorf("%s/%s already exists", o.Name, o.Version)
}
work := final + ".tmp"
_ = os.RemoveAll(work)
if err := os.MkdirAll(work, 0o755); err != nil {
return nil, err
}
defer os.RemoveAll(work)
raw := src
if info.Format != "raw" {
raw = filepath.Join(work, "disk.raw")
logf(o.Log, "converting %s (%s, %s virtual) to raw", filepath.Base(src), info.Format, human(float64(info.VirtualSize)))
cmd := exec.CommandContext(ctx, "qemu-img", "convert", "-O", "raw", "-S", "4k", src, raw)
if out, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("qemu-img convert: %v: %s", err, strings.TrimSpace(string(out)))
}
}
logf(o.Log, "compressing to disk.raw.zst")
sum, size, err := compressZstd(raw, filepath.Join(work, "disk.raw.zst"))
if err != nil {
return nil, err
}
if raw != src {
_ = os.Remove(raw)
}
m := &OSMeta{Name: o.Name, Version: o.Version, Family: o.Family, Description: o.Description, SourceURL: o.SourceURL,
SourceSHA256: o.SHA256, SourceSHA512: o.SHA512, SourceModified: o.Modified, SourceFormat: info.Format,
SHA256: sum, Size: size, RawSize: info.VirtualSize, Imported: time.Now().UTC()}
mb, _ := json.MarshalIndent(m, "", " ")
if err := os.WriteFile(filepath.Join(work, "meta.json"), mb, 0o644); err != nil {
return nil, err
}
if err := os.Rename(work, final); err != nil {
return nil, err
}
logf(o.Log, "imported %s/%s (%s compressed, %s raw)", o.Name, o.Version, human(float64(size)), human(float64(info.VirtualSize)))
return m, nil
}
type qemuImgInfo struct {
Format string `json:"format"`
VirtualSize int64 `json:"virtual-size"`
}
func qemuInfo(ctx context.Context, p string) (*qemuImgInfo, error) {
out, err := exec.CommandContext(ctx, "qemu-img", "info", "--output=json", p).Output()
if err != nil {
return nil, fmt.Errorf("qemu-img info %s: %w", p, err)
}
var i qemuImgInfo
if err := json.Unmarshal(out, &i); err != nil {
return nil, err
}
return &i, nil
}
func hashFile(p string) (s256, s512 string, err error) {
f, err := os.Open(p)
if err != nil {
return "", "", err
}
defer f.Close()
h1, h2 := sha256.New(), sha512.New()
if _, err := io.Copy(io.MultiWriter(h1, h2), bufio.NewReaderSize(f, 4<<20)); err != nil {
return "", "", err
}
return hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h2.Sum(nil)), nil
}
func compressZstd(src, dst string) (sum string, size int64, err error) {
in, err := os.Open(src)
if err != nil {
return "", 0, err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return "", 0, err
}
defer out.Close()
h := sha256.New()
cw := &countWriter{w: io.MultiWriter(out, h)}
enc, err := zstd.NewWriter(cw, zstd.WithEncoderLevel(zstd.SpeedDefault), zstd.WithEncoderConcurrency(runtime.NumCPU()))
if err != nil {
return "", 0, err
}
if _, err := io.Copy(enc, bufio.NewReaderSize(in, 4<<20)); err != nil {
enc.Close()
return "", 0, err
}
if err := enc.Close(); err != nil {
return "", 0, err
}
if err := out.Sync(); err != nil {
return "", 0, err
}
return hex.EncodeToString(h.Sum(nil)), cw.n, nil
}
type countWriter struct {
w io.Writer
n int64
}
func (c *countWriter) Write(p []byte) (int, error) {
n, err := c.w.Write(p)
c.n += int64(n)
return n, err
}
// ---------------------------------------------------------------- sync
// parseChecksums understands "HASH file", "HASH *file" and "ALGO (file) = HASH".
func parseChecksums(txt string) map[string]string {
out := map[string]string{}
bsd := regexp.MustCompile(`^\s*[A-Za-z0-9]+\s*\((.+)\)\s*=\s*([0-9a-fA-F]+)\s*$`)
gnu := regexp.MustCompile(`^\s*([0-9a-fA-F]{32,128})\s+\*?(.+?)\s*$`)
for _, line := range strings.Split(txt, "\n") {
if m := bsd.FindStringSubmatch(line); m != nil {
out[path.Base(m[1])] = strings.ToLower(m[2])
} else if m := gnu.FindStringSubmatch(line); m != nil {
out[path.Base(m[2])] = strings.ToLower(m[1])
}
}
return out
}
func httpGetText(ctx context.Context, url string) (string, error) {
req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", fmt.Errorf("%s: %s", url, resp.Status)
}
b, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
return string(b), err
}
// upstreamChecksum fetches the vendor checksum for the source's file.
func upstreamChecksum(ctx context.Context, s OSSource) (string, error) {
txt, err := httpGetText(ctx, s.ChecksumURL)
if err != nil {
return "", err
}
sums := parseChecksums(txt)
if v, ok := sums[path.Base(s.URL)]; ok {
return v, nil
}
if len(sums) == 1 {
for _, v := range sums {
return v, nil
}
}
return "", fmt.Errorf("%s: no entry for %s", s.ChecksumURL, path.Base(s.URL))
}
func metaChecksum(m *OSMeta, typ string) string {
if typ == "sha512" {
return m.SourceSHA512
}
return m.SourceSHA256
}
// download streams url to dst while hashing; verifies against want (hex, type typ).
func download(ctx context.Context, url, dst, typ, want string, log io.Writer) (s256, s512, modified string, err error) {
req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", "", "", err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", "", "", fmt.Errorf("%s: %s", url, resp.Status)
}
if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
modified = t.UTC().Format("20060102")
}
f, err := os.Create(dst)
if err != nil {
return "", "", "", err
}
defer f.Close()
h1, h2 := sha256.New(), sha512.New()
pr := &progressReader{r: resp.Body, total: resp.ContentLength, log: log, last: time.Now()}
if _, err := io.Copy(io.MultiWriter(f, h1, h2), pr); err != nil {
return "", "", "", err
}
s256, s512 = hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h2.Sum(nil))
got := s256
if typ == "sha512" {
got = s512
}
if want != "" && got != want {
return "", "", "", fmt.Errorf("%s: %s mismatch: vendor %s, downloaded %s", url, typ, want, got)
}
return s256, s512, modified, nil
}
type progressReader struct {
r io.Reader
n int64
total int64
log io.Writer
last time.Time
}
func (p *progressReader) Read(b []byte) (int, error) {
n, err := p.r.Read(b)
p.n += int64(n)
if p.log != nil && time.Since(p.last) > 10*time.Second {
p.last = time.Now()
if p.total > 0 {
logf(p.log, " %s / %s (%d%%)", human(float64(p.n)), human(float64(p.total)), p.n*100/p.total)
} else {
logf(p.log, " %s", human(float64(p.n)))
}
}
return n, err
}
// osSync refreshes the named sources (all when names is empty).
func osSync(ctx context.Context, cfg *Config, names []string, force bool, log io.Writer) error {
src, err := loadSources(cfg.OSSources)
if err != nil {
return err
}
if len(src) == 0 {
return fmt.Errorf("no sources in %s", cfg.OSSources)
}
want := map[string]bool{}
for _, n := range names {
want[n] = true
}
var failed []string
for _, s := range src {
if len(want) > 0 && !want[s.Name] {
continue
}
if err := syncOne(ctx, cfg, s, force, log); err != nil {
logf(log, "%s: FAILED: %v", s.Name, err)
failed = append(failed, s.Name)
}
}
if len(failed) > 0 {
return fmt.Errorf("failed: %s", strings.Join(failed, ", "))
}
return nil
}
func syncOne(ctx context.Context, cfg *Config, s OSSource, force bool, log io.Writer) error {
vs, cur := osVersions(cfg.OSDir, s.Name)
up, err := upstreamChecksum(ctx, s)
if err != nil {
return err
}
for i := range vs {
if metaChecksum(&vs[i], s.Checksum) == up {
if !force {
logf(log, "%s: up to date (%s, upstream %s unchanged)", s.Name, vs[i].Version, s.Checksum)
if cur == "" && s.Track == "latest" {
return osSetCurrent(cfg.OSDir, s.Name, vs[i].Version)
}
return osPrune(cfg.OSDir, s, log)
}
break
}
}
tmpDir := filepath.Join(cfg.OSDir, ".tmp")
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
return err
}
dl := filepath.Join(tmpDir, s.Name+"-"+path.Base(s.URL))
defer os.Remove(dl)
logf(log, "%s: downloading %s", s.Name, s.URL)
s256, s512, mod, err := download(ctx, s.URL, dl, s.Checksum, up, log)
if err != nil {
return err
}
logf(log, "%s: vendor %s verified", s.Name, s.Checksum)
m, err := osImportFile(ctx, dl, importOpts{Dir: cfg.OSDir, Name: s.Name, Family: s.Family, Description: s.Description,
SourceURL: s.URL, Modified: mod, SHA256: s256, SHA512: s512, Log: log})
if err != nil {
return err
}
if s.Track == "latest" || cur == "" {
if err := osSetCurrent(cfg.OSDir, s.Name, m.Version); err != nil {
return err
}
logf(log, "%s: current -> %s", s.Name, m.Version)
} else {
logf(log, "%s: %s imported but pinned to %s (rollcall os promote %s %s)", s.Name, m.Version, cur, s.Name, m.Version)
}
return osPrune(cfg.OSDir, s, log)
}
// osPrune keeps the newest `keep` versions plus current; nothing younger than a day is removed
// so a deploy job that pinned a version keeps its file.
func osPrune(dir string, s OSSource, log io.Writer) error {
vs, cur := osVersions(dir, s.Name)
kept := 0
for _, v := range vs {
if v.Version == cur || kept < s.Keep {
kept++
continue
}
if time.Since(v.Imported) < 24*time.Hour {
continue
}
logf(log, "%s: pruning %s", s.Name, v.Version)
if err := os.RemoveAll(filepath.Join(dir, s.Name, v.Version)); err != nil {
return err
}
}
return nil
}
// ---------------------------------------------------------------- CLI (local, filesystem side)
func osCmd(args []string) error {
if len(args) < 1 {
usage()
}
cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"))
if err != nil && args[0] != "list" {
return fmt.Errorf("os %s needs the server config: %w", args[0], err)
}
ctx := context.Background()
switch args[0] {
case "list":
if c, cerr := newClient(); cerr == nil {
var imgs []OSImage
if err := c.do("GET", "/api/v1/os", nil, &imgs); err != nil {
return err
}
printOSList(imgs)
return nil
}
imgs, err := osCatalog(cfg.OSDir, cfg.OSSources)
if err != nil {
return err
}
printOSList(imgs)
case "sync":
fs := newFlags("os sync")
force := fs.Bool("force", false, "re-import even when the upstream checksum is unchanged")
_ = fs.Parse(args[1:])
return osSync(ctx, cfg, fs.Args(), *force, os.Stdout)
case "import":
if len(args) < 3 {
usage()
}
fs := newFlags("os import")
fam := fs.String("family", "", "debian | rhel")
desc := fs.String("description", "", "description")
ver := fs.String("version", "", "version label (default <date>-<sha256 prefix>)")
cur := fs.Bool("current", true, "make it the current version")
_ = fs.Parse(args[3:])
m, err := osImportFile(ctx, args[2], importOpts{Dir: cfg.OSDir, Name: args[1], Version: *ver, Family: *fam, Description: *desc,
SourceURL: "file://" + args[2], Log: os.Stdout})
if err != nil {
return err
}
if *cur {
return osSetCurrent(cfg.OSDir, m.Name, m.Version)
}
case "promote":
if len(args) < 3 {
usage()
}
return osSetCurrent(cfg.OSDir, args[1], args[2])
case "rm":
if len(args) < 3 {
usage()
}
return osRemove(cfg.OSDir, args[1], args[2])
default:
usage()
}
return nil
}
func printOSList(imgs []OSImage) {
fmt.Printf("%-18s %-20s %-7s %8s %8s %-6s %-19s %s\n", "OS", "CURRENT", "FAMILY", "ZST", "RAW", "TRACK", "IMPORTED", "VERSIONS / WARNING")
for _, i := range imgs {
cur, fam, zst, raw, imp, track := "-", "", "", "", "", ""
if i.Source != nil {
track, fam = i.Source.Track, i.Source.Family
}
if i.Current != nil {
cur, zst, raw = i.Current.Version, human(float64(i.Current.Size)), human(float64(i.Current.RawSize))
imp = i.Current.Imported.Local().Format("2006-01-02 15:04")
if fam == "" {
fam = i.Current.Family
}
}
var vs []string
for _, v := range i.Versions {
vs = append(vs, v.Version)
}
extra := strconv.Itoa(len(i.Versions)) + " (" + strings.Join(vs, ", ") + ")"
if i.Warning != "" {
extra += " ! " + i.Warning
}
fmt.Printf("%-18s %-20s %-7s %8s %8s %-6s %-19s %s\n", i.Name, cur, fam, zst, raw, track, imp, extra)
}
}
// ---------------------------------------------------------------- HTTP
func (a *App) handleOS(w http.ResponseWriter, r *http.Request) {
imgs, err := osCatalog(a.cfg.OSDir, a.cfg.OSSources)
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, imgs)
}