Offline sites and Kubernetes: llmgw bundle create, import and trust, offline.mirror, server init --bundle; a Helm chart for the control plane

A bundle carries what a cluster would download — llmgw's release with its
signed checksums, every llama.cpp build per architecture with GitHub's
checksums, model files pinned and checked against Hugging Face's — into a
directory or a tar whose manifest sha256 is carried apart. Import checks
everything, the release against the signing key, before anything is put in
the mirror. With offline.mirror set, nodes install llama.cpp from the
mirror against checksums the control plane holds and sends over the agent
stream; releases default to the mirror for node add, ctl upgrade and agent
upgrades. server init and the setup form install from a bundle.

The Helm chart runs one control plane with SQLite on a claim or an external
PostgreSQL, Secrets for the agent certificate and admin token, Services,
Ingress and ServiceMonitor; deploy/container/Dockerfile builds its image.
GPU nodes stay outside Kubernetes, as docs/kubernetes.md explains.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:13:39 +02:00
1 parent e59ff52932
commit bae16426a2
51 files changed
+4177 -63

No files matched your search

+213
View File
@@ -0,0 +1,213 @@
// Package bundle carries what a cluster would otherwise fetch from the
// internet to a site that has no way to it.
//
// A bundle is made on a machine with internet access: llmgw's own release
// binaries with their signed checksums, the engine builds the nodes install
// with the checksums their publisher gave, and model files with the
// checksums their registry gave — each checked as it arrives, exactly as a
// node online would check it. Its manifest lists every file with its sha256,
// and the manifest's own sha256 is what the person carrying the bundle
// compares at the other end.
//
// Imported, the files go into the cluster's mirror: releases and engines
// under _llmgw/, which no model can be called, models where the mirror keeps
// models. The engines' checksums go to the control plane, which tells each
// node what every file it fetches from the mirror must be. The mirror serves
// plain HTTP to every node, so nothing a node runs is taken on the mirror's
// word: releases carry the project's signature, engines the checksum the
// control plane holds.
package bundle
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
// ManifestName is the manifest's file name in a bundle, and the first entry
// of a bundle packed as a tar.
const ManifestName = "manifest.json"
// Format is the manifest's layout version. A bundle of a later format is
// refused rather than half understood.
const Format = 1
// Prefix is where a mirror keeps what is not a model: _llmgw/releases and
// _llmgw/engines. A model name may not start with an underscore here, so
// none can be called this.
const Prefix = "_llmgw"
// Manifest says what a bundle holds.
type Manifest struct {
Format int `json:"format"`
Created time.Time `json:"created"`
// MadeBy is the llmgw version that made it.
MadeBy string `json:"made_by"`
Releases []Release `json:"releases,omitempty"`
Engines []Engine `json:"engines,omitempty"`
Models []Model `json:"models,omitempty"`
// Files are every file in the bundle but the manifest, with their
// checksums: what an import checks each one against.
Files []File `json:"files"`
}
// Release is llmgw's own release, for some architectures.
type Release struct {
Version string `json:"version"`
Arches []string `json:"arches"`
}
// Engine is an engine version's builds, for some architectures.
type Engine struct {
Engine string `json:"engine"`
Version string `json:"version"`
Arches []string `json:"arches"`
Variants []string `json:"variants"`
}
// Model is a model's files, as the mirror will hold them.
type Model struct {
Name string `json:"name"`
Version string `json:"version"`
// Source is what it was asked for as: hf:owner/repo/file, or a URL.
Source string `json:"source"`
// Revision is the registry's commit it was taken at, when it has one.
Revision string `json:"revision,omitempty"`
}
// File is one file of a bundle.
type File struct {
// Path is where it is in the bundle: releases/<version>/<file>,
// engines/<engine>/<version>/<file> or models/<model>/<version>/<file>.
Path string `json:"path"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
// Source is where it was downloaded from.
Source string `json:"source,omitempty"`
}
// Kinds of file, by the first element of their path.
const (
kindRelease = "releases"
kindEngine = "engines"
kindModel = "models"
)
// Digest is a manifest's sha256: what `bundle create` prints and an import
// is given to check the bundle by.
func Digest(raw []byte) string {
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// parseManifest reads a manifest and checks every path in it, before any
// file is read: a path is where the file is written.
func parseManifest(raw []byte) (Manifest, error) {
var m Manifest
if err := json.Unmarshal(raw, &m); err != nil {
return Manifest{}, fmt.Errorf("the bundle's manifest cannot be read: %w", err)
}
if m.Format != Format {
return Manifest{}, fmt.Errorf("the bundle is of format %d and this llmgw reads format %d: "+
"import it with the llmgw that made it", m.Format, Format)
}
seen := map[string]bool{}
for _, f := range m.Files {
if _, err := Destination(f.Path); err != nil {
return Manifest{}, err
}
if len(f.SHA256) != 64 {
return Manifest{}, fmt.Errorf("the manifest gives %s no sha256", f.Path)
}
if _, err := hex.DecodeString(f.SHA256); err != nil {
return Manifest{}, fmt.Errorf("the manifest's sha256 of %s is not one", f.Path)
}
if seen[f.Path] {
return Manifest{}, fmt.Errorf("the manifest lists %s twice", f.Path)
}
seen[f.Path] = true
}
return m, nil
}
// Destination is where a bundle's file goes in a mirror, relative to its
// root, or why it goes nowhere.
//
// Every path in a bundle has a fixed shape — three or four plain elements
// under one of three directories — because a path is where the import
// writes, as root. Anything else is refused, not cleaned.
func Destination(p string) (string, error) {
parts := strings.Split(p, "/")
bad := func(why string) (string, error) {
return "", fmt.Errorf("the bundle names a file %q, which %s", p, why)
}
if path.Clean(p) != p || strings.HasPrefix(p, "/") {
return bad("is not a plain path")
}
for _, e := range parts {
if err := mirror.ValidName(e); err != nil {
return bad("has an element that is not a plain name")
}
}
switch parts[0] {
case kindRelease:
if len(parts) != 3 || !release.Tag.MatchString(parts[1]) {
return bad("is not releases/<version>/<file>")
}
return Prefix + "/" + p, nil
case kindEngine:
if len(parts) != 4 {
return bad("is not engines/<engine>/<version>/<file>")
}
return Prefix + "/" + p, nil
case kindModel:
if len(parts) != 4 {
return bad("is not models/<model>/<version>/<file>")
}
if strings.HasPrefix(parts[1], "_") {
return bad("names a model starting with an underscore, which the mirror keeps for itself")
}
return strings.Join(parts[1:], "/"), nil
}
return bad("is not under releases/, engines/ or models/")
}
// EngineBase is where an engine version's files are on a mirror whose files
// are served at mirrorURL.
func EngineBase(mirrorURL, engine, version string) string {
return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindEngine + "/" + engine + "/" + version
}
// ReleasesBase is where llmgw's releases are on a mirror whose files are
// served at mirrorURL: what `releases` is on an offline cluster.
func ReleasesBase(mirrorURL string) string {
return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindRelease
}
// ErrNoDigest is an import or trust given no sha256 to check the bundle by.
var ErrNoDigest = errors.New("give the bundle's sha256, as 'llmgw bundle create' printed it, " +
"so what was carried is known to be what was made; or say it is to be taken unchecked")
// checkDigest compares a manifest with the digest it was given. Unchecked
// takes it as it is, said by the caller.
func checkDigest(raw []byte, want string, unchecked bool) error {
want = strings.ToLower(strings.TrimSpace(want))
switch {
case want == "" && unchecked:
return nil
case want == "":
return ErrNoDigest
case Digest(raw) != want:
return fmt.Errorf("the bundle's manifest is %s, not the %s it was given as: "+
"it is not the bundle that was made, or the digest was mistyped", Digest(raw)[:16], want)
}
return nil
}
+365
View File
@@ -0,0 +1,365 @@
package bundle
import (
"archive/tar"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
func hexSum(b []byte) string {
s := sha256.Sum256(b)
return hex.EncodeToString(s[:])
}
// internet is everything a bundle is made from: a release server, GitHub
// and Hugging Face, all fakes.
type internet struct {
srv *httptest.Server
files map[string][]byte
digests map[string]string // GitHub's, by asset
priv string
weights []byte
hfDigest string
}
func newInternet(t *testing.T) *internet {
t.Helper()
pub, priv, err := release.GenerateKey()
if err != nil {
t.Fatal(err)
}
t.Setenv("LLMGW_RELEASE_KEY", pub)
in := &internet{files: map[string][]byte{}, digests: map[string]string{}, priv: priv}
// llmgw v1.2.3 for both architectures, signed.
var sums strings.Builder
for _, arch := range []string{"amd64", "arm64"} {
name := release.AssetName("v1.2.3", "linux", arch)
body := []byte("llmgw for " + arch)
in.files["/releases/v1.2.3/"+name] = body
sums.WriteString(hexSum(body) + " " + name + "\n")
}
in.files["/releases/v1.2.3/checksums.txt"] = []byte(sums.String())
sig, _ := release.Sign(priv, []byte(sums.String()))
in.files["/releases/v1.2.3/"+release.SignatureName] = []byte(sig)
// llama.cpp b1: every build there is for amd64.
builds, err := provision.Builds("llamacpp", "b1", "amd64")
if err != nil {
t.Fatal(err)
}
for _, b := range builds {
for _, a := range b.Assets {
body := []byte("archive " + a)
in.files["/gh/ggml-org/llama.cpp/releases/download/b1/"+a] = body
in.digests[a] = "sha256:" + hexSum(body)
}
}
// A model on Hugging Face, with the checksum the registry publishes.
in.weights = []byte("GGUF weights")
in.hfDigest = hexSum(in.weights)
in.files["/Org/Model-GGUF/resolve/abc123/model-q4.gguf"] = in.weights
in.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/api/repos/ggml-org/llama.cpp/releases/tags/b1"):
var out struct {
Assets []map[string]string `json:"assets"`
}
for name, d := range in.digests {
out.Assets = append(out.Assets, map[string]string{"name": name, "digest": d})
}
_ = json.NewEncoder(w).Encode(out)
case r.URL.Path == "/api/models/Org/Model-GGUF":
_, _ = w.Write([]byte(`{"id":"Org/Model-GGUF","sha":"abc123"}`))
case r.URL.Path == "/api/models/Org/Model-GGUF/tree/abc123":
_ = json.NewEncoder(w).Encode([]map[string]any{
{"type": "file", "path": "model-q4.gguf", "size": 133,
"lfs": map[string]any{"oid": in.hfDigest, "size": len(in.weights)}},
})
default:
b, ok := in.files[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
_, _ = w.Write(b)
}
}))
t.Cleanup(in.srv.Close)
return in
}
func (in *internet) options(out string) CreateOptions {
return CreateOptions{
Out: out, Release: "v1.2.3", Arches: []string{"amd64"},
Releases: in.srv.URL + "/releases",
Engines: []EngineSpec{{Engine: "llamacpp", Version: "b1", Variants: []string{"cpu", "cuda-12.8"}}},
Models: []ModelSpec{{Name: "starter", Version: "2026-10", Source: "hf:Org/Model-GGUF/model-q4.gguf"}},
HTTP: in.srv.Client(),
GitHubAPI: in.srv.URL + "/api", GitHubDownload: in.srv.URL + "/gh", Registry: in.srv.URL,
}
}
func TestABundleCarriesWhatTheClusterWouldFetchAndAMirrorTakesItChecked(t *testing.T) {
in := newInternet(t)
out := filepath.Join(t.TempDir(), "bundle")
m, digest, err := Create(t.Context(), in.options(out))
if err != nil {
t.Fatal(err)
}
var paths []string
for _, f := range m.Files {
paths = append(paths, f.Path)
}
for _, want := range []string{
"releases/v1.2.3/checksums.txt", "releases/v1.2.3/" + release.SignatureName,
"releases/v1.2.3/llmgw-v1.2.3-linux-amd64",
"engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz",
"engines/llamacpp/b1/llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz",
"engines/llamacpp/b1/cudart-llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz",
"models/starter/2026-10/model-q4.gguf",
} {
if !strings.Contains(strings.Join(paths, "\n"), want) {
t.Errorf("the bundle lacks %s:\n%s", want, strings.Join(paths, "\n"))
}
}
if strings.Contains(strings.Join(paths, "\n"), "vulkan") {
t.Error("a build not asked for was carried")
}
// Carried as one file, and imported from it as from the directory.
tarball := out + ".tar"
if err := Pack(out, tarball); err != nil {
t.Fatal(err)
}
for _, from := range []string{tarball, out} {
root := t.TempDir()
res, err := Import(from, ImportOptions{Mirror: root, SHA256: digest})
if err != nil {
t.Fatalf("importing %s: %v", filepath.Base(from), err)
}
if res.Written != len(m.Files) || res.Digest != digest {
t.Errorf("wrote %d of %d files", res.Written, len(m.Files))
}
bin, _ := os.ReadFile(filepath.Join(root, "_llmgw/releases/v1.2.3/llmgw-v1.2.3-linux-amd64"))
if string(bin) != "llmgw for amd64" {
t.Errorf("the release is not where releases are fetched from: %q", bin)
}
if _, err := os.Stat(filepath.Join(root, "_llmgw/engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz")); err != nil {
t.Error(err)
}
// The model is the mirror's like any other, with its checksum.
man, err := mirror.ManifestFor(root, "starter", "2026-10")
if err != nil || len(man.Files) != 1 || man.Files[0].SHA256 != in.hfDigest {
t.Errorf("the model is not offered with its checksum: %+v %v", man, err)
}
// And the mirror lists nothing of llmgw's own as a model.
entries, _ := mirror.List(root)
if len(entries) != 1 {
t.Errorf("the mirror lists %v", entries)
}
// Again: everything is there and the same, so nothing is written.
res, err = Import(from, ImportOptions{Mirror: root, SHA256: digest})
if err != nil || res.Written != 0 || res.Kept != len(m.Files) {
t.Errorf("a second import: %+v %v", res, err)
}
}
// Made again into the same directory, a file already there and right is
// not downloaded: only the checksums are asked for again.
for p := range in.files {
if !strings.HasSuffix(p, "checksums.txt") && !strings.HasSuffix(p, release.SignatureName) {
delete(in.files, p)
}
}
_, again, err := Create(t.Context(), in.options(out))
if err != nil {
t.Fatalf("made again from what was there: %v", err)
}
var m2 Manifest
raw, _ := os.ReadFile(filepath.Join(out, ManifestName))
_ = json.Unmarshal(raw, &m2)
if again != Digest(raw) || len(m2.Files) != len(m.Files) {
t.Errorf("made again: %d files, digest %s", len(m2.Files), again)
}
}
func TestTheControlPlaneTrustsTheBundlesEngineChecksumsAndNodesAreToldThem(t *testing.T) {
in := newInternet(t)
out := filepath.Join(t.TempDir(), "bundle")
_, digest, err := Create(t.Context(), in.options(out))
if err != nil {
t.Fatal(err)
}
file := filepath.Join(t.TempDir(), "offline.sha256")
if _, _, err := Trust(out, file, "", false); err == nil {
t.Fatal("a bundle was trusted with no digest to check it by")
}
if _, _, err := Trust(out, file, strings.Repeat("0", 64), false); err == nil {
t.Fatal("a bundle was trusted against the wrong digest")
}
if _, n, err := Trust(out, file, digest, false); err != nil || n != 3 {
t.Fatalf("trusted %d: %v", n, err)
}
tr := NewTrusted(file)
got := tr.For("llamacpp", "b1")
if got["llama-b1-bin-ubuntu-x64.tar.gz"] != strings.TrimPrefix(in.digests["llama-b1-bin-ubuntu-x64.tar.gz"], "sha256:") {
t.Errorf("the control plane holds %v", got)
}
if len(tr.For("llamacpp", "b2")) != 0 {
t.Error("checksums were given for a version no bundle carried")
}
// A second bundle adds to what is trusted, and the control plane sees
// it without a restart.
m2 := Manifest{Format: Format, Files: []File{{Path: "engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz", Bytes: 1, SHA256: strings.Repeat("b", 64)}}}
if n, err := TrustManifest(m2, file); err != nil || n != 4 {
t.Fatalf("trusted %d: %v", n, err)
}
if tr.For("llamacpp", "b2")["llama-b2-bin-ubuntu-x64.tar.gz"] != strings.Repeat("b", 64) {
t.Error("a bundle trusted while running was not seen")
}
if len(tr.For("llamacpp", "b1")) != 3 {
t.Error("the first bundle's checksums were lost")
}
}
func TestADamagedBundlePutsNothingInTheMirror(t *testing.T) {
in := newInternet(t)
out := filepath.Join(t.TempDir(), "bundle")
_, digest, err := Create(t.Context(), in.options(out))
if err != nil {
t.Fatal(err)
}
// The wrong digest, and none at all, are refused before anything is read.
root := t.TempDir()
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: strings.Repeat("0", 64)}); err == nil {
t.Error("a bundle not matching its digest was imported")
}
if _, err := Import(out, ImportOptions{Mirror: root}); err == nil {
t.Error("a bundle was imported with no digest")
}
if _, err := Import(out, ImportOptions{Mirror: root, Unchecked: true}); err != nil {
t.Errorf("a bundle taken unchecked, as asked: %v", err)
}
// An engine changed on the way.
engine := filepath.Join(out, "engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz")
good, _ := os.ReadFile(engine)
_ = os.WriteFile(engine, []byte("swapped"), 0o644)
root = t.TempDir()
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "damaged") {
t.Errorf("a changed engine: %v", err)
}
if entries := walk(t, root); len(entries) > 0 {
t.Errorf("a damaged bundle left %v in the mirror", entries)
}
_ = os.WriteFile(engine, good, 0o644)
// A release whose checksums the project did not sign, with a manifest
// made to agree: what the manifest says is not enough for a binary.
_, other, _ := release.GenerateKey()
sums, _ := os.ReadFile(filepath.Join(out, "releases/v1.2.3/checksums.txt"))
forged, _ := release.Sign(other, sums)
_ = os.WriteFile(filepath.Join(out, "releases/v1.2.3/"+release.SignatureName), []byte(forged), 0o644)
raw := remanifest(t, out)
root = t.TempDir()
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: Digest(raw)}); err == nil || !strings.Contains(err.Error(), "not signed") {
t.Errorf("a release not signed by the project: %v", err)
}
if entries := walk(t, root); len(entries) > 0 {
t.Errorf("an unsigned release left %v in the mirror", entries)
}
}
func TestAVersionAlreadyInTheMirrorIsNeverReplaced(t *testing.T) {
in := newInternet(t)
out := filepath.Join(t.TempDir(), "bundle")
_, digest, err := Create(t.Context(), in.options(out))
if err != nil {
t.Fatal(err)
}
root := t.TempDir()
_ = os.MkdirAll(filepath.Join(root, "starter", "2026-10"), 0o755)
_ = os.WriteFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf"), []byte("other weights"), 0o644)
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "never replaced") {
t.Fatalf("a different file under the same version: %v", err)
}
if b, _ := os.ReadFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf")); string(b) != "other weights" {
t.Error("the mirror's file was replaced")
}
}
func TestABundleCannotWriteOutsideTheMirror(t *testing.T) {
for _, p := range []string{
"../etc/cron.d/x", "models/../../x/y/z", "models/a/b/../c", "/abs/olute/path/x",
"engines/llamacpp/b1", "releases/latest/llmgw", "models/_llmgw/releases/x",
"models/.hidden/v/f", "other/a/b/c",
} {
if _, err := Destination(p); err == nil {
t.Errorf("%q was given a destination", p)
}
}
if d, err := Destination("models/m/v/f.gguf"); err != nil || d != "m/v/f.gguf" {
t.Errorf("a model: %q %v", d, err)
}
// A tar carrying a file its manifest does not list is refused.
dir := t.TempDir()
raw, _ := json.Marshal(Manifest{Format: Format, Files: []File{}})
tarball := filepath.Join(dir, "b.tar")
f, _ := os.Create(tarball)
tw := tar.NewWriter(f)
_ = tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), Typeflag: tar.TypeReg})
_, _ = tw.Write(raw)
_ = tw.WriteHeader(&tar.Header{Name: "models/m/v/extra", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
_, _ = tw.Write([]byte("x"))
_ = tw.Close()
_ = f.Close()
if _, err := Import(tarball, ImportOptions{Mirror: t.TempDir(), SHA256: Digest(raw)}); err == nil {
t.Error("a file the manifest does not list was imported")
}
}
// remanifest rewrites a bundle's manifest to agree with its files, as
// someone changing a bundle on the way would.
func remanifest(t *testing.T, dir string) []byte {
t.Helper()
raw, _ := os.ReadFile(filepath.Join(dir, ManifestName))
var m Manifest
_ = json.Unmarshal(raw, &m)
for i, f := range m.Files {
b, _ := os.ReadFile(filepath.Join(dir, f.Path))
m.Files[i].SHA256, m.Files[i].Bytes = hexSum(b), int64(len(b))
}
raw, _ = json.MarshalIndent(m, "", " ")
_ = os.WriteFile(filepath.Join(dir, ManifestName), raw, 0o644)
return raw
}
func walk(t *testing.T, root string) []string {
t.Helper()
var out []string
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() {
out = append(out, p)
}
return nil
})
return out
}
+6
View File
@@ -0,0 +1,6 @@
//go:build !unix
package bundle
// chownLike does nothing where files have no Unix owner.
func chownLike(string, string) {}
+26
View File
@@ -0,0 +1,26 @@
//go:build unix
package bundle
import (
"os"
"syscall"
)
// chownLike gives p the owner of root, when this runs as root: the mirror
// runs as its own user, and a version it cannot remove is one `mirror
// remove` and the admin page fail on.
func chownLike(root, p string) {
if os.Geteuid() != 0 {
return
}
info, err := os.Stat(root)
if err != nil {
return
}
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return
}
_ = os.Lchown(p, int(st.Uid), int(st.Gid))
}
+439
View File
@@ -0,0 +1,439 @@
package bundle
import (
"archive/tar"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// EngineSpec is an engine version to carry, and which of its builds.
type EngineSpec struct {
Engine, Version string
// Variants are the builds to carry ("cpu", "cuda-13.3"); empty is every
// one, which is what a site whose nodes are not all known wants.
Variants []string
}
// ModelSpec is a model file to carry, and where the mirror will hold it.
type ModelSpec struct {
Name, Version string
// Source is hf:owner/repo/path/in/repo, or an http(s) URL.
Source string
}
// CreateOptions say what goes into a bundle.
type CreateOptions struct {
// Out is the directory the bundle is made in. A bundle made there
// before is added to: a file already there with the right checksum is
// not downloaded again.
Out string
// Release is llmgw's release to carry, v0.10.4; empty carries none.
Release string
// Arches are the architectures, of the releases and of the engines:
// amd64, arm64.
Arches []string
// Releases is where release files are published.
Releases string
Engines []EngineSpec
Models []ModelSpec
// HTTP fetches everything; nil is a client with no overall timeout,
// because a model file is tens of gigabytes.
HTTP *http.Client
// GitHubAPI, GitHubDownload and Registry default to GitHub and Hugging
// Face; tests point them at fakes.
GitHubAPI, GitHubDownload, Registry string
// Log hears what is happening, a line at a time.
Log func(format string, args ...any)
}
// Create downloads everything a bundle is asked to carry, checking each
// file as a node online would, and writes its manifest. It returns the
// manifest and its digest: the value to carry to the site separately and
// give to the import.
func Create(ctx context.Context, o CreateOptions) (Manifest, string, error) {
c := &creator{o: o, client: o.HTTP}
if c.client == nil {
c.client = &http.Client{}
}
if len(o.Arches) == 0 {
return Manifest{}, "", errors.New("give at least one architecture: amd64, arm64")
}
for _, a := range o.Arches {
if a != "amd64" && a != "arm64" {
return Manifest{}, "", fmt.Errorf("%q is not an architecture llmgw is built for: amd64 or arm64", a)
}
}
if err := os.MkdirAll(o.Out, 0o755); err != nil {
return Manifest{}, "", err
}
c.m = Manifest{Format: Format, Created: time.Now().UTC().Truncate(time.Second), MadeBy: version.Version}
if o.Release != "" {
if err := c.release(ctx); err != nil {
return Manifest{}, "", err
}
}
for _, e := range o.Engines {
if err := c.engine(ctx, e); err != nil {
return Manifest{}, "", err
}
}
for _, m := range o.Models {
if err := c.model(ctx, m); err != nil {
return Manifest{}, "", err
}
}
raw, err := json.MarshalIndent(c.m, "", " ")
if err != nil {
return Manifest{}, "", err
}
raw = append(raw, '\n')
if err := os.WriteFile(filepath.Join(o.Out, ManifestName), raw, 0o644); err != nil { //nolint:gosec // a list of checksums
return Manifest{}, "", err
}
return c.m, Digest(raw), nil
}
type creator struct {
o CreateOptions
client *http.Client
m Manifest
}
func (c *creator) log(format string, args ...any) {
if c.o.Log != nil {
c.o.Log(format, args...)
}
}
// release carries llmgw's release: its checksums and their signature,
// checked against the key this build trusts, and a binary per
// architecture checked against them.
func (c *creator) release(ctx context.Context) error {
v := c.o.Release
base := c.o.Releases
if base == "" {
base = release.DefaultBase
}
c.log("llmgw %s: its checksums and their signature", v)
sums, err := release.FetchChecksums(ctx, c.client, base, v)
if err != nil {
return fmt.Errorf("llmgw %s: %w", v, err)
}
dir := kindRelease + "/" + v + "/"
if err := c.writeSmall(dir+"checksums.txt", sums.Text); err != nil {
return err
}
if sums.Signature != "" {
if err := c.writeSmall(dir+release.SignatureName, []byte(sums.Signature)); err != nil {
return err
}
}
root := strings.TrimSuffix(base, "/") + "/" + v + "/"
for _, arch := range c.o.Arches {
asset := release.AssetName(v, "linux", arch)
want, err := sums.For(asset)
if err != nil {
return fmt.Errorf("llmgw %s: %w", v, err)
}
if err := c.fetch(ctx, root+asset, dir+asset, want); err != nil {
return err
}
}
c.m.Releases = append(c.m.Releases, Release{Version: v, Arches: slices.Clone(c.o.Arches)})
return nil
}
// engine carries an engine version's builds for each architecture, checked
// against the checksums its publisher gives — the same the nodes ask for
// online.
func (c *creator) engine(ctx context.Context, e EngineSpec) error {
in := &provision.Installer{HTTP: c.client, APIBase: c.o.GitHubAPI, DownloadBase: c.o.GitHubDownload}
c.log("%s %s: the checksums its publisher gives", e.Engine, e.Version)
sums, url, err := in.Published(ctx, e.Engine, e.Version)
if err != nil {
return fmt.Errorf("%s %s: %w", e.Engine, e.Version, err)
}
var variants []string
done := map[string]bool{}
for _, arch := range c.o.Arches {
builds, err := provision.Builds(e.Engine, e.Version, arch)
if err != nil {
return err
}
var names []string
for _, b := range builds {
names = append(names, b.Variant)
if len(e.Variants) > 0 && !slices.Contains(e.Variants, b.Variant) {
continue
}
if !slices.Contains(variants, b.Variant) {
variants = append(variants, b.Variant)
}
for _, asset := range b.Assets {
if done[asset] {
continue
}
want, ok := sums[asset]
if !ok {
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be carried checked", e.Engine, e.Version, asset)
}
if err := c.fetch(ctx, url(asset), kindEngine+"/"+e.Engine+"/"+e.Version+"/"+asset, want); err != nil {
return err
}
done[asset] = true
}
}
for _, v := range e.Variants {
if !slices.Contains(names, v) && !anyArchHas(e, v, c.o.Arches) {
return fmt.Errorf("%s has no build %q; it has %s", e.Engine, v, strings.Join(names, ", "))
}
}
}
c.m.Engines = append(c.m.Engines, Engine{Engine: e.Engine, Version: e.Version,
Arches: slices.Clone(c.o.Arches), Variants: variants})
return nil
}
// anyArchHas says whether a variant is a build on any of the architectures:
// CUDA 12.8 is published for x64 alone, and asking for it with arm64 too is
// not a mistake.
func anyArchHas(e EngineSpec, variant string, arches []string) bool {
for _, a := range arches {
bs, _ := provision.Builds(e.Engine, e.Version, a)
for _, b := range bs {
if b.Variant == variant {
return true
}
}
}
return false
}
// model carries a model file, checked against the checksum its registry
// publishes when it publishes one, at the commit the registry had when it
// was asked — so the bundle says exactly which weights it carries.
func (c *creator) model(ctx context.Context, s ModelSpec) error {
for _, n := range []string{s.Name, s.Version} {
if err := mirror.ValidName(n); err != nil {
return fmt.Errorf("model %s/%s: %w", s.Name, s.Version, err)
}
}
if strings.HasPrefix(s.Name, "_") {
return fmt.Errorf("model %q: a name starting with an underscore is the mirror's own", s.Name)
}
var url, want, rev string
switch {
case strings.HasPrefix(s.Source, "hf:"):
parts := strings.SplitN(strings.TrimPrefix(s.Source, "hf:"), "/", 3)
if len(parts) != 3 || parts[2] == "" {
return fmt.Errorf("%s names a repository, not a file: hf:owner/repo/file", s.Source)
}
repoID, file := parts[0]+"/"+parts[1], parts[2]
reg := hf.New(c.o.Registry, time.Minute)
repo, err := reg.Repo(ctx, repoID)
if err != nil {
return fmt.Errorf("%s: %w", s.Source, err)
}
found := false
for _, f := range repo.Files {
if f.Path == file {
want, found = f.SHA256, true
}
}
if !found {
return fmt.Errorf("%s has no file %s at its latest commit", repoID, file)
}
url, rev = repo.DownloadURL(reg.Base(), file), repo.Revision
case strings.HasPrefix(s.Source, "https://"), strings.HasPrefix(s.Source, "http://"):
url = s.Source
default:
return fmt.Errorf("model source %q: hf:owner/repo/file or an http(s) URL", s.Source)
}
name := path.Base(strings.SplitN(url, "?", 2)[0])
if err := mirror.ValidName(name); err != nil {
return fmt.Errorf("%s: %w", s.Source, err)
}
if err := c.fetch(ctx, url, kindModel+"/"+s.Name+"/"+s.Version+"/"+name, want); err != nil {
return err
}
c.m.Models = append(c.m.Models, Model{Name: s.Name, Version: s.Version, Source: s.Source, Revision: rev})
return nil
}
// writeSmall puts a file the bundle makes itself, rather than downloads.
func (c *creator) writeSmall(rel string, b []byte) error {
dst := filepath.Join(c.o.Out, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return err
}
if err := os.WriteFile(dst, b, 0o644); err != nil { //nolint:gosec // published files
return err
}
sum := sha256.Sum256(b)
c.add(File{Path: rel, Bytes: int64(len(b)), SHA256: hex.EncodeToString(sum[:])})
return nil
}
func (c *creator) add(f File) {
for i := range c.m.Files {
if c.m.Files[i].Path == f.Path {
c.m.Files[i] = f
return
}
}
c.m.Files = append(c.m.Files, f)
}
// fetch downloads url into the bundle at rel, checked against want when
// there is one; a file already there that matches is kept.
func (c *creator) fetch(ctx context.Context, url, rel, want string) error {
dst := filepath.Join(c.o.Out, filepath.FromSlash(rel))
if want != "" {
if got, err := fileSHA256(dst); err == nil && got == want {
info, _ := os.Stat(dst)
c.log(" have %s", rel)
c.add(File{Path: rel, Bytes: info.Size(), SHA256: want, Source: url})
return nil
}
}
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := c.client.Do(req)
if err != nil {
return fmt.Errorf("fetching %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("fetching %s: %s", url, resp.Status)
}
if resp.ContentLength > 0 {
c.log(" fetching %s (%d MB)", rel, resp.ContentLength>>20)
} else {
c.log(" fetching %s", rel)
}
part := dst + partSuffix
f, err := os.Create(part) //nolint:gosec // inside the bundle
if err != nil {
return err
}
h := sha256.New()
n, err := io.Copy(io.MultiWriter(f, h), &progressReader{r: resp.Body, total: resp.ContentLength, log: c.log, name: rel})
if cerr := f.Close(); err == nil {
err = cerr
}
if err == nil && resp.ContentLength > 0 && n != resp.ContentLength {
err = fmt.Errorf("got %d bytes of %d", n, resp.ContentLength)
}
if err != nil {
_ = os.Remove(part)
return fmt.Errorf("fetching %s: %w", url, err)
}
got := hex.EncodeToString(h.Sum(nil))
if want != "" && got != want {
_ = os.Remove(part)
return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)", url, got[:12], want[:12])
}
if err := os.Rename(part, dst); err != nil {
return err
}
c.add(File{Path: rel, Bytes: n, SHA256: got, Source: url})
return nil
}
// progressReader says how far a long download has got, every few seconds.
type progressReader struct {
r io.Reader
total int64
done int64
last time.Time
log func(string, ...any)
name string
}
func (p *progressReader) Read(b []byte) (int, error) {
n, err := p.r.Read(b)
p.done += int64(n)
if p.total > 0 && time.Since(p.last) > 10*time.Second {
if !p.last.IsZero() {
p.log(" %d%% of %s", p.done*100/p.total, p.name)
}
p.last = time.Now()
}
return n, err
}
// Pack writes a bundle directory as one tar file, manifest first, so it can
// be carried as one file and imported as a stream. Weights do not compress,
// so it is not compressed.
func Pack(dir, out string) error {
src, m, err := open(dir)
if err != nil {
return err
}
defer src.close() //nolint:errcheck // read only
f, err := os.Create(out + partSuffix) //nolint:gosec // the operator's path
if err != nil {
return err
}
defer os.Remove(out + partSuffix) //nolint:errcheck // gone once renamed
tw := tar.NewWriter(f)
now := time.Now()
raw := src.manifest()
if err := tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), ModTime: now, Typeflag: tar.TypeReg}); err != nil {
f.Close()
return err
}
if _, err := tw.Write(raw); err != nil {
f.Close()
return err
}
for _, mf := range m.Files {
in, err := os.Open(filepath.Join(dir, filepath.FromSlash(mf.Path))) //nolint:gosec // a checked path in the bundle
if err != nil {
f.Close()
return err
}
err = tw.WriteHeader(&tar.Header{Name: mf.Path, Mode: 0o644, Size: mf.Bytes, ModTime: now, Typeflag: tar.TypeReg})
if err == nil {
_, err = io.Copy(tw, in)
}
in.Close()
if err != nil {
f.Close()
return fmt.Errorf("packing %s: %w", mf.Path, err)
}
}
if err := tw.Close(); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
return os.Rename(out+partSuffix, out)
}
+335
View File
@@ -0,0 +1,335 @@
package bundle
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"maps"
"os"
"path"
"path/filepath"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
// ImportOptions say where a bundle goes and how it is known to be the one
// that was made.
type ImportOptions struct {
// Mirror is the mirror's root directory, /srv/models. Empty checks the
// bundle and writes nothing.
Mirror string
// SHA256 is the manifest's digest as `bundle create` printed it.
// Unchecked takes the bundle without one, said by whoever runs it.
SHA256 string
Unchecked bool
// Progress hears each file as it is done: kept is one the mirror
// already had, the same.
Progress func(f File, kept bool)
}
// Result is what an import did.
type Result struct {
Manifest Manifest
// Digest is the manifest's sha256.
Digest string
// Written and Kept count files: put in the mirror, and already there.
Written, Kept int
Bytes int64
}
// partSuffix marks a file still being written; the mirror lists none.
const partSuffix = ".part"
// Import checks every file of a bundle and puts it in a mirror.
//
// Nothing is in place until everything is checked. Each file is written
// beside where it goes, under a name the mirror does not serve, and checked
// against the manifest as it is written; each release's checksums are
// checked against the project's signing key and its binaries against them.
// Only then is each renamed into place — so a bundle damaged anywhere
// leaves the mirror as it was.
//
// A file the mirror already has is kept when it is the same, and refused
// when it is not: a version in the mirror is never replaced, because that
// is what makes going back to it possible.
func Import(p string, o ImportOptions) (Result, error) {
src, m, err := open(p)
if err != nil {
return Result{}, err
}
defer src.close() //nolint:errcheck // read only
res := Result{Manifest: m, Digest: Digest(src.manifest())}
if err := checkDigest(src.manifest(), o.SHA256, o.Unchecked); err != nil {
return res, err
}
byPath := map[string]File{}
dest := map[string]string{}
keep := map[string]bool{}
for _, f := range m.Files {
byPath[f.Path] = f
if o.Mirror == "" {
continue
}
rel, _ := Destination(f.Path) // checked when the manifest was read
dest[f.Path] = filepath.Join(o.Mirror, filepath.FromSlash(rel))
same, err := alreadyHas(dest[f.Path], f)
if err != nil {
return res, err
}
keep[f.Path] = same
}
// Each release's checksums are kept as they pass, to be checked against
// the signing key once the whole release has been read.
small := map[string][]byte{}
var written []string
defer func() {
// Whatever was staged and not put in place goes.
for _, p := range written {
_ = os.Remove(dest[p] + partSuffix)
}
}()
seen := map[string]bool{}
err = src.each(func(name string, r io.Reader) error {
f, ok := byPath[name]
if !ok {
return fmt.Errorf("the bundle holds %s, which its manifest does not list", name)
}
if seen[name] {
return fmt.Errorf("the bundle holds %s twice", name)
}
seen[name] = true
var w io.Writer = io.Discard
var out *os.File
if o.Mirror != "" && !keep[name] {
if err := mkdirsLike(o.Mirror, filepath.Dir(dest[name])); err != nil {
return err
}
out, err = os.OpenFile(dest[name]+partSuffix, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // a checked path under the mirror
if err != nil {
return err
}
written = append(written, name)
w = out
}
if keep[name] {
r = eofReader{}
}
var buf bytes.Buffer
if isReleaseSums(name) {
w = io.MultiWriter(w, &limited{w: &buf, n: 1 << 20})
}
h := sha256.New()
n, err := io.Copy(io.MultiWriter(w, h), r)
if out != nil {
if cerr := out.Close(); err == nil {
err = cerr
}
}
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
if keep[name] {
if isReleaseSums(name) {
b, err := os.ReadFile(dest[name])
if err != nil {
return err
}
small[name] = b
}
res.Kept++
if o.Progress != nil {
o.Progress(f, true)
}
return nil
}
if got := hex.EncodeToString(h.Sum(nil)); got != f.SHA256 || n != f.Bytes {
return fmt.Errorf("%s is not what the bundle was made with (sha256 %s, expected %s): "+
"the bundle was damaged on the way; nothing was put in the mirror", name, got[:12], f.SHA256[:12])
}
if isReleaseSums(name) {
small[name] = buf.Bytes()
}
res.Written++
res.Bytes += n
if o.Progress != nil {
o.Progress(f, false)
}
return nil
})
if err != nil {
return res, err
}
for _, f := range m.Files {
if !seen[f.Path] {
return res, fmt.Errorf("the bundle lacks %s, which its manifest lists", f.Path)
}
}
if err := checkReleases(m, small); err != nil {
return res, err
}
if o.Mirror == "" {
return res, nil
}
for _, name := range written {
d := dest[name]
f := byPath[name]
if strings.HasPrefix(name, kindModel+"/") {
// Beside the file before it is in place, as a download into the
// mirror does, so a node is never offered it without its checksum.
if err := mirror.Record(d, f.SHA256, f.Source); err != nil {
return res, err
}
}
if err := os.Rename(d+partSuffix, d); err != nil {
return res, err
}
chownLike(o.Mirror, d)
if strings.HasPrefix(name, kindModel+"/") {
dir, base := filepath.Split(d)
chownLike(o.Mirror, filepath.Join(dir, "."+base+".sha256"))
chownLike(o.Mirror, filepath.Join(dir, "."+base+".source"))
}
}
written = nil
return res, nil
}
// alreadyHas says whether the mirror has this file already, the same; an
// error when it has a different one there.
func alreadyHas(dst string, f File) (bool, error) {
info, err := os.Stat(dst)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
sum := mirror.Recorded(dst)
if sum == "" || info.Size() != f.Bytes {
if sum, err = fileSHA256(dst); err != nil {
return false, err
}
}
if sum != f.SHA256 || info.Size() != f.Bytes {
return false, fmt.Errorf("the mirror already has %s, and it is not the bundle's: "+
"a version in the mirror is never replaced; give the model another version in a new bundle", dst)
}
return true, nil
}
func isReleaseSums(name string) bool {
if !strings.HasPrefix(name, kindRelease+"/") {
return false
}
b := path.Base(name)
return b == "checksums.txt" || b == release.SignatureName
}
// checkReleases holds each release's binaries to its checksums, and the
// checksums to the project's signing key — as `node add` and an upgrade
// would online. The manifest's own checksums say the bundle arrived as it
// was made; these say what was made is the project's release.
func checkReleases(m Manifest, small map[string][]byte) error {
byVersion := map[string][]File{}
for _, f := range m.Files {
if strings.HasPrefix(f.Path, kindRelease+"/") {
v := strings.Split(f.Path, "/")[1]
byVersion[v] = append(byVersion[v], f)
}
}
for _, v := range slices.Sorted(maps.Keys(byVersion)) {
root := kindRelease + "/" + v + "/"
text, ok := small[root+"checksums.txt"]
if !ok {
return fmt.Errorf("the bundle's llmgw %s has no checksums.txt", v)
}
c := release.Checksums{Text: text, Signature: string(small[root+release.SignatureName])}
if err := c.Verify(); err != nil {
return fmt.Errorf("llmgw %s in the bundle: %w", v, err)
}
for _, f := range byVersion[v] {
name := path.Base(f.Path)
if name == "checksums.txt" || name == release.SignatureName {
continue
}
want, err := c.For(name)
if err != nil {
return fmt.Errorf("llmgw %s in the bundle: %w", v, err)
}
if want != f.SHA256 {
return fmt.Errorf("%s in the bundle is not the release's (the release gives sha256 %s)", f.Path, want[:12])
}
}
}
return nil
}
// mkdirsLike makes dir and every directory above it up to root, each owned
// as root is: the mirror runs as its own user, and removes what it holds.
func mkdirsLike(root, dir string) error {
rel, err := filepath.Rel(root, dir)
if err != nil || strings.HasPrefix(rel, "..") {
return fmt.Errorf("%s is outside the mirror at %s", dir, root)
}
cur := root
for _, e := range strings.Split(rel, string(filepath.Separator)) {
if e == "." {
continue
}
cur = filepath.Join(cur, e)
if _, err := os.Stat(cur); err == nil {
continue
}
if err := os.Mkdir(cur, 0o755); err != nil && !errors.Is(err, os.ErrExist) {
return err
}
chownLike(root, cur)
}
return nil
}
func fileSHA256(p string) (string, error) {
f, err := os.Open(p) //nolint:gosec // a file in the mirror
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// eofReader is a file passed over: a tar moves past it on its own.
type eofReader struct{}
func (eofReader) Read([]byte) (int, error) { return 0, io.EOF }
// limited keeps at most n bytes and quietly drops the rest: a checksums
// file larger than that is refused by its signature, not by a write error.
type limited struct {
w io.Writer
n int64
}
func (l *limited) Write(p []byte) (int, error) {
if l.n <= 0 {
return len(p), nil
}
k := min(int64(len(p)), l.n)
l.n -= k
if _, err := l.w.Write(p[:k]); err != nil {
return 0, err
}
return len(p), nil
}
+157
View File
@@ -0,0 +1,157 @@
package bundle
import (
"archive/tar"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// maxManifest bounds the manifest read into memory.
const maxManifest = 16 << 20
// source is a bundle being read: a directory, or the same packed as a tar
// with the manifest first, read as a stream so tens of gigabytes of weights
// are never unpacked twice.
type source interface {
// manifest is the manifest's bytes, as they are in the bundle.
manifest() []byte
// each hands over every file, in the order the bundle has them. A file
// fn does not read is passed over.
each(fn func(path string, r io.Reader) error) error
close() error
}
// open reads a bundle's manifest, from a directory or a tar.
func open(p string) (source, Manifest, error) {
info, err := os.Stat(p)
if err != nil {
return nil, Manifest{}, fmt.Errorf("the bundle: %w", err)
}
var src source
if info.IsDir() {
src, err = openDir(p)
} else {
src, err = openTar(p)
}
if err != nil {
return nil, Manifest{}, err
}
m, err := parseManifest(src.manifest())
if err != nil {
_ = src.close()
return nil, Manifest{}, err
}
if d, ok := src.(*dirSource); ok {
d.files = m.Files
}
return src, m, nil
}
// ReadManifest reads a bundle's manifest and its digest, without reading
// its files.
func ReadManifest(p string) (Manifest, string, error) {
src, m, err := open(p)
if err != nil {
return Manifest{}, "", err
}
defer src.close() //nolint:errcheck // read only
return m, Digest(src.manifest()), nil
}
type dirSource struct {
dir string
raw []byte
files []File
}
func openDir(dir string) (*dirSource, error) {
raw, err := readLimited(filepath.Join(dir, ManifestName))
if err != nil {
return nil, fmt.Errorf("%s is not a bundle: %w", dir, err)
}
return &dirSource{dir: dir, raw: raw}, nil
}
func (d *dirSource) manifest() []byte { return d.raw }
func (d *dirSource) close() error { return nil }
func (d *dirSource) each(fn func(string, io.Reader) error) error {
for _, f := range d.files {
// The path was checked when the manifest was read: plain names
// under the bundle's three directories.
in, err := os.Open(filepath.Join(d.dir, filepath.FromSlash(f.Path)))
if err != nil {
return fmt.Errorf("the bundle lists %s and does not have it: %w", f.Path, err)
}
err = fn(f.Path, in)
in.Close()
if err != nil {
return err
}
}
return nil
}
type tarSource struct {
f *os.File
tr *tar.Reader
raw []byte
}
func openTar(p string) (*tarSource, error) {
f, err := os.Open(p) //nolint:gosec // the operator's bundle
if err != nil {
return nil, err
}
tr := tar.NewReader(f)
h, err := tr.Next()
if err != nil || h.Name != ManifestName || h.Typeflag != tar.TypeReg {
f.Close()
return nil, fmt.Errorf("%s is not a bundle: its first entry is not %s", p, ManifestName)
}
raw, err := io.ReadAll(io.LimitReader(tr, maxManifest))
if err != nil {
f.Close()
return nil, err
}
return &tarSource{f: f, tr: tr, raw: raw}, nil
}
func (t *tarSource) manifest() []byte { return t.raw }
func (t *tarSource) close() error { return t.f.Close() }
func (t *tarSource) each(fn func(string, io.Reader) error) error {
for {
h, err := t.tr.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("reading the bundle: %w", err)
}
if h.Typeflag == tar.TypeDir {
continue
}
if h.Typeflag != tar.TypeReg {
// Links and devices have no place in a bundle; the files it
// lists are all it holds.
return fmt.Errorf("the bundle holds %s, which is not a plain file", h.Name)
}
if err := fn(strings.TrimPrefix(h.Name, "./"), t.tr); err != nil {
return err
}
}
}
func readLimited(p string) ([]byte, error) {
f, err := os.Open(p) //nolint:gosec // the operator's bundle
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(io.LimitReader(f, maxManifest))
}
+132
View File
@@ -0,0 +1,132 @@
package bundle
import (
"bufio"
"bytes"
"errors"
"fmt"
"maps"
"os"
"path"
"path/filepath"
"slices"
"strings"
"sync"
"time"
)
// Trust records a bundle's engine checksums in the file the control plane
// reads them from, beside any recorded before, and says how many it holds
// now. Only the manifest is read: the checksums are what each node holds
// the files it fetches to, so the manifest must be the one that was made —
// which is what the digest says.
func Trust(p, file, digest string, unchecked bool) (Manifest, int, error) {
src, m, err := open(p)
if err != nil {
return Manifest{}, 0, err
}
defer src.close() //nolint:errcheck // read only
if err := checkDigest(src.manifest(), digest, unchecked); err != nil {
return m, 0, err
}
n, err := TrustManifest(m, file)
return m, n, err
}
// TrustManifest records a manifest's engine checksums, for one already
// checked by an import.
func TrustManifest(m Manifest, file string) (int, error) {
sums, err := readTrusted(file)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return 0, err
}
if sums == nil {
sums = map[string]string{}
}
for _, f := range m.Files {
if strings.HasPrefix(f.Path, kindEngine+"/") {
sums[Prefix+"/"+f.Path] = f.SHA256
}
}
var b strings.Builder
b.WriteString("# Engine files the nodes may install, as the offline bundles they came in\n" +
"# gave them. Kept by 'llmgw bundle import' and 'llmgw bundle trust'.\n")
for _, k := range slices.Sorted(maps.Keys(sums)) {
fmt.Fprintf(&b, "%s %s\n", sums[k], k)
}
if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil {
return 0, err
}
tmp := file + partSuffix
if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { //nolint:gosec // checksums, for anyone to read
return 0, err
}
return len(sums), os.Rename(tmp, file)
}
// readTrusted reads a checksums file: "<sha256> <path>" a line, as
// sha256sum writes them, with paths as the mirror has the files.
func readTrusted(file string) (map[string]string, error) {
b, err := os.ReadFile(file) //nolint:gosec // the control plane's own file
if err != nil {
return nil, err
}
out := map[string]string{}
sc := bufio.NewScanner(bytes.NewReader(b))
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
f := strings.Fields(line)
if len(f) != 2 || len(f[0]) != 64 {
return nil, fmt.Errorf("%s: %q is not a checksum and a file", file, line)
}
out[strings.TrimPrefix(f[1], "*")] = f[0]
}
return out, sc.Err()
}
// Trusted is the engine checksums a control plane holds, read from their
// file again whenever it changes — so a bundle imported while the control
// plane runs is offered to the next node that asks.
type Trusted struct {
file string
mu sync.Mutex
mod time.Time
size int64
sums map[string]string
err error
}
// NewTrusted reads checksums from file as they are asked for.
func NewTrusted(file string) *Trusted { return &Trusted{file: file} }
// For is an engine version's files and their checksums, by file name; none
// when no bundle imported here carried it.
func (t *Trusted) For(engine, version string) map[string]string {
t.mu.Lock()
defer t.mu.Unlock()
if info, err := os.Stat(t.file); err != nil {
t.sums, t.err, t.mod, t.size = nil, err, time.Time{}, 0
} else if !info.ModTime().Equal(t.mod) || info.Size() != t.size {
t.sums, t.err = readTrusted(t.file)
t.mod, t.size = info.ModTime(), info.Size()
}
prefix := Prefix + "/" + kindEngine + "/" + engine + "/" + version + "/"
out := map[string]string{}
for p, sum := range t.sums {
if rest, ok := strings.CutPrefix(p, prefix); ok && path.Base(rest) == rest {
out[rest] = sum
}
}
return out
}
// Err is why the file could not be read, the last time it was.
func (t *Trusted) Err() error {
t.mu.Lock()
defer t.mu.Unlock()
return t.err
}