Merge C18 offline bundles and the Helm chart
Conflicts: server.go imports (both kept); agent.pb.go regenerated with buf from the merged proto. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
6f85c17fc5
51 files changed
+4177
-63
No files matched your search
@@ -88,7 +88,13 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) {
|
||||
// the installed build is used first — and quietly replace theirs.
|
||||
continue
|
||||
}
|
||||
wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()})
|
||||
w := provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()}
|
||||
if base := e.GetOfflineBase(); base != "" {
|
||||
// An offline cluster: from its mirror, checked against what the
|
||||
// control plane holds, never the publisher.
|
||||
w.Offline = &provision.Offline{Base: base, SHA256: maps.Clone(e.GetOfflineSha256())}
|
||||
}
|
||||
wants = append(wants, w)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.provWants = wants
|
||||
|
||||
@@ -34,6 +34,24 @@ type Want struct {
|
||||
// Variant is a build chosen for this node ("cu129"); empty leaves it to
|
||||
// the node's hardware.
|
||||
Variant string
|
||||
// Offline, when set, is where the engine comes from instead of its
|
||||
// publisher: a cluster with no way to the internet installs from its
|
||||
// mirror's copy of an offline bundle.
|
||||
Offline *Offline
|
||||
}
|
||||
|
||||
// Offline is an engine version's files on a cluster's own mirror.
|
||||
//
|
||||
// The files come from the mirror and their checksums from the control plane,
|
||||
// which read them from the bundle they were imported from. The mirror serves
|
||||
// plain HTTP to every node; a checksum taken from beside the file there would
|
||||
// prove only that the file and the checksum came from the same place.
|
||||
type Offline struct {
|
||||
// Base is where the version's files are: <mirror>/_llmgw/engines/<engine>/<version>.
|
||||
Base string
|
||||
// SHA256 is each file's checksum, by name, as the publisher gave it
|
||||
// when the bundle was made.
|
||||
SHA256 map[string]string
|
||||
}
|
||||
|
||||
// The states an engine's installation can be in.
|
||||
@@ -156,6 +174,16 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
|
||||
}
|
||||
return st
|
||||
}
|
||||
if w.Offline != nil && recipe.Repo == "" {
|
||||
// Said, not attempted: uv would spend minutes failing to reach PyPI,
|
||||
// and the failure would read like a network fault rather than the
|
||||
// design of an offline site.
|
||||
st.State = Unsupported
|
||||
st.Detail = "this cluster installs offline, and " + w.Engine + " comes from PyPI, which an offline bundle " +
|
||||
"does not carry: install it on the node by hand and name it in agent.yaml, or set engines." +
|
||||
w.Engine + ` to "" in server.yaml`
|
||||
return st
|
||||
}
|
||||
|
||||
// Installed before: whichever candidate is, best first, checked that it
|
||||
// still runs and trusted for nothing else.
|
||||
@@ -273,10 +301,15 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
return err
|
||||
}
|
||||
|
||||
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
|
||||
sums, err := in.checksums(ctx, r.Repo, w.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
var sums map[string]string
|
||||
if w.Offline != nil {
|
||||
sums = w.Offline.SHA256
|
||||
} else {
|
||||
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
|
||||
var err error
|
||||
if sums, err = in.checksums(ctx, r.Repo, w.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tmp := dir + ".partial"
|
||||
@@ -289,14 +322,22 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
got := map[string]string{}
|
||||
for _, asset := range b.Assets {
|
||||
want, ok := sums[asset]
|
||||
if !ok {
|
||||
switch {
|
||||
case !ok && w.Offline != nil:
|
||||
return fmt.Errorf("the offline bundle has no %s for %s %s: make one that has "+
|
||||
"(llmgw bundle create --engine %s=%s) and import it", asset, w.Engine, w.Version, w.Engine, w.Version)
|
||||
case !ok:
|
||||
// Refused rather than installed unchecked: this is a binary that
|
||||
// will run as root on every node.
|
||||
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
|
||||
w.Engine, w.Version, asset)
|
||||
}
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + w.Version + "/" + asset
|
||||
if w.Offline != nil {
|
||||
url = strings.TrimSuffix(w.Offline.Base, "/") + "/" + asset
|
||||
}
|
||||
archive := filepath.Join(tmp, asset+".download")
|
||||
if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
|
||||
if err := in.download(ctx, url, asset, archive, want, st); err != nil {
|
||||
return err
|
||||
}
|
||||
in.progress(st, "unpacking %s", asset)
|
||||
@@ -553,6 +594,26 @@ func (in *Installer) has(tool string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Published is the sha256 an engine's publisher gives for each file of a
|
||||
// version, and where each is downloaded from: what an offline bundle is made
|
||||
// of, read the way a node online reads it.
|
||||
func (in *Installer) Published(ctx context.Context, engine, version string) (sums map[string]string, url func(asset string) string, err error) {
|
||||
r, ok := Recipes[engine]
|
||||
if !ok || r.Repo == "" {
|
||||
return nil, nil, fmt.Errorf("%s is not published as release files", engine)
|
||||
}
|
||||
if !validVersion.MatchString(version) {
|
||||
return nil, nil, fmt.Errorf("%q is not a version of %s", version, engine)
|
||||
}
|
||||
sums, err = in.checksums(ctx, r.Repo, version)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return sums, func(asset string) string {
|
||||
return strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + version + "/" + asset
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checksums reads the sha256 GitHub publishes for each asset of a release.
|
||||
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
|
||||
url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
|
||||
@@ -594,8 +655,7 @@ func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[strin
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
|
||||
func (in *Installer) download(ctx context.Context, url, asset, dst, want string, st *Status) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
|
||||
// offlineMirror is a cluster's mirror holding a bundle's engine files, and
|
||||
// nothing that answers for GitHub.
|
||||
type offlineMirror struct {
|
||||
files map[string][]byte
|
||||
mu sync.Mutex
|
||||
paths []string
|
||||
}
|
||||
|
||||
func (m *offlineMirror) serve(t *testing.T) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
m.paths = append(m.paths, r.URL.Path)
|
||||
m.mu.Unlock()
|
||||
b, ok := m.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(b)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestAnOfflineNodeInstallsFromTheMirrorWithChecksumsFromTheControlPlane(t *testing.T) {
|
||||
rel := llamaRelease(t)
|
||||
asset := "llama-b1-bin-ubuntu-x64.tar.gz"
|
||||
m := &offlineMirror{files: map[string][]byte{"/_llmgw/engines/llamacpp/b1/" + asset: rel.assets[asset]}}
|
||||
srv := m.serve(t)
|
||||
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
|
||||
// GitHub is the mirror too, so a request for it would be seen and
|
||||
// answered 404: an offline install must not ask.
|
||||
in, _ := installer(t, srv, sys, ubuntu)
|
||||
|
||||
want := Want{Engine: "llamacpp", Version: "b1", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b1",
|
||||
SHA256: map[string]string{asset: strings.TrimPrefix(rel.digests[asset], "sha256:")},
|
||||
}}
|
||||
got := in.Converge(t.Context(), []Want{want})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("not installed from the mirror: %+v", got[0])
|
||||
}
|
||||
for _, p := range m.paths {
|
||||
if strings.HasPrefix(p, "/repos/") || strings.Contains(p, "/releases/download/") {
|
||||
t.Errorf("an offline install asked for %s", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A file on the mirror that is not what the bundle recorded is refused,
|
||||
// whatever the mirror says of it.
|
||||
m.files["/_llmgw/engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz"] = []byte("replaced on the mirror")
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b2", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b2",
|
||||
SHA256: map[string]string{"llama-b2-bin-ubuntu-x64.tar.gz": strings.Repeat("a", 64)},
|
||||
}}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") {
|
||||
t.Errorf("a file not matching the bundle's checksum was not refused: %+v", got[0])
|
||||
}
|
||||
|
||||
// One the bundle has no checksum for is not fetched at all, and the
|
||||
// operator is told which bundle would have it.
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b3", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b3",
|
||||
}}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "bundle create --engine llamacpp=b3") {
|
||||
t.Errorf("a version missing from the bundle did not say how to add it: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOfflineNodeIsToldPythonEnginesComeFromPyPI(t *testing.T) {
|
||||
gpu := ubuntu
|
||||
gpu.NVIDIADriver, gpu.MinComputeCapability = "580.65", 8.9
|
||||
in := &Installer{Root: t.TempDir(), Host: gpu, Run: (&fakeSystem{}).run}
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}})
|
||||
if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "PyPI") {
|
||||
t.Fatalf("vLLM offline: %+v", got[0])
|
||||
}
|
||||
|
||||
// A node it is not meant for still says nothing is wrong.
|
||||
in.Host = ubuntu
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}})
|
||||
if got[0].State != Skipped {
|
||||
t.Errorf("vLLM offline on a CPU node: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleCarriesEveryBuildTheNodesMightChoose(t *testing.T) {
|
||||
variants := func(arch string) []string {
|
||||
bs, err := Builds("llamacpp", "b1", arch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, b := range bs {
|
||||
out = append(out, b.Variant)
|
||||
}
|
||||
return out
|
||||
}
|
||||
if got := variants("amd64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "cuda-12.8", "vulkan"}) {
|
||||
t.Errorf("amd64 builds: %v", got)
|
||||
}
|
||||
if got := variants("arm64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "vulkan"}) {
|
||||
t.Errorf("arm64 builds: %v", got)
|
||||
}
|
||||
|
||||
// Each one a node would choose is among them, file for file.
|
||||
bundled := map[string][]string{}
|
||||
bs, _ := Builds("llamacpp", "b1", "amd64")
|
||||
for _, b := range bs {
|
||||
bundled[b.Variant] = b.Assets
|
||||
}
|
||||
for _, h := range []discovery.Host{
|
||||
ubuntu,
|
||||
{Arch: "amd64", NVIDIADriver: "550.1", MinComputeCapability: 8.6},
|
||||
{Arch: "amd64", NVIDIADriver: "470.1", MinComputeCapability: 6.1},
|
||||
} {
|
||||
chosen, err := llamaCppCandidates("b1", "", h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range chosen {
|
||||
if !slices.Equal(bundled[b.Variant], b.Assets) {
|
||||
t.Errorf("%+v chooses %s %v, which the bundle carries as %v", h, b.Variant, b.Assets, bundled[b.Variant])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := Builds("vllm", "0.30.0", "amd64"); err == nil || !strings.Contains(err.Error(), "PyPI") {
|
||||
t.Errorf("a Python engine was offered for a bundle: %v", err)
|
||||
}
|
||||
if _, err := Builds("llamacpp", "../b1", "amd64"); err == nil {
|
||||
t.Error("a version that is a path was accepted")
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,49 @@ func Installable() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// Builds are every build of an engine version published for an
|
||||
// architecture: what an offline bundle carries, for nodes it has not seen.
|
||||
// Read from the same table the nodes choose from, asked as a machine with
|
||||
// no GPU and as one with any NVIDIA GPU, so a bundle and the nodes that
|
||||
// install from it cannot disagree about which files a build is. Archive
|
||||
// engines only: a Python engine is a PyPI environment, which no bundle
|
||||
// carries.
|
||||
func Builds(engine, version, arch string) ([]Build, error) {
|
||||
r, ok := Recipes[engine]
|
||||
switch {
|
||||
case !ok:
|
||||
return nil, fmt.Errorf("this agent does not know how to install %s", engine)
|
||||
case r.Repo == "":
|
||||
return nil, fmt.Errorf("%s is installed from PyPI, which an offline bundle does not carry", engine)
|
||||
case !validVersion.MatchString(version):
|
||||
return nil, fmt.Errorf("%q is not a version of %s", version, engine)
|
||||
}
|
||||
var out []Build
|
||||
seen := map[string]bool{}
|
||||
var errs []string
|
||||
for _, h := range []discovery.Host{
|
||||
{Arch: arch},
|
||||
// Any card and the newest driver: every GPU build is a candidate.
|
||||
{Arch: arch, NVIDIADriver: "9999"},
|
||||
} {
|
||||
bs, err := r.Candidates(version, "", h)
|
||||
if err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
continue
|
||||
}
|
||||
for _, b := range bs {
|
||||
if !seen[b.Variant] {
|
||||
seen[b.Variant] = true
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("no build of %s %s for %s: %s", engine, version, arch, strings.Join(errs, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UVVersion is the uv that installs the Python engines.
|
||||
const UVVersion = "0.12.20"
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
// Package bundle carries what a cluster would otherwise fetch from the
|
||||
// internet to a site that has no way to it.
|
||||
//
|
||||
// A bundle is made on a machine with internet access: llmgw's own release
|
||||
// binaries with their signed checksums, the engine builds the nodes install
|
||||
// with the checksums their publisher gave, and model files with the
|
||||
// checksums their registry gave — each checked as it arrives, exactly as a
|
||||
// node online would check it. Its manifest lists every file with its sha256,
|
||||
// and the manifest's own sha256 is what the person carrying the bundle
|
||||
// compares at the other end.
|
||||
//
|
||||
// Imported, the files go into the cluster's mirror: releases and engines
|
||||
// under _llmgw/, which no model can be called, models where the mirror keeps
|
||||
// models. The engines' checksums go to the control plane, which tells each
|
||||
// node what every file it fetches from the mirror must be. The mirror serves
|
||||
// plain HTTP to every node, so nothing a node runs is taken on the mirror's
|
||||
// word: releases carry the project's signature, engines the checksum the
|
||||
// control plane holds.
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
// ManifestName is the manifest's file name in a bundle, and the first entry
|
||||
// of a bundle packed as a tar.
|
||||
const ManifestName = "manifest.json"
|
||||
|
||||
// Format is the manifest's layout version. A bundle of a later format is
|
||||
// refused rather than half understood.
|
||||
const Format = 1
|
||||
|
||||
// Prefix is where a mirror keeps what is not a model: _llmgw/releases and
|
||||
// _llmgw/engines. A model name may not start with an underscore here, so
|
||||
// none can be called this.
|
||||
const Prefix = "_llmgw"
|
||||
|
||||
// Manifest says what a bundle holds.
|
||||
type Manifest struct {
|
||||
Format int `json:"format"`
|
||||
Created time.Time `json:"created"`
|
||||
// MadeBy is the llmgw version that made it.
|
||||
MadeBy string `json:"made_by"`
|
||||
Releases []Release `json:"releases,omitempty"`
|
||||
Engines []Engine `json:"engines,omitempty"`
|
||||
Models []Model `json:"models,omitempty"`
|
||||
// Files are every file in the bundle but the manifest, with their
|
||||
// checksums: what an import checks each one against.
|
||||
Files []File `json:"files"`
|
||||
}
|
||||
|
||||
// Release is llmgw's own release, for some architectures.
|
||||
type Release struct {
|
||||
Version string `json:"version"`
|
||||
Arches []string `json:"arches"`
|
||||
}
|
||||
|
||||
// Engine is an engine version's builds, for some architectures.
|
||||
type Engine struct {
|
||||
Engine string `json:"engine"`
|
||||
Version string `json:"version"`
|
||||
Arches []string `json:"arches"`
|
||||
Variants []string `json:"variants"`
|
||||
}
|
||||
|
||||
// Model is a model's files, as the mirror will hold them.
|
||||
type Model struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
// Source is what it was asked for as: hf:owner/repo/file, or a URL.
|
||||
Source string `json:"source"`
|
||||
// Revision is the registry's commit it was taken at, when it has one.
|
||||
Revision string `json:"revision,omitempty"`
|
||||
}
|
||||
|
||||
// File is one file of a bundle.
|
||||
type File struct {
|
||||
// Path is where it is in the bundle: releases/<version>/<file>,
|
||||
// engines/<engine>/<version>/<file> or models/<model>/<version>/<file>.
|
||||
Path string `json:"path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
// Source is where it was downloaded from.
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// Kinds of file, by the first element of their path.
|
||||
const (
|
||||
kindRelease = "releases"
|
||||
kindEngine = "engines"
|
||||
kindModel = "models"
|
||||
)
|
||||
|
||||
// Digest is a manifest's sha256: what `bundle create` prints and an import
|
||||
// is given to check the bundle by.
|
||||
func Digest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest and checks every path in it, before any
|
||||
// file is read: a path is where the file is written.
|
||||
func parseManifest(raw []byte) (Manifest, error) {
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return Manifest{}, fmt.Errorf("the bundle's manifest cannot be read: %w", err)
|
||||
}
|
||||
if m.Format != Format {
|
||||
return Manifest{}, fmt.Errorf("the bundle is of format %d and this llmgw reads format %d: "+
|
||||
"import it with the llmgw that made it", m.Format, Format)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, f := range m.Files {
|
||||
if _, err := Destination(f.Path); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if len(f.SHA256) != 64 {
|
||||
return Manifest{}, fmt.Errorf("the manifest gives %s no sha256", f.Path)
|
||||
}
|
||||
if _, err := hex.DecodeString(f.SHA256); err != nil {
|
||||
return Manifest{}, fmt.Errorf("the manifest's sha256 of %s is not one", f.Path)
|
||||
}
|
||||
if seen[f.Path] {
|
||||
return Manifest{}, fmt.Errorf("the manifest lists %s twice", f.Path)
|
||||
}
|
||||
seen[f.Path] = true
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Destination is where a bundle's file goes in a mirror, relative to its
|
||||
// root, or why it goes nowhere.
|
||||
//
|
||||
// Every path in a bundle has a fixed shape — three or four plain elements
|
||||
// under one of three directories — because a path is where the import
|
||||
// writes, as root. Anything else is refused, not cleaned.
|
||||
func Destination(p string) (string, error) {
|
||||
parts := strings.Split(p, "/")
|
||||
bad := func(why string) (string, error) {
|
||||
return "", fmt.Errorf("the bundle names a file %q, which %s", p, why)
|
||||
}
|
||||
if path.Clean(p) != p || strings.HasPrefix(p, "/") {
|
||||
return bad("is not a plain path")
|
||||
}
|
||||
for _, e := range parts {
|
||||
if err := mirror.ValidName(e); err != nil {
|
||||
return bad("has an element that is not a plain name")
|
||||
}
|
||||
}
|
||||
switch parts[0] {
|
||||
case kindRelease:
|
||||
if len(parts) != 3 || !release.Tag.MatchString(parts[1]) {
|
||||
return bad("is not releases/<version>/<file>")
|
||||
}
|
||||
return Prefix + "/" + p, nil
|
||||
case kindEngine:
|
||||
if len(parts) != 4 {
|
||||
return bad("is not engines/<engine>/<version>/<file>")
|
||||
}
|
||||
return Prefix + "/" + p, nil
|
||||
case kindModel:
|
||||
if len(parts) != 4 {
|
||||
return bad("is not models/<model>/<version>/<file>")
|
||||
}
|
||||
if strings.HasPrefix(parts[1], "_") {
|
||||
return bad("names a model starting with an underscore, which the mirror keeps for itself")
|
||||
}
|
||||
return strings.Join(parts[1:], "/"), nil
|
||||
}
|
||||
return bad("is not under releases/, engines/ or models/")
|
||||
}
|
||||
|
||||
// EngineBase is where an engine version's files are on a mirror whose files
|
||||
// are served at mirrorURL.
|
||||
func EngineBase(mirrorURL, engine, version string) string {
|
||||
return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindEngine + "/" + engine + "/" + version
|
||||
}
|
||||
|
||||
// ReleasesBase is where llmgw's releases are on a mirror whose files are
|
||||
// served at mirrorURL: what `releases` is on an offline cluster.
|
||||
func ReleasesBase(mirrorURL string) string {
|
||||
return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindRelease
|
||||
}
|
||||
|
||||
// ErrNoDigest is an import or trust given no sha256 to check the bundle by.
|
||||
var ErrNoDigest = errors.New("give the bundle's sha256, as 'llmgw bundle create' printed it, " +
|
||||
"so what was carried is known to be what was made; or say it is to be taken unchecked")
|
||||
|
||||
// checkDigest compares a manifest with the digest it was given. Unchecked
|
||||
// takes it as it is, said by the caller.
|
||||
func checkDigest(raw []byte, want string, unchecked bool) error {
|
||||
want = strings.ToLower(strings.TrimSpace(want))
|
||||
switch {
|
||||
case want == "" && unchecked:
|
||||
return nil
|
||||
case want == "":
|
||||
return ErrNoDigest
|
||||
case Digest(raw) != want:
|
||||
return fmt.Errorf("the bundle's manifest is %s, not the %s it was given as: "+
|
||||
"it is not the bundle that was made, or the digest was mistyped", Digest(raw)[:16], want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
func hexSum(b []byte) string {
|
||||
s := sha256.Sum256(b)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
// internet is everything a bundle is made from: a release server, GitHub
|
||||
// and Hugging Face, all fakes.
|
||||
type internet struct {
|
||||
srv *httptest.Server
|
||||
files map[string][]byte
|
||||
digests map[string]string // GitHub's, by asset
|
||||
priv string
|
||||
weights []byte
|
||||
hfDigest string
|
||||
}
|
||||
|
||||
func newInternet(t *testing.T) *internet {
|
||||
t.Helper()
|
||||
pub, priv, err := release.GenerateKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("LLMGW_RELEASE_KEY", pub)
|
||||
in := &internet{files: map[string][]byte{}, digests: map[string]string{}, priv: priv}
|
||||
|
||||
// llmgw v1.2.3 for both architectures, signed.
|
||||
var sums strings.Builder
|
||||
for _, arch := range []string{"amd64", "arm64"} {
|
||||
name := release.AssetName("v1.2.3", "linux", arch)
|
||||
body := []byte("llmgw for " + arch)
|
||||
in.files["/releases/v1.2.3/"+name] = body
|
||||
sums.WriteString(hexSum(body) + " " + name + "\n")
|
||||
}
|
||||
in.files["/releases/v1.2.3/checksums.txt"] = []byte(sums.String())
|
||||
sig, _ := release.Sign(priv, []byte(sums.String()))
|
||||
in.files["/releases/v1.2.3/"+release.SignatureName] = []byte(sig)
|
||||
|
||||
// llama.cpp b1: every build there is for amd64.
|
||||
builds, err := provision.Builds("llamacpp", "b1", "amd64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range builds {
|
||||
for _, a := range b.Assets {
|
||||
body := []byte("archive " + a)
|
||||
in.files["/gh/ggml-org/llama.cpp/releases/download/b1/"+a] = body
|
||||
in.digests[a] = "sha256:" + hexSum(body)
|
||||
}
|
||||
}
|
||||
|
||||
// A model on Hugging Face, with the checksum the registry publishes.
|
||||
in.weights = []byte("GGUF weights")
|
||||
in.hfDigest = hexSum(in.weights)
|
||||
in.files["/Org/Model-GGUF/resolve/abc123/model-q4.gguf"] = in.weights
|
||||
|
||||
in.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/api/repos/ggml-org/llama.cpp/releases/tags/b1"):
|
||||
var out struct {
|
||||
Assets []map[string]string `json:"assets"`
|
||||
}
|
||||
for name, d := range in.digests {
|
||||
out.Assets = append(out.Assets, map[string]string{"name": name, "digest": d})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case r.URL.Path == "/api/models/Org/Model-GGUF":
|
||||
_, _ = w.Write([]byte(`{"id":"Org/Model-GGUF","sha":"abc123"}`))
|
||||
case r.URL.Path == "/api/models/Org/Model-GGUF/tree/abc123":
|
||||
_ = json.NewEncoder(w).Encode([]map[string]any{
|
||||
{"type": "file", "path": "model-q4.gguf", "size": 133,
|
||||
"lfs": map[string]any{"oid": in.hfDigest, "size": len(in.weights)}},
|
||||
})
|
||||
default:
|
||||
b, ok := in.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(b)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(in.srv.Close)
|
||||
return in
|
||||
}
|
||||
|
||||
func (in *internet) options(out string) CreateOptions {
|
||||
return CreateOptions{
|
||||
Out: out, Release: "v1.2.3", Arches: []string{"amd64"},
|
||||
Releases: in.srv.URL + "/releases",
|
||||
Engines: []EngineSpec{{Engine: "llamacpp", Version: "b1", Variants: []string{"cpu", "cuda-12.8"}}},
|
||||
Models: []ModelSpec{{Name: "starter", Version: "2026-10", Source: "hf:Org/Model-GGUF/model-q4.gguf"}},
|
||||
HTTP: in.srv.Client(),
|
||||
GitHubAPI: in.srv.URL + "/api", GitHubDownload: in.srv.URL + "/gh", Registry: in.srv.URL,
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleCarriesWhatTheClusterWouldFetchAndAMirrorTakesItChecked(t *testing.T) {
|
||||
in := newInternet(t)
|
||||
out := filepath.Join(t.TempDir(), "bundle")
|
||||
m, digest, err := Create(t.Context(), in.options(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paths []string
|
||||
for _, f := range m.Files {
|
||||
paths = append(paths, f.Path)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"releases/v1.2.3/checksums.txt", "releases/v1.2.3/" + release.SignatureName,
|
||||
"releases/v1.2.3/llmgw-v1.2.3-linux-amd64",
|
||||
"engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz",
|
||||
"engines/llamacpp/b1/llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz",
|
||||
"engines/llamacpp/b1/cudart-llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz",
|
||||
"models/starter/2026-10/model-q4.gguf",
|
||||
} {
|
||||
if !strings.Contains(strings.Join(paths, "\n"), want) {
|
||||
t.Errorf("the bundle lacks %s:\n%s", want, strings.Join(paths, "\n"))
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.Join(paths, "\n"), "vulkan") {
|
||||
t.Error("a build not asked for was carried")
|
||||
}
|
||||
|
||||
// Carried as one file, and imported from it as from the directory.
|
||||
tarball := out + ".tar"
|
||||
if err := Pack(out, tarball); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, from := range []string{tarball, out} {
|
||||
root := t.TempDir()
|
||||
res, err := Import(from, ImportOptions{Mirror: root, SHA256: digest})
|
||||
if err != nil {
|
||||
t.Fatalf("importing %s: %v", filepath.Base(from), err)
|
||||
}
|
||||
if res.Written != len(m.Files) || res.Digest != digest {
|
||||
t.Errorf("wrote %d of %d files", res.Written, len(m.Files))
|
||||
}
|
||||
bin, _ := os.ReadFile(filepath.Join(root, "_llmgw/releases/v1.2.3/llmgw-v1.2.3-linux-amd64"))
|
||||
if string(bin) != "llmgw for amd64" {
|
||||
t.Errorf("the release is not where releases are fetched from: %q", bin)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "_llmgw/engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz")); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
// The model is the mirror's like any other, with its checksum.
|
||||
man, err := mirror.ManifestFor(root, "starter", "2026-10")
|
||||
if err != nil || len(man.Files) != 1 || man.Files[0].SHA256 != in.hfDigest {
|
||||
t.Errorf("the model is not offered with its checksum: %+v %v", man, err)
|
||||
}
|
||||
// And the mirror lists nothing of llmgw's own as a model.
|
||||
entries, _ := mirror.List(root)
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("the mirror lists %v", entries)
|
||||
}
|
||||
|
||||
// Again: everything is there and the same, so nothing is written.
|
||||
res, err = Import(from, ImportOptions{Mirror: root, SHA256: digest})
|
||||
if err != nil || res.Written != 0 || res.Kept != len(m.Files) {
|
||||
t.Errorf("a second import: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Made again into the same directory, a file already there and right is
|
||||
// not downloaded: only the checksums are asked for again.
|
||||
for p := range in.files {
|
||||
if !strings.HasSuffix(p, "checksums.txt") && !strings.HasSuffix(p, release.SignatureName) {
|
||||
delete(in.files, p)
|
||||
}
|
||||
}
|
||||
_, again, err := Create(t.Context(), in.options(out))
|
||||
if err != nil {
|
||||
t.Fatalf("made again from what was there: %v", err)
|
||||
}
|
||||
var m2 Manifest
|
||||
raw, _ := os.ReadFile(filepath.Join(out, ManifestName))
|
||||
_ = json.Unmarshal(raw, &m2)
|
||||
if again != Digest(raw) || len(m2.Files) != len(m.Files) {
|
||||
t.Errorf("made again: %d files, digest %s", len(m2.Files), again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControlPlaneTrustsTheBundlesEngineChecksumsAndNodesAreToldThem(t *testing.T) {
|
||||
in := newInternet(t)
|
||||
out := filepath.Join(t.TempDir(), "bundle")
|
||||
_, digest, err := Create(t.Context(), in.options(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "offline.sha256")
|
||||
if _, _, err := Trust(out, file, "", false); err == nil {
|
||||
t.Fatal("a bundle was trusted with no digest to check it by")
|
||||
}
|
||||
if _, _, err := Trust(out, file, strings.Repeat("0", 64), false); err == nil {
|
||||
t.Fatal("a bundle was trusted against the wrong digest")
|
||||
}
|
||||
if _, n, err := Trust(out, file, digest, false); err != nil || n != 3 {
|
||||
t.Fatalf("trusted %d: %v", n, err)
|
||||
}
|
||||
|
||||
tr := NewTrusted(file)
|
||||
got := tr.For("llamacpp", "b1")
|
||||
if got["llama-b1-bin-ubuntu-x64.tar.gz"] != strings.TrimPrefix(in.digests["llama-b1-bin-ubuntu-x64.tar.gz"], "sha256:") {
|
||||
t.Errorf("the control plane holds %v", got)
|
||||
}
|
||||
if len(tr.For("llamacpp", "b2")) != 0 {
|
||||
t.Error("checksums were given for a version no bundle carried")
|
||||
}
|
||||
|
||||
// A second bundle adds to what is trusted, and the control plane sees
|
||||
// it without a restart.
|
||||
m2 := Manifest{Format: Format, Files: []File{{Path: "engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz", Bytes: 1, SHA256: strings.Repeat("b", 64)}}}
|
||||
if n, err := TrustManifest(m2, file); err != nil || n != 4 {
|
||||
t.Fatalf("trusted %d: %v", n, err)
|
||||
}
|
||||
if tr.For("llamacpp", "b2")["llama-b2-bin-ubuntu-x64.tar.gz"] != strings.Repeat("b", 64) {
|
||||
t.Error("a bundle trusted while running was not seen")
|
||||
}
|
||||
if len(tr.For("llamacpp", "b1")) != 3 {
|
||||
t.Error("the first bundle's checksums were lost")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADamagedBundlePutsNothingInTheMirror(t *testing.T) {
|
||||
in := newInternet(t)
|
||||
out := filepath.Join(t.TempDir(), "bundle")
|
||||
_, digest, err := Create(t.Context(), in.options(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The wrong digest, and none at all, are refused before anything is read.
|
||||
root := t.TempDir()
|
||||
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: strings.Repeat("0", 64)}); err == nil {
|
||||
t.Error("a bundle not matching its digest was imported")
|
||||
}
|
||||
if _, err := Import(out, ImportOptions{Mirror: root}); err == nil {
|
||||
t.Error("a bundle was imported with no digest")
|
||||
}
|
||||
if _, err := Import(out, ImportOptions{Mirror: root, Unchecked: true}); err != nil {
|
||||
t.Errorf("a bundle taken unchecked, as asked: %v", err)
|
||||
}
|
||||
|
||||
// An engine changed on the way.
|
||||
engine := filepath.Join(out, "engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz")
|
||||
good, _ := os.ReadFile(engine)
|
||||
_ = os.WriteFile(engine, []byte("swapped"), 0o644)
|
||||
root = t.TempDir()
|
||||
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "damaged") {
|
||||
t.Errorf("a changed engine: %v", err)
|
||||
}
|
||||
if entries := walk(t, root); len(entries) > 0 {
|
||||
t.Errorf("a damaged bundle left %v in the mirror", entries)
|
||||
}
|
||||
_ = os.WriteFile(engine, good, 0o644)
|
||||
|
||||
// A release whose checksums the project did not sign, with a manifest
|
||||
// made to agree: what the manifest says is not enough for a binary.
|
||||
_, other, _ := release.GenerateKey()
|
||||
sums, _ := os.ReadFile(filepath.Join(out, "releases/v1.2.3/checksums.txt"))
|
||||
forged, _ := release.Sign(other, sums)
|
||||
_ = os.WriteFile(filepath.Join(out, "releases/v1.2.3/"+release.SignatureName), []byte(forged), 0o644)
|
||||
raw := remanifest(t, out)
|
||||
root = t.TempDir()
|
||||
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: Digest(raw)}); err == nil || !strings.Contains(err.Error(), "not signed") {
|
||||
t.Errorf("a release not signed by the project: %v", err)
|
||||
}
|
||||
if entries := walk(t, root); len(entries) > 0 {
|
||||
t.Errorf("an unsigned release left %v in the mirror", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVersionAlreadyInTheMirrorIsNeverReplaced(t *testing.T) {
|
||||
in := newInternet(t)
|
||||
out := filepath.Join(t.TempDir(), "bundle")
|
||||
_, digest, err := Create(t.Context(), in.options(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(root, "starter", "2026-10"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf"), []byte("other weights"), 0o644)
|
||||
if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "never replaced") {
|
||||
t.Fatalf("a different file under the same version: %v", err)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf")); string(b) != "other weights" {
|
||||
t.Error("the mirror's file was replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleCannotWriteOutsideTheMirror(t *testing.T) {
|
||||
for _, p := range []string{
|
||||
"../etc/cron.d/x", "models/../../x/y/z", "models/a/b/../c", "/abs/olute/path/x",
|
||||
"engines/llamacpp/b1", "releases/latest/llmgw", "models/_llmgw/releases/x",
|
||||
"models/.hidden/v/f", "other/a/b/c",
|
||||
} {
|
||||
if _, err := Destination(p); err == nil {
|
||||
t.Errorf("%q was given a destination", p)
|
||||
}
|
||||
}
|
||||
if d, err := Destination("models/m/v/f.gguf"); err != nil || d != "m/v/f.gguf" {
|
||||
t.Errorf("a model: %q %v", d, err)
|
||||
}
|
||||
|
||||
// A tar carrying a file its manifest does not list is refused.
|
||||
dir := t.TempDir()
|
||||
raw, _ := json.Marshal(Manifest{Format: Format, Files: []File{}})
|
||||
tarball := filepath.Join(dir, "b.tar")
|
||||
f, _ := os.Create(tarball)
|
||||
tw := tar.NewWriter(f)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write(raw)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "models/m/v/extra", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("x"))
|
||||
_ = tw.Close()
|
||||
_ = f.Close()
|
||||
if _, err := Import(tarball, ImportOptions{Mirror: t.TempDir(), SHA256: Digest(raw)}); err == nil {
|
||||
t.Error("a file the manifest does not list was imported")
|
||||
}
|
||||
}
|
||||
|
||||
// remanifest rewrites a bundle's manifest to agree with its files, as
|
||||
// someone changing a bundle on the way would.
|
||||
func remanifest(t *testing.T, dir string) []byte {
|
||||
t.Helper()
|
||||
raw, _ := os.ReadFile(filepath.Join(dir, ManifestName))
|
||||
var m Manifest
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
for i, f := range m.Files {
|
||||
b, _ := os.ReadFile(filepath.Join(dir, f.Path))
|
||||
m.Files[i].SHA256, m.Files[i].Bytes = hexSum(b), int64(len(b))
|
||||
}
|
||||
raw, _ = json.MarshalIndent(m, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(dir, ManifestName), raw, 0o644)
|
||||
return raw
|
||||
}
|
||||
|
||||
func walk(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && !info.IsDir() {
|
||||
out = append(out, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
//go:build !unix
|
||||
|
||||
package bundle
|
||||
|
||||
// chownLike does nothing where files have no Unix owner.
|
||||
func chownLike(string, string) {}
|
||||
@@ -0,0 +1,26 @@
|
||||
//go:build unix
|
||||
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// chownLike gives p the owner of root, when this runs as root: the mirror
|
||||
// runs as its own user, and a version it cannot remove is one `mirror
|
||||
// remove` and the admin page fail on.
|
||||
func chownLike(root, p string) {
|
||||
if os.Geteuid() != 0 {
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
st, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_ = os.Lchown(p, int(st.Uid), int(st.Gid))
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
// EngineSpec is an engine version to carry, and which of its builds.
|
||||
type EngineSpec struct {
|
||||
Engine, Version string
|
||||
// Variants are the builds to carry ("cpu", "cuda-13.3"); empty is every
|
||||
// one, which is what a site whose nodes are not all known wants.
|
||||
Variants []string
|
||||
}
|
||||
|
||||
// ModelSpec is a model file to carry, and where the mirror will hold it.
|
||||
type ModelSpec struct {
|
||||
Name, Version string
|
||||
// Source is hf:owner/repo/path/in/repo, or an http(s) URL.
|
||||
Source string
|
||||
}
|
||||
|
||||
// CreateOptions say what goes into a bundle.
|
||||
type CreateOptions struct {
|
||||
// Out is the directory the bundle is made in. A bundle made there
|
||||
// before is added to: a file already there with the right checksum is
|
||||
// not downloaded again.
|
||||
Out string
|
||||
// Release is llmgw's release to carry, v0.10.4; empty carries none.
|
||||
Release string
|
||||
// Arches are the architectures, of the releases and of the engines:
|
||||
// amd64, arm64.
|
||||
Arches []string
|
||||
// Releases is where release files are published.
|
||||
Releases string
|
||||
Engines []EngineSpec
|
||||
Models []ModelSpec
|
||||
|
||||
// HTTP fetches everything; nil is a client with no overall timeout,
|
||||
// because a model file is tens of gigabytes.
|
||||
HTTP *http.Client
|
||||
// GitHubAPI, GitHubDownload and Registry default to GitHub and Hugging
|
||||
// Face; tests point them at fakes.
|
||||
GitHubAPI, GitHubDownload, Registry string
|
||||
// Log hears what is happening, a line at a time.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
// Create downloads everything a bundle is asked to carry, checking each
|
||||
// file as a node online would, and writes its manifest. It returns the
|
||||
// manifest and its digest: the value to carry to the site separately and
|
||||
// give to the import.
|
||||
func Create(ctx context.Context, o CreateOptions) (Manifest, string, error) {
|
||||
c := &creator{o: o, client: o.HTTP}
|
||||
if c.client == nil {
|
||||
c.client = &http.Client{}
|
||||
}
|
||||
if len(o.Arches) == 0 {
|
||||
return Manifest{}, "", errors.New("give at least one architecture: amd64, arm64")
|
||||
}
|
||||
for _, a := range o.Arches {
|
||||
if a != "amd64" && a != "arm64" {
|
||||
return Manifest{}, "", fmt.Errorf("%q is not an architecture llmgw is built for: amd64 or arm64", a)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(o.Out, 0o755); err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
c.m = Manifest{Format: Format, Created: time.Now().UTC().Truncate(time.Second), MadeBy: version.Version}
|
||||
|
||||
if o.Release != "" {
|
||||
if err := c.release(ctx); err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
}
|
||||
for _, e := range o.Engines {
|
||||
if err := c.engine(ctx, e); err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
}
|
||||
for _, m := range o.Models {
|
||||
if err := c.model(ctx, m); err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
}
|
||||
raw, err := json.MarshalIndent(c.m, "", " ")
|
||||
if err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(filepath.Join(o.Out, ManifestName), raw, 0o644); err != nil { //nolint:gosec // a list of checksums
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
return c.m, Digest(raw), nil
|
||||
}
|
||||
|
||||
type creator struct {
|
||||
o CreateOptions
|
||||
client *http.Client
|
||||
m Manifest
|
||||
}
|
||||
|
||||
func (c *creator) log(format string, args ...any) {
|
||||
if c.o.Log != nil {
|
||||
c.o.Log(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// release carries llmgw's release: its checksums and their signature,
|
||||
// checked against the key this build trusts, and a binary per
|
||||
// architecture checked against them.
|
||||
func (c *creator) release(ctx context.Context) error {
|
||||
v := c.o.Release
|
||||
base := c.o.Releases
|
||||
if base == "" {
|
||||
base = release.DefaultBase
|
||||
}
|
||||
c.log("llmgw %s: its checksums and their signature", v)
|
||||
sums, err := release.FetchChecksums(ctx, c.client, base, v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llmgw %s: %w", v, err)
|
||||
}
|
||||
dir := kindRelease + "/" + v + "/"
|
||||
if err := c.writeSmall(dir+"checksums.txt", sums.Text); err != nil {
|
||||
return err
|
||||
}
|
||||
if sums.Signature != "" {
|
||||
if err := c.writeSmall(dir+release.SignatureName, []byte(sums.Signature)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
root := strings.TrimSuffix(base, "/") + "/" + v + "/"
|
||||
for _, arch := range c.o.Arches {
|
||||
asset := release.AssetName(v, "linux", arch)
|
||||
want, err := sums.For(asset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llmgw %s: %w", v, err)
|
||||
}
|
||||
if err := c.fetch(ctx, root+asset, dir+asset, want); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
c.m.Releases = append(c.m.Releases, Release{Version: v, Arches: slices.Clone(c.o.Arches)})
|
||||
return nil
|
||||
}
|
||||
|
||||
// engine carries an engine version's builds for each architecture, checked
|
||||
// against the checksums its publisher gives — the same the nodes ask for
|
||||
// online.
|
||||
func (c *creator) engine(ctx context.Context, e EngineSpec) error {
|
||||
in := &provision.Installer{HTTP: c.client, APIBase: c.o.GitHubAPI, DownloadBase: c.o.GitHubDownload}
|
||||
c.log("%s %s: the checksums its publisher gives", e.Engine, e.Version)
|
||||
sums, url, err := in.Published(ctx, e.Engine, e.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s: %w", e.Engine, e.Version, err)
|
||||
}
|
||||
var variants []string
|
||||
done := map[string]bool{}
|
||||
for _, arch := range c.o.Arches {
|
||||
builds, err := provision.Builds(e.Engine, e.Version, arch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var names []string
|
||||
for _, b := range builds {
|
||||
names = append(names, b.Variant)
|
||||
if len(e.Variants) > 0 && !slices.Contains(e.Variants, b.Variant) {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(variants, b.Variant) {
|
||||
variants = append(variants, b.Variant)
|
||||
}
|
||||
for _, asset := range b.Assets {
|
||||
if done[asset] {
|
||||
continue
|
||||
}
|
||||
want, ok := sums[asset]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be carried checked", e.Engine, e.Version, asset)
|
||||
}
|
||||
if err := c.fetch(ctx, url(asset), kindEngine+"/"+e.Engine+"/"+e.Version+"/"+asset, want); err != nil {
|
||||
return err
|
||||
}
|
||||
done[asset] = true
|
||||
}
|
||||
}
|
||||
for _, v := range e.Variants {
|
||||
if !slices.Contains(names, v) && !anyArchHas(e, v, c.o.Arches) {
|
||||
return fmt.Errorf("%s has no build %q; it has %s", e.Engine, v, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
c.m.Engines = append(c.m.Engines, Engine{Engine: e.Engine, Version: e.Version,
|
||||
Arches: slices.Clone(c.o.Arches), Variants: variants})
|
||||
return nil
|
||||
}
|
||||
|
||||
// anyArchHas says whether a variant is a build on any of the architectures:
|
||||
// CUDA 12.8 is published for x64 alone, and asking for it with arm64 too is
|
||||
// not a mistake.
|
||||
func anyArchHas(e EngineSpec, variant string, arches []string) bool {
|
||||
for _, a := range arches {
|
||||
bs, _ := provision.Builds(e.Engine, e.Version, a)
|
||||
for _, b := range bs {
|
||||
if b.Variant == variant {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// model carries a model file, checked against the checksum its registry
|
||||
// publishes when it publishes one, at the commit the registry had when it
|
||||
// was asked — so the bundle says exactly which weights it carries.
|
||||
func (c *creator) model(ctx context.Context, s ModelSpec) error {
|
||||
for _, n := range []string{s.Name, s.Version} {
|
||||
if err := mirror.ValidName(n); err != nil {
|
||||
return fmt.Errorf("model %s/%s: %w", s.Name, s.Version, err)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(s.Name, "_") {
|
||||
return fmt.Errorf("model %q: a name starting with an underscore is the mirror's own", s.Name)
|
||||
}
|
||||
var url, want, rev string
|
||||
switch {
|
||||
case strings.HasPrefix(s.Source, "hf:"):
|
||||
parts := strings.SplitN(strings.TrimPrefix(s.Source, "hf:"), "/", 3)
|
||||
if len(parts) != 3 || parts[2] == "" {
|
||||
return fmt.Errorf("%s names a repository, not a file: hf:owner/repo/file", s.Source)
|
||||
}
|
||||
repoID, file := parts[0]+"/"+parts[1], parts[2]
|
||||
reg := hf.New(c.o.Registry, time.Minute)
|
||||
repo, err := reg.Repo(ctx, repoID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", s.Source, err)
|
||||
}
|
||||
found := false
|
||||
for _, f := range repo.Files {
|
||||
if f.Path == file {
|
||||
want, found = f.SHA256, true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%s has no file %s at its latest commit", repoID, file)
|
||||
}
|
||||
url, rev = repo.DownloadURL(reg.Base(), file), repo.Revision
|
||||
case strings.HasPrefix(s.Source, "https://"), strings.HasPrefix(s.Source, "http://"):
|
||||
url = s.Source
|
||||
default:
|
||||
return fmt.Errorf("model source %q: hf:owner/repo/file or an http(s) URL", s.Source)
|
||||
}
|
||||
name := path.Base(strings.SplitN(url, "?", 2)[0])
|
||||
if err := mirror.ValidName(name); err != nil {
|
||||
return fmt.Errorf("%s: %w", s.Source, err)
|
||||
}
|
||||
if err := c.fetch(ctx, url, kindModel+"/"+s.Name+"/"+s.Version+"/"+name, want); err != nil {
|
||||
return err
|
||||
}
|
||||
c.m.Models = append(c.m.Models, Model{Name: s.Name, Version: s.Version, Source: s.Source, Revision: rev})
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeSmall puts a file the bundle makes itself, rather than downloads.
|
||||
func (c *creator) writeSmall(rel string, b []byte) error {
|
||||
dst := filepath.Join(c.o.Out, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dst, b, 0o644); err != nil { //nolint:gosec // published files
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(b)
|
||||
c.add(File{Path: rel, Bytes: int64(len(b)), SHA256: hex.EncodeToString(sum[:])})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *creator) add(f File) {
|
||||
for i := range c.m.Files {
|
||||
if c.m.Files[i].Path == f.Path {
|
||||
c.m.Files[i] = f
|
||||
return
|
||||
}
|
||||
}
|
||||
c.m.Files = append(c.m.Files, f)
|
||||
}
|
||||
|
||||
// fetch downloads url into the bundle at rel, checked against want when
|
||||
// there is one; a file already there that matches is kept.
|
||||
func (c *creator) fetch(ctx context.Context, url, rel, want string) error {
|
||||
dst := filepath.Join(c.o.Out, filepath.FromSlash(rel))
|
||||
if want != "" {
|
||||
if got, err := fileSHA256(dst); err == nil && got == want {
|
||||
info, _ := os.Stat(dst)
|
||||
c.log(" have %s", rel)
|
||||
c.add(File{Path: rel, Bytes: info.Size(), SHA256: want, Source: url})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("fetching %s: %s", url, resp.Status)
|
||||
}
|
||||
if resp.ContentLength > 0 {
|
||||
c.log(" fetching %s (%d MB)", rel, resp.ContentLength>>20)
|
||||
} else {
|
||||
c.log(" fetching %s", rel)
|
||||
}
|
||||
part := dst + partSuffix
|
||||
f, err := os.Create(part) //nolint:gosec // inside the bundle
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(f, h), &progressReader{r: resp.Body, total: resp.ContentLength, log: c.log, name: rel})
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil && resp.ContentLength > 0 && n != resp.ContentLength {
|
||||
err = fmt.Errorf("got %d bytes of %d", n, resp.ContentLength)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return fmt.Errorf("fetching %s: %w", url, err)
|
||||
}
|
||||
got := hex.EncodeToString(h.Sum(nil))
|
||||
if want != "" && got != want {
|
||||
_ = os.Remove(part)
|
||||
return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)", url, got[:12], want[:12])
|
||||
}
|
||||
if err := os.Rename(part, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
c.add(File{Path: rel, Bytes: n, SHA256: got, Source: url})
|
||||
return nil
|
||||
}
|
||||
|
||||
// progressReader says how far a long download has got, every few seconds.
|
||||
type progressReader struct {
|
||||
r io.Reader
|
||||
total int64
|
||||
done int64
|
||||
last time.Time
|
||||
log func(string, ...any)
|
||||
name string
|
||||
}
|
||||
|
||||
func (p *progressReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
p.done += int64(n)
|
||||
if p.total > 0 && time.Since(p.last) > 10*time.Second {
|
||||
if !p.last.IsZero() {
|
||||
p.log(" %d%% of %s", p.done*100/p.total, p.name)
|
||||
}
|
||||
p.last = time.Now()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Pack writes a bundle directory as one tar file, manifest first, so it can
|
||||
// be carried as one file and imported as a stream. Weights do not compress,
|
||||
// so it is not compressed.
|
||||
func Pack(dir, out string) error {
|
||||
src, m, err := open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.close() //nolint:errcheck // read only
|
||||
f, err := os.Create(out + partSuffix) //nolint:gosec // the operator's path
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(out + partSuffix) //nolint:errcheck // gone once renamed
|
||||
tw := tar.NewWriter(f)
|
||||
now := time.Now()
|
||||
raw := src.manifest()
|
||||
if err := tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), ModTime: now, Typeflag: tar.TypeReg}); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write(raw); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
for _, mf := range m.Files {
|
||||
in, err := os.Open(filepath.Join(dir, filepath.FromSlash(mf.Path))) //nolint:gosec // a checked path in the bundle
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
err = tw.WriteHeader(&tar.Header{Name: mf.Path, Mode: 0o644, Size: mf.Bytes, ModTime: now, Typeflag: tar.TypeReg})
|
||||
if err == nil {
|
||||
_, err = io.Copy(tw, in)
|
||||
}
|
||||
in.Close()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("packing %s: %w", mf.Path, err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(out+partSuffix, out)
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
// ImportOptions say where a bundle goes and how it is known to be the one
|
||||
// that was made.
|
||||
type ImportOptions struct {
|
||||
// Mirror is the mirror's root directory, /srv/models. Empty checks the
|
||||
// bundle and writes nothing.
|
||||
Mirror string
|
||||
// SHA256 is the manifest's digest as `bundle create` printed it.
|
||||
// Unchecked takes the bundle without one, said by whoever runs it.
|
||||
SHA256 string
|
||||
Unchecked bool
|
||||
// Progress hears each file as it is done: kept is one the mirror
|
||||
// already had, the same.
|
||||
Progress func(f File, kept bool)
|
||||
}
|
||||
|
||||
// Result is what an import did.
|
||||
type Result struct {
|
||||
Manifest Manifest
|
||||
// Digest is the manifest's sha256.
|
||||
Digest string
|
||||
// Written and Kept count files: put in the mirror, and already there.
|
||||
Written, Kept int
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
// partSuffix marks a file still being written; the mirror lists none.
|
||||
const partSuffix = ".part"
|
||||
|
||||
// Import checks every file of a bundle and puts it in a mirror.
|
||||
//
|
||||
// Nothing is in place until everything is checked. Each file is written
|
||||
// beside where it goes, under a name the mirror does not serve, and checked
|
||||
// against the manifest as it is written; each release's checksums are
|
||||
// checked against the project's signing key and its binaries against them.
|
||||
// Only then is each renamed into place — so a bundle damaged anywhere
|
||||
// leaves the mirror as it was.
|
||||
//
|
||||
// A file the mirror already has is kept when it is the same, and refused
|
||||
// when it is not: a version in the mirror is never replaced, because that
|
||||
// is what makes going back to it possible.
|
||||
func Import(p string, o ImportOptions) (Result, error) {
|
||||
src, m, err := open(p)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer src.close() //nolint:errcheck // read only
|
||||
res := Result{Manifest: m, Digest: Digest(src.manifest())}
|
||||
if err := checkDigest(src.manifest(), o.SHA256, o.Unchecked); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
byPath := map[string]File{}
|
||||
dest := map[string]string{}
|
||||
keep := map[string]bool{}
|
||||
for _, f := range m.Files {
|
||||
byPath[f.Path] = f
|
||||
if o.Mirror == "" {
|
||||
continue
|
||||
}
|
||||
rel, _ := Destination(f.Path) // checked when the manifest was read
|
||||
dest[f.Path] = filepath.Join(o.Mirror, filepath.FromSlash(rel))
|
||||
same, err := alreadyHas(dest[f.Path], f)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
keep[f.Path] = same
|
||||
}
|
||||
|
||||
// Each release's checksums are kept as they pass, to be checked against
|
||||
// the signing key once the whole release has been read.
|
||||
small := map[string][]byte{}
|
||||
var written []string
|
||||
defer func() {
|
||||
// Whatever was staged and not put in place goes.
|
||||
for _, p := range written {
|
||||
_ = os.Remove(dest[p] + partSuffix)
|
||||
}
|
||||
}()
|
||||
seen := map[string]bool{}
|
||||
err = src.each(func(name string, r io.Reader) error {
|
||||
f, ok := byPath[name]
|
||||
if !ok {
|
||||
return fmt.Errorf("the bundle holds %s, which its manifest does not list", name)
|
||||
}
|
||||
if seen[name] {
|
||||
return fmt.Errorf("the bundle holds %s twice", name)
|
||||
}
|
||||
seen[name] = true
|
||||
var w io.Writer = io.Discard
|
||||
var out *os.File
|
||||
if o.Mirror != "" && !keep[name] {
|
||||
if err := mkdirsLike(o.Mirror, filepath.Dir(dest[name])); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = os.OpenFile(dest[name]+partSuffix, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // a checked path under the mirror
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
written = append(written, name)
|
||||
w = out
|
||||
}
|
||||
if keep[name] {
|
||||
r = eofReader{}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if isReleaseSums(name) {
|
||||
w = io.MultiWriter(w, &limited{w: &buf, n: 1 << 20})
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(w, h), r)
|
||||
if out != nil {
|
||||
if cerr := out.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
if keep[name] {
|
||||
if isReleaseSums(name) {
|
||||
b, err := os.ReadFile(dest[name])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
small[name] = b
|
||||
}
|
||||
res.Kept++
|
||||
if o.Progress != nil {
|
||||
o.Progress(f, true)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got := hex.EncodeToString(h.Sum(nil)); got != f.SHA256 || n != f.Bytes {
|
||||
return fmt.Errorf("%s is not what the bundle was made with (sha256 %s, expected %s): "+
|
||||
"the bundle was damaged on the way; nothing was put in the mirror", name, got[:12], f.SHA256[:12])
|
||||
}
|
||||
if isReleaseSums(name) {
|
||||
small[name] = buf.Bytes()
|
||||
}
|
||||
res.Written++
|
||||
res.Bytes += n
|
||||
if o.Progress != nil {
|
||||
o.Progress(f, false)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
for _, f := range m.Files {
|
||||
if !seen[f.Path] {
|
||||
return res, fmt.Errorf("the bundle lacks %s, which its manifest lists", f.Path)
|
||||
}
|
||||
}
|
||||
if err := checkReleases(m, small); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if o.Mirror == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
for _, name := range written {
|
||||
d := dest[name]
|
||||
f := byPath[name]
|
||||
if strings.HasPrefix(name, kindModel+"/") {
|
||||
// Beside the file before it is in place, as a download into the
|
||||
// mirror does, so a node is never offered it without its checksum.
|
||||
if err := mirror.Record(d, f.SHA256, f.Source); err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
if err := os.Rename(d+partSuffix, d); err != nil {
|
||||
return res, err
|
||||
}
|
||||
chownLike(o.Mirror, d)
|
||||
if strings.HasPrefix(name, kindModel+"/") {
|
||||
dir, base := filepath.Split(d)
|
||||
chownLike(o.Mirror, filepath.Join(dir, "."+base+".sha256"))
|
||||
chownLike(o.Mirror, filepath.Join(dir, "."+base+".source"))
|
||||
}
|
||||
}
|
||||
written = nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// alreadyHas says whether the mirror has this file already, the same; an
|
||||
// error when it has a different one there.
|
||||
func alreadyHas(dst string, f File) (bool, error) {
|
||||
info, err := os.Stat(dst)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
sum := mirror.Recorded(dst)
|
||||
if sum == "" || info.Size() != f.Bytes {
|
||||
if sum, err = fileSHA256(dst); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if sum != f.SHA256 || info.Size() != f.Bytes {
|
||||
return false, fmt.Errorf("the mirror already has %s, and it is not the bundle's: "+
|
||||
"a version in the mirror is never replaced; give the model another version in a new bundle", dst)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func isReleaseSums(name string) bool {
|
||||
if !strings.HasPrefix(name, kindRelease+"/") {
|
||||
return false
|
||||
}
|
||||
b := path.Base(name)
|
||||
return b == "checksums.txt" || b == release.SignatureName
|
||||
}
|
||||
|
||||
// checkReleases holds each release's binaries to its checksums, and the
|
||||
// checksums to the project's signing key — as `node add` and an upgrade
|
||||
// would online. The manifest's own checksums say the bundle arrived as it
|
||||
// was made; these say what was made is the project's release.
|
||||
func checkReleases(m Manifest, small map[string][]byte) error {
|
||||
byVersion := map[string][]File{}
|
||||
for _, f := range m.Files {
|
||||
if strings.HasPrefix(f.Path, kindRelease+"/") {
|
||||
v := strings.Split(f.Path, "/")[1]
|
||||
byVersion[v] = append(byVersion[v], f)
|
||||
}
|
||||
}
|
||||
for _, v := range slices.Sorted(maps.Keys(byVersion)) {
|
||||
root := kindRelease + "/" + v + "/"
|
||||
text, ok := small[root+"checksums.txt"]
|
||||
if !ok {
|
||||
return fmt.Errorf("the bundle's llmgw %s has no checksums.txt", v)
|
||||
}
|
||||
c := release.Checksums{Text: text, Signature: string(small[root+release.SignatureName])}
|
||||
if err := c.Verify(); err != nil {
|
||||
return fmt.Errorf("llmgw %s in the bundle: %w", v, err)
|
||||
}
|
||||
for _, f := range byVersion[v] {
|
||||
name := path.Base(f.Path)
|
||||
if name == "checksums.txt" || name == release.SignatureName {
|
||||
continue
|
||||
}
|
||||
want, err := c.For(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llmgw %s in the bundle: %w", v, err)
|
||||
}
|
||||
if want != f.SHA256 {
|
||||
return fmt.Errorf("%s in the bundle is not the release's (the release gives sha256 %s)", f.Path, want[:12])
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mkdirsLike makes dir and every directory above it up to root, each owned
|
||||
// as root is: the mirror runs as its own user, and removes what it holds.
|
||||
func mkdirsLike(root, dir string) error {
|
||||
rel, err := filepath.Rel(root, dir)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return fmt.Errorf("%s is outside the mirror at %s", dir, root)
|
||||
}
|
||||
cur := root
|
||||
for _, e := range strings.Split(rel, string(filepath.Separator)) {
|
||||
if e == "." {
|
||||
continue
|
||||
}
|
||||
cur = filepath.Join(cur, e)
|
||||
if _, err := os.Stat(cur); err == nil {
|
||||
continue
|
||||
}
|
||||
if err := os.Mkdir(cur, 0o755); err != nil && !errors.Is(err, os.ErrExist) {
|
||||
return err
|
||||
}
|
||||
chownLike(root, cur)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileSHA256(p string) (string, error) {
|
||||
f, err := os.Open(p) //nolint:gosec // a file in the mirror
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// eofReader is a file passed over: a tar moves past it on its own.
|
||||
type eofReader struct{}
|
||||
|
||||
func (eofReader) Read([]byte) (int, error) { return 0, io.EOF }
|
||||
|
||||
// limited keeps at most n bytes and quietly drops the rest: a checksums
|
||||
// file larger than that is refused by its signature, not by a write error.
|
||||
type limited struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (l *limited) Write(p []byte) (int, error) {
|
||||
if l.n <= 0 {
|
||||
return len(p), nil
|
||||
}
|
||||
k := min(int64(len(p)), l.n)
|
||||
l.n -= k
|
||||
if _, err := l.w.Write(p[:k]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// maxManifest bounds the manifest read into memory.
|
||||
const maxManifest = 16 << 20
|
||||
|
||||
// source is a bundle being read: a directory, or the same packed as a tar
|
||||
// with the manifest first, read as a stream so tens of gigabytes of weights
|
||||
// are never unpacked twice.
|
||||
type source interface {
|
||||
// manifest is the manifest's bytes, as they are in the bundle.
|
||||
manifest() []byte
|
||||
// each hands over every file, in the order the bundle has them. A file
|
||||
// fn does not read is passed over.
|
||||
each(fn func(path string, r io.Reader) error) error
|
||||
close() error
|
||||
}
|
||||
|
||||
// open reads a bundle's manifest, from a directory or a tar.
|
||||
func open(p string) (source, Manifest, error) {
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, Manifest{}, fmt.Errorf("the bundle: %w", err)
|
||||
}
|
||||
var src source
|
||||
if info.IsDir() {
|
||||
src, err = openDir(p)
|
||||
} else {
|
||||
src, err = openTar(p)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, Manifest{}, err
|
||||
}
|
||||
m, err := parseManifest(src.manifest())
|
||||
if err != nil {
|
||||
_ = src.close()
|
||||
return nil, Manifest{}, err
|
||||
}
|
||||
if d, ok := src.(*dirSource); ok {
|
||||
d.files = m.Files
|
||||
}
|
||||
return src, m, nil
|
||||
}
|
||||
|
||||
// ReadManifest reads a bundle's manifest and its digest, without reading
|
||||
// its files.
|
||||
func ReadManifest(p string) (Manifest, string, error) {
|
||||
src, m, err := open(p)
|
||||
if err != nil {
|
||||
return Manifest{}, "", err
|
||||
}
|
||||
defer src.close() //nolint:errcheck // read only
|
||||
return m, Digest(src.manifest()), nil
|
||||
}
|
||||
|
||||
type dirSource struct {
|
||||
dir string
|
||||
raw []byte
|
||||
files []File
|
||||
}
|
||||
|
||||
func openDir(dir string) (*dirSource, error) {
|
||||
raw, err := readLimited(filepath.Join(dir, ManifestName))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s is not a bundle: %w", dir, err)
|
||||
}
|
||||
return &dirSource{dir: dir, raw: raw}, nil
|
||||
}
|
||||
|
||||
func (d *dirSource) manifest() []byte { return d.raw }
|
||||
func (d *dirSource) close() error { return nil }
|
||||
|
||||
func (d *dirSource) each(fn func(string, io.Reader) error) error {
|
||||
for _, f := range d.files {
|
||||
// The path was checked when the manifest was read: plain names
|
||||
// under the bundle's three directories.
|
||||
in, err := os.Open(filepath.Join(d.dir, filepath.FromSlash(f.Path)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bundle lists %s and does not have it: %w", f.Path, err)
|
||||
}
|
||||
err = fn(f.Path, in)
|
||||
in.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type tarSource struct {
|
||||
f *os.File
|
||||
tr *tar.Reader
|
||||
raw []byte
|
||||
}
|
||||
|
||||
func openTar(p string) (*tarSource, error) {
|
||||
f, err := os.Open(p) //nolint:gosec // the operator's bundle
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := tar.NewReader(f)
|
||||
h, err := tr.Next()
|
||||
if err != nil || h.Name != ManifestName || h.Typeflag != tar.TypeReg {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("%s is not a bundle: its first entry is not %s", p, ManifestName)
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(tr, maxManifest))
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &tarSource{f: f, tr: tr, raw: raw}, nil
|
||||
}
|
||||
|
||||
func (t *tarSource) manifest() []byte { return t.raw }
|
||||
func (t *tarSource) close() error { return t.f.Close() }
|
||||
|
||||
func (t *tarSource) each(fn func(string, io.Reader) error) error {
|
||||
for {
|
||||
h, err := t.tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the bundle: %w", err)
|
||||
}
|
||||
if h.Typeflag == tar.TypeDir {
|
||||
continue
|
||||
}
|
||||
if h.Typeflag != tar.TypeReg {
|
||||
// Links and devices have no place in a bundle; the files it
|
||||
// lists are all it holds.
|
||||
return fmt.Errorf("the bundle holds %s, which is not a plain file", h.Name)
|
||||
}
|
||||
if err := fn(strings.TrimPrefix(h.Name, "./"), t.tr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readLimited(p string) ([]byte, error) {
|
||||
f, err := os.Open(p) //nolint:gosec // the operator's bundle
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return io.ReadAll(io.LimitReader(f, maxManifest))
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Trust records a bundle's engine checksums in the file the control plane
|
||||
// reads them from, beside any recorded before, and says how many it holds
|
||||
// now. Only the manifest is read: the checksums are what each node holds
|
||||
// the files it fetches to, so the manifest must be the one that was made —
|
||||
// which is what the digest says.
|
||||
func Trust(p, file, digest string, unchecked bool) (Manifest, int, error) {
|
||||
src, m, err := open(p)
|
||||
if err != nil {
|
||||
return Manifest{}, 0, err
|
||||
}
|
||||
defer src.close() //nolint:errcheck // read only
|
||||
if err := checkDigest(src.manifest(), digest, unchecked); err != nil {
|
||||
return m, 0, err
|
||||
}
|
||||
n, err := TrustManifest(m, file)
|
||||
return m, n, err
|
||||
}
|
||||
|
||||
// TrustManifest records a manifest's engine checksums, for one already
|
||||
// checked by an import.
|
||||
func TrustManifest(m Manifest, file string) (int, error) {
|
||||
sums, err := readTrusted(file)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return 0, err
|
||||
}
|
||||
if sums == nil {
|
||||
sums = map[string]string{}
|
||||
}
|
||||
for _, f := range m.Files {
|
||||
if strings.HasPrefix(f.Path, kindEngine+"/") {
|
||||
sums[Prefix+"/"+f.Path] = f.SHA256
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# Engine files the nodes may install, as the offline bundles they came in\n" +
|
||||
"# gave them. Kept by 'llmgw bundle import' and 'llmgw bundle trust'.\n")
|
||||
for _, k := range slices.Sorted(maps.Keys(sums)) {
|
||||
fmt.Fprintf(&b, "%s %s\n", sums[k], k)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tmp := file + partSuffix
|
||||
if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { //nolint:gosec // checksums, for anyone to read
|
||||
return 0, err
|
||||
}
|
||||
return len(sums), os.Rename(tmp, file)
|
||||
}
|
||||
|
||||
// readTrusted reads a checksums file: "<sha256> <path>" a line, as
|
||||
// sha256sum writes them, with paths as the mirror has the files.
|
||||
func readTrusted(file string) (map[string]string, error) {
|
||||
b, err := os.ReadFile(file) //nolint:gosec // the control plane's own file
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
sc := bufio.NewScanner(bytes.NewReader(b))
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if len(f) != 2 || len(f[0]) != 64 {
|
||||
return nil, fmt.Errorf("%s: %q is not a checksum and a file", file, line)
|
||||
}
|
||||
out[strings.TrimPrefix(f[1], "*")] = f[0]
|
||||
}
|
||||
return out, sc.Err()
|
||||
}
|
||||
|
||||
// Trusted is the engine checksums a control plane holds, read from their
|
||||
// file again whenever it changes — so a bundle imported while the control
|
||||
// plane runs is offered to the next node that asks.
|
||||
type Trusted struct {
|
||||
file string
|
||||
|
||||
mu sync.Mutex
|
||||
mod time.Time
|
||||
size int64
|
||||
sums map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
// NewTrusted reads checksums from file as they are asked for.
|
||||
func NewTrusted(file string) *Trusted { return &Trusted{file: file} }
|
||||
|
||||
// For is an engine version's files and their checksums, by file name; none
|
||||
// when no bundle imported here carried it.
|
||||
func (t *Trusted) For(engine, version string) map[string]string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if info, err := os.Stat(t.file); err != nil {
|
||||
t.sums, t.err, t.mod, t.size = nil, err, time.Time{}, 0
|
||||
} else if !info.ModTime().Equal(t.mod) || info.Size() != t.size {
|
||||
t.sums, t.err = readTrusted(t.file)
|
||||
t.mod, t.size = info.ModTime(), info.Size()
|
||||
}
|
||||
prefix := Prefix + "/" + kindEngine + "/" + engine + "/" + version + "/"
|
||||
out := map[string]string{}
|
||||
for p, sum := range t.sums {
|
||||
if rest, ok := strings.CutPrefix(p, prefix); ok && path.Base(rest) == rest {
|
||||
out[rest] = sum
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Err is why the file could not be read, the last time it was.
|
||||
func (t *Trusted) Err() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.err
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
// newBundleCmd carries a cluster's downloads to a site with no internet.
|
||||
func newBundleCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "bundle",
|
||||
Short: "Carry releases, engines and models to a site with no internet",
|
||||
Long: "A cluster fetches llmgw's releases, the llama.cpp builds its nodes install,\n" +
|
||||
"and model files from the internet. A site that has none gets them in a\n" +
|
||||
"bundle: made where there is internet with 'bundle create', carried in, and\n" +
|
||||
"put into the cluster's mirror with 'bundle import'. Every file is checked\n" +
|
||||
"as a node online would check it — releases against the project's signing\n" +
|
||||
"key, engines against their publisher's checksums — when the bundle is made,\n" +
|
||||
"when it is imported, and again on the node that installs it.",
|
||||
}
|
||||
c.AddCommand(newBundleCreateCmd(), newBundleImportCmd(), newBundleTrustCmd(), newBundleShowCmd())
|
||||
return c
|
||||
}
|
||||
|
||||
func newBundleCreateCmd() *cobra.Command {
|
||||
var (
|
||||
out, rel, releases, registry string
|
||||
arches, engines, variants []string
|
||||
models []string
|
||||
noRelease, noStarter, pack bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "create",
|
||||
Short: "Make a bundle, on a machine with internet",
|
||||
Long: "Downloads into --out, checking each file as it arrives: llmgw's release for\n" +
|
||||
"each --arch with its signed checksums, every build of each --engine for those\n" +
|
||||
"architectures (or the --variant builds), and each --model's file from Hugging\n" +
|
||||
"Face, pinned to the commit it was taken at. The starter model and the\n" +
|
||||
"guardrails' judge that 'server init' serves are carried unless\n" +
|
||||
"--no-starter-models.\n\n" +
|
||||
"Run again with the same --out to add to a bundle: what is there and right is\n" +
|
||||
"not downloaded again.\n\n" +
|
||||
"It prints the manifest's sha256. Carry that apart from the bundle — read it\n" +
|
||||
"out over the phone, write it in the ticket — and give it to 'bundle import':\n" +
|
||||
"it is what says the bundle that arrived is the one that was made.\n\n" +
|
||||
"vLLM and SGLang are Python environments from PyPI, and no bundle carries\n" +
|
||||
"them; offline nodes serve with llama.cpp, or have them installed by hand.",
|
||||
Example: ` llmgw bundle create --out llmgw-bundle --arch amd64 --tar
|
||||
llmgw bundle create --out site-b --engine llamacpp=b10991 --variant cpu --variant cuda-13.3 \
|
||||
--model qwen2.5-14b-instruct/2026-10=hf:Qwen/Qwen2.5-14B-Instruct-GGUF/qwen2.5-14b-instruct-q4_k_m.gguf`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if out == "" {
|
||||
return errors.New("--out: the directory to make the bundle in")
|
||||
}
|
||||
o := bundle.CreateOptions{Out: out, Arches: arches, Releases: releases, Registry: registry}
|
||||
switch {
|
||||
case noRelease:
|
||||
case rel != "":
|
||||
o.Release = rel
|
||||
case release.Tag.MatchString(version.Version):
|
||||
o.Release = version.Version
|
||||
default:
|
||||
return fmt.Errorf("this llmgw is %s, which has no release to carry: give --version v0.x.y, "+
|
||||
"or --no-release to carry none", version.Version)
|
||||
}
|
||||
if len(engines) == 0 {
|
||||
engines = []string{"llamacpp=" + config.DefaultEngines["llamacpp"]}
|
||||
}
|
||||
for _, e := range engines {
|
||||
name, v, ok := strings.Cut(e, "=")
|
||||
if !ok || name == "" || v == "" {
|
||||
return fmt.Errorf("--engine %q: engine=version, llamacpp=%s", e, config.DefaultEngines["llamacpp"])
|
||||
}
|
||||
o.Engines = append(o.Engines, bundle.EngineSpec{Engine: name, Version: v, Variants: variants})
|
||||
}
|
||||
if !noStarter {
|
||||
o.Models = append(o.Models,
|
||||
bundle.ModelSpec{Name: starterName, Version: starterVersion, Source: starterSource},
|
||||
bundle.ModelSpec{Name: guardModelName, Version: guardModelVersion, Source: guardModelSource})
|
||||
}
|
||||
for _, m := range models {
|
||||
spec, err := parseModelSpec(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.Models = append(o.Models, spec)
|
||||
}
|
||||
w := cmd.OutOrStdout()
|
||||
o.Log = func(format string, args ...any) { fmt.Fprintf(w, format+"\n", args...) }
|
||||
|
||||
m, digest, err := bundle.Create(cmd.Context(), o)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
where := out
|
||||
if pack {
|
||||
where = strings.TrimSuffix(out, "/") + ".tar"
|
||||
fmt.Fprintf(w, "packing it into %s\n", where)
|
||||
if err := bundle.Pack(out, where); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(out); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
printManifest(w, m)
|
||||
fmt.Fprintf(w, "\nThe bundle is %s. Its sha256, to carry apart from it:\n\n %s\n\n", where, digest)
|
||||
fmt.Fprintf(w, "At the site, on the machine with the mirror (and the control plane):\n\n"+
|
||||
" sudo llmgw bundle import %s --sha256 %s\n\n"+
|
||||
"or, installing a control plane from it:\n\n"+
|
||||
" sudo llmgw server init --bundle %s --bundle-sha256 %s\n", where, digest, where, digest)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&out, "out", "", "the directory to make the bundle in; added to when it holds one already")
|
||||
f.BoolVar(&pack, "tar", false, "pack the bundle into <out>.tar, one file to carry, and remove the directory")
|
||||
f.StringVar(&rel, "version", "", "llmgw's release to carry (default: this llmgw's own)")
|
||||
f.BoolVar(&noRelease, "no-release", false, "carry no llmgw release")
|
||||
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
|
||||
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
|
||||
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
|
||||
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)")
|
||||
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
|
||||
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
|
||||
f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face")
|
||||
return c
|
||||
}
|
||||
|
||||
// parseModelSpec reads name/version=source.
|
||||
func parseModelSpec(s string) (bundle.ModelSpec, error) {
|
||||
where, source, ok := strings.Cut(s, "=")
|
||||
name, ver, ok2 := strings.Cut(where, "/")
|
||||
if !ok || !ok2 || name == "" || ver == "" || source == "" {
|
||||
return bundle.ModelSpec{}, fmt.Errorf("--model %q: name/version=hf:owner/repo/file", s)
|
||||
}
|
||||
return bundle.ModelSpec{Name: name, Version: ver, Source: source}, nil
|
||||
}
|
||||
|
||||
func newBundleImportCmd() *cobra.Command {
|
||||
var (
|
||||
dir, digest, trust, serverCfg string
|
||||
unchecked, noTrust bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "import PATH",
|
||||
Short: "Put a bundle into this machine's mirror",
|
||||
Long: "Checks every file of the bundle — against its manifest, and llmgw's release\n" +
|
||||
"against the project's signing key — and only then puts them into the mirror:\n" +
|
||||
"releases and engines under _llmgw/, models where the mirror keeps models.\n" +
|
||||
"A file the mirror has already, the same, is kept; a different one under the\n" +
|
||||
"same version is refused, because a version is never replaced.\n\n" +
|
||||
"On a control plane (server.yaml is there) it also records the engines'\n" +
|
||||
"checksums in offline.checksums, which the nodes check each engine file they\n" +
|
||||
"fetch against. When the mirror is on another machine, import there, and\n" +
|
||||
"run 'llmgw bundle trust' with the same bundle on the control plane.\n\n" +
|
||||
"PATH is the bundle's directory or its .tar.",
|
||||
Example: ` sudo llmgw bundle import /media/usb/llmgw-bundle.tar --sha256 3f2a…`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
w := cmd.OutOrStdout()
|
||||
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
|
||||
return fmt.Errorf("%s is not the mirror's directory: give --dir, or set the mirror up first "+
|
||||
"(sudo llmgw mirror init, or server init)", dir)
|
||||
}
|
||||
fmt.Fprintf(w, "checking %s and putting it into the mirror at %s\n", args[0], dir)
|
||||
res, err := bundle.Import(args[0], bundle.ImportOptions{
|
||||
Mirror: dir, SHA256: digest, Unchecked: unchecked,
|
||||
Progress: func(f bundle.File, kept bool) {
|
||||
if kept {
|
||||
fmt.Fprintf(w, " kept %s\n", f.Path)
|
||||
} else {
|
||||
fmt.Fprintf(w, " ok %s (%s)\n", f.Path, fileSize(f.Bytes))
|
||||
}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if unchecked {
|
||||
fmt.Fprintf(w, "\nTaken unchecked: its manifest's sha256 is %s; compare it with what was made.\n", res.Digest)
|
||||
}
|
||||
fmt.Fprintf(w, "\n%d files put in the mirror (%s), %d already there.\n", res.Written, fileSize(res.Bytes), res.Kept)
|
||||
if noTrust {
|
||||
return nil
|
||||
}
|
||||
cfg, haveCfg := loadServerConfig(serverCfg)
|
||||
if trust == "" && !haveCfg {
|
||||
fmt.Fprintf(w, "\nThis machine is not a control plane. On the control plane:\n\n"+
|
||||
" sudo llmgw bundle trust %s --sha256 %s\n", args[0], res.Digest)
|
||||
return nil
|
||||
}
|
||||
if trust == "" {
|
||||
trust = cfg.Offline.ChecksumsFile()
|
||||
}
|
||||
n, err := bundle.TrustManifest(res.Manifest, trust)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording the engines' checksums in %s: %w", trust, err)
|
||||
}
|
||||
fmt.Fprintf(w, "The control plane trusts %d engine files (%s).\n", n, trust)
|
||||
offlineHint(w, cfg, haveCfg)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&dir, "dir", mirrorDir, "the mirror's directory")
|
||||
f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it")
|
||||
f.BoolVar(&unchecked, "unchecked", false, "take the bundle without its sha256 (its files are still checked against its manifest)")
|
||||
f.StringVar(&trust, "trust", "", "where to record the engines' checksums (default: offline.checksums, on a control plane)")
|
||||
f.BoolVar(&noTrust, "no-trust", false, "do not record the engines' checksums here")
|
||||
f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, when this is one")
|
||||
return c
|
||||
}
|
||||
|
||||
func newBundleTrustCmd() *cobra.Command {
|
||||
var digest, file, serverCfg string
|
||||
var unchecked bool
|
||||
c := &cobra.Command{
|
||||
Use: "trust PATH",
|
||||
Short: "Have this control plane trust a bundle's engine checksums",
|
||||
Long: "Records the checksum of every engine file in the bundle in offline.checksums,\n" +
|
||||
"where the control plane reads what it tells each node a file from the\n" +
|
||||
"mirror must be. Only the manifest is read, so this is the step for a control\n" +
|
||||
"plane whose mirror is on another machine: import the bundle there, and run\n" +
|
||||
"this here with the same bundle and sha256. The control plane sees the new\n" +
|
||||
"checksums without a restart.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
w := cmd.OutOrStdout()
|
||||
cfg, haveCfg := loadServerConfig(serverCfg)
|
||||
if file == "" {
|
||||
file = cfg.Offline.ChecksumsFile()
|
||||
}
|
||||
m, n, err := bundle.Trust(args[0], file, digest, unchecked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printManifest(w, m)
|
||||
fmt.Fprintf(w, "\nThe control plane trusts %d engine files (%s).\n", n, file)
|
||||
offlineHint(w, cfg, haveCfg)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it")
|
||||
f.BoolVar(&unchecked, "unchecked", false, "take the manifest without its sha256")
|
||||
f.StringVar(&file, "file", "", "where to record them (default: offline.checksums from server.yaml, else "+config.DefaultOfflineChecksums+")")
|
||||
f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
|
||||
return c
|
||||
}
|
||||
|
||||
func newBundleShowCmd() *cobra.Command {
|
||||
var verify bool
|
||||
var digest string
|
||||
c := &cobra.Command{
|
||||
Use: "show PATH",
|
||||
Short: "Say what a bundle holds, and check it",
|
||||
Long: "Prints what the bundle carries and its sha256. With --verify every file is\n" +
|
||||
"read and checked, as an import would, and nothing is written: worth doing\n" +
|
||||
"after copying a bundle onto the disk that carries it.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
w := cmd.OutOrStdout()
|
||||
m, d, err := bundle.ReadManifest(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printManifest(w, m)
|
||||
fmt.Fprintf(w, "\nsha256 %s\n", d)
|
||||
if !verify {
|
||||
return nil
|
||||
}
|
||||
res, err := bundle.Import(args[0], bundle.ImportOptions{SHA256: digest, Unchecked: digest == ""})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(w, "Every file is as it was made (%d files, %s).\n", res.Written, fileSize(res.Bytes))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&verify, "verify", false, "read and check every file")
|
||||
c.Flags().StringVar(&digest, "sha256", "", "with --verify, the sha256 the bundle should have")
|
||||
return c
|
||||
}
|
||||
|
||||
// printManifest says what a bundle carries.
|
||||
func printManifest(w io.Writer, m bundle.Manifest) {
|
||||
var total int64
|
||||
for _, f := range m.Files {
|
||||
total += f.Bytes
|
||||
}
|
||||
fmt.Fprintf(w, "\nA bundle made %s by llmgw %s: %d files, %s.\n",
|
||||
m.Created.Format("2006-01-02 15:04 UTC"), m.MadeBy, len(m.Files), fileSize(total))
|
||||
for _, r := range m.Releases {
|
||||
fmt.Fprintf(w, " llmgw %s for %s\n", r.Version, strings.Join(r.Arches, ", "))
|
||||
}
|
||||
for _, e := range m.Engines {
|
||||
fmt.Fprintf(w, " %-9s %s for %s: %s\n", e.Engine, e.Version, strings.Join(e.Arches, ", "), strings.Join(e.Variants, ", "))
|
||||
}
|
||||
for _, md := range m.Models {
|
||||
at := ""
|
||||
if md.Revision != "" {
|
||||
at = " at " + md.Revision[:min(len(md.Revision), 8)]
|
||||
}
|
||||
fmt.Fprintf(w, " model %s/%s from %s%s\n", md.Name, md.Version, md.Source, at)
|
||||
}
|
||||
}
|
||||
|
||||
// loadServerConfig reads a control plane's configuration, when this
|
||||
// machine is one.
|
||||
func loadServerConfig(path string) (config.Server, bool) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return config.Server{}, false
|
||||
}
|
||||
cfg, err := config.Load(path)
|
||||
if err != nil {
|
||||
return config.Server{}, false
|
||||
}
|
||||
return cfg, true
|
||||
}
|
||||
|
||||
// offlineHint says what is left once a bundle is trusted.
|
||||
func offlineHint(w io.Writer, cfg config.Server, haveCfg bool) {
|
||||
if haveCfg && !cfg.Offline.Enabled() {
|
||||
fmt.Fprintf(w, "\nserver.yaml does not yet say the cluster is offline. Add the mirror's file\n"+
|
||||
"address, as nodes reach it, and restart the control plane:\n\n"+
|
||||
" offline:\n mirror: http://<mirror>:8081\n\n"+
|
||||
" sudo systemctl restart llmgw-server\n")
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,16 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
ctx := cmd.Context()
|
||||
if releases == "" {
|
||||
// Where the control plane sends its nodes: an offline
|
||||
// cluster's mirror, so this machine needs no internet either.
|
||||
if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok {
|
||||
releases = cfg.ReleasesBase()
|
||||
}
|
||||
if releases == "" {
|
||||
releases = release.DefaultBase
|
||||
}
|
||||
}
|
||||
if !skipCP {
|
||||
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
|
||||
return err
|
||||
@@ -60,7 +70,8 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
|
||||
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
|
||||
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
|
||||
c.Flags().StringVar(&releases, "releases", "", "where release binaries are fetched from, for this machine "+
|
||||
"(default: as server.yaml says, the mirror's copy on an offline cluster, else the project's)")
|
||||
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
|
||||
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
|
||||
return c
|
||||
|
||||
@@ -53,9 +53,15 @@ func newJoinCmd() *cobra.Command {
|
||||
// screen that cuts long ones off.
|
||||
func joinInstall() []string {
|
||||
v := version.Version
|
||||
// From where this cluster takes its releases: on an offline one, its
|
||||
// mirror, which the new machine can reach when the internet is not.
|
||||
base := release.DefaultBase
|
||||
if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok && cfg.ReleasesBase() != "" {
|
||||
base = cfg.ReleasesBase()
|
||||
}
|
||||
return []string{
|
||||
`a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`,
|
||||
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v),
|
||||
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", base, v, v),
|
||||
"sudo chmod 755 /usr/local/bin/llmgw",
|
||||
}
|
||||
}
|
||||
+12
-1
@@ -17,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
@@ -120,7 +121,7 @@ engines are kept.`,
|
||||
f.StringVar(&na.opts.NvidiaSMI, "nvidia-smi", "", "where nvidia-smi is, when it is not on PATH")
|
||||
f.StringVar(&na.opts.IBRoot, "ib-root", "", "the sysfs InfiniBand directory, when not /sys/class/infiniband")
|
||||
f.StringVar(&na.opts.PKI, "pki", defaultPKIDir, "directory holding the certificate authority")
|
||||
f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
|
||||
f.StringVar(&na.opts.Releases, "releases", "", "where release files are downloaded from (default: releases in server.yaml, the mirror's copy on an offline cluster, else the project's)")
|
||||
f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
|
||||
f.BoolVar(&na.opts.NoNodeExporter, "no-node-exporter", false, "leave Prometheus' node exporter off the machine")
|
||||
f.BoolVar(&na.opts.NoFirewall, "no-firewall", false, "leave the machine's firewall alone, instead of keeping llmgw's ports to the cluster's networks")
|
||||
@@ -176,6 +177,16 @@ func (na *nodeAdd) prepare() error {
|
||||
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
|
||||
o.FirewallAllow, _ = cfg.FirewallAllow()
|
||||
}
|
||||
// Where the control plane upgrades nodes from, so a node joins on the
|
||||
// same release from the same place: an offline cluster's mirror.
|
||||
if o.Releases == "" {
|
||||
if cfg, err := config.Load(na.serverCfg); err == nil {
|
||||
o.Releases = cfg.ReleasesBase()
|
||||
}
|
||||
if o.Releases == "" {
|
||||
o.Releases = release.DefaultBase
|
||||
}
|
||||
}
|
||||
o.Version = version.Version
|
||||
if o.Binary == "" {
|
||||
exe, err := os.Executable()
|
||||
|
||||
@@ -52,6 +52,7 @@ func Root() *cobra.Command {
|
||||
newUsersCmd(),
|
||||
newOrgCmd(),
|
||||
newMirrorCmd(),
|
||||
newBundleCmd(),
|
||||
newSimEngineCmd(),
|
||||
newDoctorCmd(),
|
||||
newNodeCmd(),
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
@@ -103,6 +104,12 @@ func newServerCmd() *cobra.Command {
|
||||
return errors.New("nothing to serve: declare models with profiles, " +
|
||||
"or give --endpoint for a router-only run")
|
||||
}
|
||||
// Offline, nothing is asked of the internet: releases come from
|
||||
// the mirror's copy, and no registry is asked about newer weights.
|
||||
if cfg.Offline.Enabled() {
|
||||
cfg.Releases = cfg.ReleasesBase()
|
||||
cfg.Mirror.UpdateCheck = 0
|
||||
}
|
||||
|
||||
// Before the router, so its tracer is the configured one. The
|
||||
// host tells two control planes' traces apart.
|
||||
@@ -685,6 +692,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels)
|
||||
})
|
||||
log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides))
|
||||
if cfg.Offline.Enabled() {
|
||||
// From the mirror, each file held to the checksum the bundle gave,
|
||||
// which this control plane keeps and the mirror has no say in.
|
||||
trusted := bundle.NewTrusted(cfg.Offline.ChecksumsFile())
|
||||
svc.SetOffline(func(engine, version string) (string, map[string]string) {
|
||||
return bundle.EngineBase(cfg.Offline.Mirror, engine, version), trusted.For(engine, version)
|
||||
})
|
||||
if v, ok := cfg.WantedEngines()["llamacpp"]; ok && len(trusted.For("llamacpp", v)) == 0 {
|
||||
log.Warn("offline, and no bundle imported here carries llama.cpp at this version: nodes cannot "+
|
||||
"install it until one does (llmgw bundle create --engine llamacpp="+v+", then bundle import or trust)",
|
||||
"version", v, "checksums", cfg.Offline.ChecksumsFile(), "err", trusted.Err())
|
||||
}
|
||||
log.Info("offline: nodes install engines from the mirror", "mirror", cfg.Offline.Mirror,
|
||||
"releases", cfg.Releases)
|
||||
}
|
||||
if ch := ctl.EngineChoices(); ch != nil {
|
||||
svc.SetEngineChoices(ch.For)
|
||||
ch.OnChange(svc.Reprovision)
|
||||
|
||||
@@ -89,6 +89,12 @@ type serverInit struct {
|
||||
noFirewall bool
|
||||
// mirrorURL is a mirror on another machine to use instead of running one.
|
||||
mirrorURL string
|
||||
// bundle is an offline bundle to install from, and bundleSHA its
|
||||
// manifest's sha256: the cluster is then set up offline.
|
||||
bundle, bundleSHA string
|
||||
// mirrorPath is the mirror's directory here; "" is /srv/models. A seam
|
||||
// for tests.
|
||||
mirrorPath string
|
||||
// pgSetup prepares PostgreSQL and returns the store address; a seam for
|
||||
// tests. Nil does it for real.
|
||||
pgSetup func(context.Context) (string, error)
|
||||
@@ -165,6 +171,9 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.BoolVar(&in.noMirror, "no-mirror", false, "do not run a model mirror on this machine")
|
||||
f.StringVar(&in.mirrorURL, "mirror", "",
|
||||
"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
|
||||
f.StringVar(&in.bundle, "bundle", "",
|
||||
"install offline from this bundle ('llmgw bundle create'): into the mirror, with the cluster set to reach nothing outside")
|
||||
f.StringVar(&in.bundleSHA, "bundle-sha256", "", "the bundle's sha256, as 'llmgw bundle create' printed it")
|
||||
f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
|
||||
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
|
||||
f.BoolVar(&in.noGuardModel, "no-guard-model", false,
|
||||
@@ -180,6 +189,9 @@ func newServerInitCmd() *cobra.Command {
|
||||
}
|
||||
|
||||
func (in *serverInit) run(ctx context.Context) error {
|
||||
if err := in.checkBundle(); err != nil {
|
||||
return fmt.Errorf("offline bundle: %w", err)
|
||||
}
|
||||
steps := []struct {
|
||||
name string
|
||||
fn func(context.Context) error
|
||||
@@ -214,6 +226,13 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
return fmt.Errorf("mirror: %w", err)
|
||||
}
|
||||
}
|
||||
// Into the mirror, and server.yaml told, before the service starts:
|
||||
// the control plane reads at start that the cluster is offline.
|
||||
if in.bundle != "" {
|
||||
if err := in.ensureBundle(ctx); err != nil {
|
||||
return fmt.Errorf("offline bundle: %w", err)
|
||||
}
|
||||
}
|
||||
if err := in.ensureService(ctx); err != nil {
|
||||
return fmt.Errorf("service: %w", err)
|
||||
}
|
||||
@@ -256,6 +275,11 @@ func (in *serverInit) judgeWanted() bool {
|
||||
if in.noGuardModel || (in.noMirror && in.mirrorURL == "") || in.unitPath == "" {
|
||||
return false
|
||||
}
|
||||
// Offline, only a judge the bundle carries: a policy asking a judge
|
||||
// that never comes refuses every request.
|
||||
if in.bundle != "" && !in.bundleCarries(guardModelName) {
|
||||
return false
|
||||
}
|
||||
_, err := exec.LookPath("systemctl")
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// --- installing from an offline bundle ---
|
||||
|
||||
// checkBundle refuses, before anything is done, a bundle init could not
|
||||
// install from: one that is not there, or not the one that was made.
|
||||
func (in *serverInit) checkBundle() error {
|
||||
if in.bundle == "" {
|
||||
return nil
|
||||
}
|
||||
if in.noMirror && in.mirrorURL == "" {
|
||||
return errors.New("a bundle goes into the cluster's mirror: leave out --no-mirror, or give --mirror")
|
||||
}
|
||||
if in.bundleSHA == "" {
|
||||
return errors.New("give --bundle-sha256, the sha256 'llmgw bundle create' printed: " +
|
||||
"it is what says the bundle that arrived is the one that was made")
|
||||
}
|
||||
_, digest, err := bundle.ReadManifest(in.bundle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if digest != in.bundleSHA {
|
||||
return fmt.Errorf("the bundle's sha256 is %s, not the %s given: it is not the bundle that was made, "+
|
||||
"or the digest was mistyped", digest, in.bundleSHA)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bundleCarries says whether the bundle holds a model of this name.
|
||||
func (in *serverInit) bundleCarries(model string) bool {
|
||||
m, _, err := bundle.ReadManifest(in.bundle)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, md := range m.Models {
|
||||
if md.Name == model {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// offline says whether this cluster is set up to reach nothing outside.
|
||||
func (in *serverInit) offline() bool {
|
||||
if in.bundle != "" {
|
||||
return true
|
||||
}
|
||||
cfg, err := config.Load(in.configPath())
|
||||
return err == nil && cfg.Offline.Enabled()
|
||||
}
|
||||
|
||||
func (in *serverInit) mirrorRoot() string {
|
||||
if in.mirrorPath != "" {
|
||||
return in.mirrorPath
|
||||
}
|
||||
return mirrorDir
|
||||
}
|
||||
|
||||
// ensureBundle puts the bundle into the mirror beside this control plane,
|
||||
// has the control plane trust its engines' checksums, and says in
|
||||
// server.yaml that the cluster is offline. With a mirror on another
|
||||
// machine, the files are imported there; here only the checksums are kept.
|
||||
func (in *serverInit) ensureBundle(ctx context.Context) error {
|
||||
cfg, err := config.Load(in.configPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var (
|
||||
m bundle.Manifest
|
||||
files string
|
||||
)
|
||||
if in.mirrorURL == "" {
|
||||
res, err := bundle.Import(in.bundle, bundle.ImportOptions{Mirror: in.mirrorRoot(), SHA256: in.bundleSHA})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.Written > 0 {
|
||||
in.done("offline bundle in the mirror: %d files, %s, each checked", res.Written, fileSize(res.Bytes))
|
||||
} else {
|
||||
in.kept("offline bundle in the mirror (%d files)", res.Kept)
|
||||
}
|
||||
m = res.Manifest
|
||||
files = "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
|
||||
} else {
|
||||
mc, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 10*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
list, err := mc.List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asking the mirror at %s where nodes reach it: %w", in.mirrorURL, err)
|
||||
}
|
||||
files = list.Base
|
||||
if m, _, err = bundle.ReadManifest(in.bundle); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(in.out, " !! the mirror is on another machine: import the bundle there too, "+
|
||||
"sudo llmgw bundle import <bundle> --sha256 %s\n", in.bundleSHA)
|
||||
}
|
||||
sums := cfg.Offline.Checksums
|
||||
if sums == "" {
|
||||
sums = filepath.Join(in.etcDir, "offline.sha256")
|
||||
}
|
||||
n, err := bundle.TrustManifest(m, sums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("the control plane trusts %d engine files from the bundle (%s)", n, sums)
|
||||
|
||||
if cfg.Offline.Enabled() {
|
||||
in.kept("offline, installing from %s, as server.yaml says", cfg.Offline.Mirror)
|
||||
return nil
|
||||
}
|
||||
if sums == config.DefaultOfflineChecksums {
|
||||
sums = ""
|
||||
}
|
||||
if err := appendOfflineBlock(in.configPath(), files, sums); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("server.yaml: offline, nodes install from the mirror at %s", files)
|
||||
return nil
|
||||
}
|
||||
|
||||
// appendOfflineBlock says in server.yaml that the cluster is offline,
|
||||
// without touching the rest.
|
||||
func appendOfflineBlock(path, files, sums string) error {
|
||||
b, err := os.ReadFile(path) //nolint:gosec // server.yaml
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
block := "\n# Offline: nothing in the cluster reaches the internet. Nodes install\n" +
|
||||
"# engines, and upgrades fetch releases, from the mirror, out of the bundle\n" +
|
||||
"# 'llmgw server init --bundle' imported; 'llmgw bundle import' adds more.\n" +
|
||||
"offline:\n mirror: " + files + "\n"
|
||||
if sums != "" {
|
||||
block += " checksums: " + sums + "\n"
|
||||
}
|
||||
return os.WriteFile(path, append(b, []byte(block)...), info.Mode().Perm())
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
)
|
||||
|
||||
// handMadeBundle is a bundle as 'bundle create' leaves it: an engine build
|
||||
// and the starter model, with their checksums in the manifest.
|
||||
func handMadeBundle(t *testing.T) (dir, digest string) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
m := bundle.Manifest{Format: bundle.Format, MadeBy: "test",
|
||||
Models: []bundle.Model{{Name: starterName, Version: starterVersion, Source: starterSource}}}
|
||||
for p, body := range map[string]string{
|
||||
"engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz": "an engine",
|
||||
"models/" + starterName + "/" + starterVersion + "/qwen-q4_k_m.gguf": "weights",
|
||||
} {
|
||||
full := filepath.Join(dir, p)
|
||||
_ = os.MkdirAll(filepath.Dir(full), 0o755)
|
||||
_ = os.WriteFile(full, []byte(body), 0o644)
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
m.Files = append(m.Files, bundle.File{Path: p, Bytes: int64(len(body)), SHA256: hex.EncodeToString(sum[:])})
|
||||
}
|
||||
raw, _ := json.Marshal(m)
|
||||
_ = os.WriteFile(filepath.Join(dir, bundle.ManifestName), raw, 0o644)
|
||||
return dir, bundle.Digest(raw)
|
||||
}
|
||||
|
||||
func TestInitFromABundleLeavesAnOfflineClusterThatTrustsItsEngines(t *testing.T) {
|
||||
in, out := newTestInit(t)
|
||||
dir, digest := handMadeBundle(t)
|
||||
in.bundle, in.hosts, in.mirrorPath = dir, []string{"10.9.8.7"}, t.TempDir()
|
||||
|
||||
in.bundleSHA = ""
|
||||
if err := in.checkBundle(); err == nil || !strings.Contains(err.Error(), "--bundle-sha256") {
|
||||
t.Errorf("a bundle with no digest: %v", err)
|
||||
}
|
||||
in.bundleSHA = strings.Repeat("0", 64)
|
||||
if err := in.checkBundle(); err == nil {
|
||||
t.Error("a bundle not matching its digest was taken")
|
||||
}
|
||||
in.bundleSHA = digest
|
||||
if err := in.run(context.Background()); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if err := in.ensureBundle(context.Background()); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(in.configPath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Offline.Mirror != "http://10.9.8.7:8081" || cfg.ReleasesBase() != "http://10.9.8.7:8081/_llmgw/releases" {
|
||||
t.Errorf("server.yaml: offline %+v, releases %s", cfg.Offline, cfg.ReleasesBase())
|
||||
}
|
||||
got := bundle.NewTrusted(cfg.Offline.ChecksumsFile()).For("llamacpp", "b1")
|
||||
if len(got) != 1 {
|
||||
t.Errorf("the control plane trusts %v", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(in.mirrorPath, starterName, starterVersion, "qwen-q4_k_m.gguf")); err != nil {
|
||||
t.Errorf("the starter model is not in the mirror: %v", err)
|
||||
}
|
||||
if !in.bundleCarries(starterName) || in.bundleCarries(guardModelName) {
|
||||
t.Error("what the bundle carries was misread")
|
||||
}
|
||||
|
||||
// Again, and nothing changes: not the mirror, not server.yaml.
|
||||
before, _ := os.ReadFile(in.configPath())
|
||||
if err := in.ensureBundle(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after, _ := os.ReadFile(in.configPath()); string(after) != string(before) {
|
||||
t.Error("init from the same bundle changed server.yaml again")
|
||||
}
|
||||
}
|
||||
@@ -516,6 +516,11 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st
|
||||
var apiErr *mirror.APIError
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.As(err, &apiErr) && apiErr.Status == 404 && in.offline():
|
||||
// Asked of the internet, it would fail after a timeout and stop init
|
||||
// over a model nobody carried in.
|
||||
fmt.Fprintf(in.out, " skip %s: not in the mirror, and an offline cluster does not download it\n", name)
|
||||
return nil
|
||||
case errors.As(err, &apiErr) && apiErr.Status == 404:
|
||||
if st, err = m.Fetch(ctx, mirror.FetchRequest{
|
||||
Source: upstream, Model: name, Version: version,
|
||||
|
||||
@@ -55,6 +55,8 @@ const (
|
||||
fNode
|
||||
fModel
|
||||
fGuard
|
||||
fBundle
|
||||
fBundleSHA
|
||||
fStart
|
||||
fCount
|
||||
)
|
||||
@@ -68,6 +70,8 @@ type initModel struct {
|
||||
focus int
|
||||
admin textField
|
||||
hosts textField
|
||||
bundle textField
|
||||
bundleSHA textField
|
||||
toggles [fCount]bool
|
||||
formError string
|
||||
|
||||
@@ -99,8 +103,19 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel {
|
||||
hosts.SetValue(strings.Join(in.hosts, ", "))
|
||||
hosts.CharLimit = 400
|
||||
hosts.Width = 40
|
||||
bundle := newTextField()
|
||||
bundle.Placeholder = "/media/usb/llmgw-bundle.tar"
|
||||
bundle.SetValue(in.bundle)
|
||||
bundle.CharLimit = 400
|
||||
bundle.Width = 40
|
||||
bundleSHA := newTextField()
|
||||
bundleSHA.Placeholder = "the sha256 'llmgw bundle create' printed"
|
||||
bundleSHA.SetValue(in.bundleSHA)
|
||||
bundleSHA.CharLimit = 64
|
||||
bundleSHA.Width = 64
|
||||
sp := newSpinner()
|
||||
m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, spin: sp, width: 100}
|
||||
m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts,
|
||||
bundle: bundle, bundleSHA: bundleSHA, spin: sp, width: 100}
|
||||
m.toggles[fPostgres] = in.postgres
|
||||
m.toggles[fMirror] = !in.noMirror
|
||||
m.toggles[fNode] = !in.noNode
|
||||
@@ -207,6 +222,20 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
bundle, sum := strings.TrimSpace(m.bundle.Value()), strings.ToLower(strings.TrimSpace(m.bundleSHA.Value()))
|
||||
switch {
|
||||
case bundle != "" && sum == "":
|
||||
m.formError = "give the bundle's sha256, as 'llmgw bundle create' printed it"
|
||||
m.focus = fBundleSHA
|
||||
m.moveFocus(0)
|
||||
return m, nil
|
||||
case bundle != "" && !m.toggles[fMirror] && m.in.mirrorURL == "":
|
||||
m.formError = "an offline bundle goes into the model mirror: tick it"
|
||||
m.focus = fMirror
|
||||
m.moveFocus(0)
|
||||
return m, nil
|
||||
}
|
||||
m.in.bundle, m.in.bundleSHA = bundle, sum
|
||||
m.in.admin = admin
|
||||
m.in.hosts = nil
|
||||
for _, h := range strings.Split(m.hosts.Value(), ",") {
|
||||
@@ -231,6 +260,14 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
var cmd tea.Cmd
|
||||
m.hosts, cmd = m.hosts.Update(k)
|
||||
return m, cmd
|
||||
case fBundle:
|
||||
var cmd tea.Cmd
|
||||
m.bundle, cmd = m.bundle.Update(k)
|
||||
return m, cmd
|
||||
case fBundleSHA:
|
||||
var cmd tea.Cmd
|
||||
m.bundleSHA, cmd = m.bundleSHA.Update(k)
|
||||
return m, cmd
|
||||
case fPostgres, fMirror, fNode, fModel, fGuard:
|
||||
if k.String() == " " || k.String() == "x" {
|
||||
m.toggles[m.focus] = !m.toggles[m.focus]
|
||||
@@ -247,11 +284,17 @@ func (m *initModel) moveFocus(d int) {
|
||||
}
|
||||
m.admin.Blur()
|
||||
m.hosts.Blur()
|
||||
m.bundle.Blur()
|
||||
m.bundleSHA.Blur()
|
||||
switch m.focus {
|
||||
case fAdmin:
|
||||
m.admin.Focus()
|
||||
case fHosts:
|
||||
m.hosts.Focus()
|
||||
case fBundle:
|
||||
m.bundle.Focus()
|
||||
case fBundleSHA:
|
||||
m.bundleSHA.Focus()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -375,6 +418,9 @@ func (m initModel) formView() string {
|
||||
b.WriteString(check(fNode, "This machine as a node", "serves models here too, on its GPUs or CPU"))
|
||||
b.WriteString(check(fModel, "Starter model", "a small Qwen, so there is something to ask at once"))
|
||||
b.WriteString(check(fGuard, "Guardrails' judge", "Llama Guard 3 1B on a CPU, refusing violent, criminal and abusive requests"))
|
||||
fmt.Fprintf(&b, "\n%s\n %s\n", label(fBundle, "Offline bundle"), m.bundle.View())
|
||||
b.WriteString(" " + sDim.Render("For a site with no internet: engines, releases and models from 'llmgw bundle create'. Empty downloads them.") + "\n")
|
||||
fmt.Fprintf(&b, "%s\n %s\n", label(fBundleSHA, "Its sha256"), m.bundleSHA.View())
|
||||
b.WriteString("\n" + label(fStart, lipgloss.NewStyle().Bold(true).Render("Set up")) + "\n")
|
||||
if m.formError != "" {
|
||||
b.WriteString("\n" + sBad.Render(m.formError) + "\n")
|
||||
|
||||
@@ -23,8 +23,9 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab},
|
||||
tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}) // no bundle
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyEnter})
|
||||
|
||||
if in.admin != "ops@example.com" || strings.Join(in.hosts, "|") != "llm.example.com|203.0.113.10" {
|
||||
t.Errorf("admin %q hosts %q", in.admin, in.hosts)
|
||||
@@ -37,6 +38,27 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOfflineBundleIsGivenInTheFormWithItsDigest(t *testing.T) {
|
||||
in := &serverInit{}
|
||||
var m tea.Model = newInitModel(in, nil, true)
|
||||
if v := m.View(); !strings.Contains(v, "Offline bundle") {
|
||||
t.Fatalf("the form offers no bundle:\n%s", v)
|
||||
}
|
||||
for range fBundle {
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab})
|
||||
}
|
||||
m = typeIn(m, "/media/usb/b.tar")
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
|
||||
if !m.(initModel).form || !strings.Contains(m.View(), "sha256") || m.(initModel).focus != fBundleSHA {
|
||||
t.Fatalf("a bundle without its digest was accepted:\n%s", m.View())
|
||||
}
|
||||
m = typeIn(m, strings.Repeat("AB", 32))
|
||||
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
|
||||
if m.(initModel).form || in.bundle != "/media/usb/b.tar" || in.bundleSHA != strings.Repeat("ab", 32) {
|
||||
t.Errorf("bundle %q sha %q, form up %v", in.bundle, in.bundleSHA, m.(initModel).form)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmailThatIsNotOneIsSaidSo(t *testing.T) {
|
||||
in := &serverInit{}
|
||||
var m tea.Model = newInitModel(in, nil, true)
|
||||
|
||||
@@ -105,6 +105,11 @@ type Server struct {
|
||||
// Empty means there is none, and /admin/mirror stays unmounted.
|
||||
Mirror Mirror `koanf:"mirror"`
|
||||
|
||||
// Offline is a cluster that reaches nothing outside itself: its nodes
|
||||
// install engines, and upgrades fetch releases, from the mirror, out of
|
||||
// a bundle carried in (llmgw bundle create, llmgw bundle import).
|
||||
Offline Offline `koanf:"offline"`
|
||||
|
||||
// Engines are the engines every node installs, and at which version. The
|
||||
// agents install them themselves — the right build for their hardware,
|
||||
// checked against its publisher's checksum and seen to run — so a node is
|
||||
@@ -777,6 +782,9 @@ func (s Server) Validate() error {
|
||||
if err := s.AgentTLS.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Offline.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Auth.Enabled && s.Auth.Store == "" {
|
||||
return fmt.Errorf("auth.store is required when auth is enabled")
|
||||
@@ -925,3 +933,75 @@ func (s Server) WantedEngines() map[string]string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Offline is a cluster with no way to the internet.
|
||||
//
|
||||
// Everything it would otherwise fetch from outside — llmgw's releases, the
|
||||
// engine builds its nodes install, model files — is carried in as a bundle,
|
||||
// made where there is internet (llmgw bundle create) and imported into the
|
||||
// mirror here (llmgw bundle import). Nodes then fetch from the mirror, and
|
||||
// check what they fetch as strictly as they would online: releases against
|
||||
// the project's signing key, engines against the checksums the control plane
|
||||
// holds from the bundle.
|
||||
type Offline struct {
|
||||
// Mirror is the mirror's file address, as nodes reach it:
|
||||
// http://10.0.0.5:8081 — the port model files are served on, not the
|
||||
// management API. Set, the cluster is offline: nodes install engines
|
||||
// from it and never from their publishers, releases come from it unless
|
||||
// releases names another place, and no registry is asked whether a
|
||||
// model has newer weights. Python engines (vLLM, SGLang) come from PyPI,
|
||||
// which no bundle carries; on an offline node they are installed by hand.
|
||||
Mirror string `koanf:"mirror"`
|
||||
// Checksums is the file of engine checksums this control plane trusts,
|
||||
// kept by llmgw bundle import and llmgw bundle trust; every engine file
|
||||
// a node fetches from the mirror is checked against it. Read again
|
||||
// whenever it changes. Empty is /etc/llmgw/offline.sha256.
|
||||
Checksums string `koanf:"checksums"`
|
||||
}
|
||||
|
||||
// DefaultOfflineChecksums is where the engine checksums are kept when
|
||||
// offline.checksums does not say.
|
||||
const DefaultOfflineChecksums = "/etc/llmgw/offline.sha256"
|
||||
|
||||
// Enabled reports whether the cluster is offline.
|
||||
func (o Offline) Enabled() bool { return o.Mirror != "" }
|
||||
|
||||
// ChecksumsFile is the engine checksums' file.
|
||||
func (o Offline) ChecksumsFile() string {
|
||||
if o.Checksums != "" {
|
||||
return o.Checksums
|
||||
}
|
||||
return DefaultOfflineChecksums
|
||||
}
|
||||
|
||||
// ReleasesBase is where an offline cluster's releases are on its mirror.
|
||||
func (o Offline) ReleasesBase() string {
|
||||
return strings.TrimSuffix(o.Mirror, "/") + "/_llmgw/releases"
|
||||
}
|
||||
|
||||
// Validate refuses a mirror address nodes could not fetch from.
|
||||
func (o Offline) Validate() error {
|
||||
if o.Mirror == "" {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasPrefix(o.Mirror, "http://") && !strings.HasPrefix(o.Mirror, "https://") {
|
||||
return fmt.Errorf("offline.mirror %q: give the mirror's file address, http://host:8081", o.Mirror)
|
||||
}
|
||||
if strings.HasSuffix(strings.TrimSuffix(o.Mirror, "/"), ":8082") {
|
||||
return fmt.Errorf("offline.mirror %q is the mirror's management port: give the one nodes fetch files from, :8081", o.Mirror)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleasesBase is where release binaries are fetched from: releases when
|
||||
// set, the mirror's copy on an offline cluster, and empty — the caller's
|
||||
// default, the project's own releases — otherwise.
|
||||
func (s Server) ReleasesBase() string {
|
||||
if s.Releases != "" {
|
||||
return s.Releases
|
||||
}
|
||||
if s.Offline.Enabled() {
|
||||
return s.Offline.ReleasesBase()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -55,6 +55,23 @@ func TestEachNodeIsToldTheEnginesDecidedForIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOfflineClusterTellsNodesWhereOnTheMirrorAndWhatEachFileMustBe(t *testing.T) {
|
||||
s := &Service{}
|
||||
s.SetEngines(map[string]string{"llamacpp": "b1"})
|
||||
if w := s.provisionFor(inventory.Node{ID: "n"}, []string{"llamacpp"}).GetEngines()[0]; w.GetOfflineBase() != "" {
|
||||
t.Fatalf("an online cluster sent a mirror: %v", w)
|
||||
}
|
||||
s.SetOffline(func(engine, version string) (string, map[string]string) {
|
||||
return "http://mirror:8081/_llmgw/engines/" + engine + "/" + version,
|
||||
map[string]string{"llama-" + version + "-bin-ubuntu-x64.tar.gz": "abc"}
|
||||
})
|
||||
w := s.provisionFor(inventory.Node{ID: "n"}, []string{"llamacpp"}).GetEngines()[0]
|
||||
if w.GetOfflineBase() != "http://mirror:8081/_llmgw/engines/llamacpp/b1" ||
|
||||
w.GetOfflineSha256()["llama-b1-bin-ubuntu-x64.tar.gz"] != "abc" {
|
||||
t.Errorf("the node was told %v", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildChosenForANodeIsWhatItIsTold(t *testing.T) {
|
||||
s := &Service{}
|
||||
s.SetEngines(map[string]string{"llamacpp": "b1", "vllm": "0.30.0"})
|
||||
|
||||
@@ -66,6 +66,9 @@ type Service struct {
|
||||
// each connected node said it can install, for telling it again.
|
||||
choices func(nodeID string) map[string]string
|
||||
installableBy map[string][]string
|
||||
// offline, when set, is where engines come from on a cluster with no
|
||||
// way to the internet; see SetOffline.
|
||||
offline func(engine, version string) (string, map[string]string)
|
||||
|
||||
// controlPlanes lists what agents may dial; see SetControlPlanes.
|
||||
controlPlanes func(context.Context) ([]string, error)
|
||||
@@ -989,11 +992,24 @@ func (s *Service) provisionFor(node inventory.Node, can []string) *llmgwv1.Provi
|
||||
if !slices.Contains(can, e) || chosen[e] == "none" {
|
||||
continue
|
||||
}
|
||||
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]})
|
||||
w := &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]}
|
||||
if s.offline != nil {
|
||||
w.OfflineBase, w.OfflineSha256 = s.offline(e, wants[e])
|
||||
}
|
||||
p.Engines = append(p.Engines, w)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// SetOffline makes nodes install engines from the cluster's mirror rather
|
||||
// than from their publishers: f gives where an engine version's files are
|
||||
// there, and each file's checksum from the bundle the control plane trusts.
|
||||
// A version no bundle carried has no checksums, and the node says which
|
||||
// bundle to import rather than reaching for the internet.
|
||||
func (s *Service) SetOffline(f func(engine, version string) (base string, sha256 map[string]string)) {
|
||||
s.offline = f
|
||||
}
|
||||
|
||||
// installable are the engines a node can be asked to install. An agent older
|
||||
// than the list said nothing, and could install llama.cpp alone: asked for
|
||||
// vLLM too, it reported it unsupported, and a cluster half way through an
|
||||
|
||||
@@ -168,6 +168,10 @@ var Reference = []Block{
|
||||
{Key: "memberships", Type: "map[string]string", Doc: "Memberships maps a group to a place in the tree — org:NAME,\nteam:ORG/TEAM, project:ORG/TEAM/PROJECT — joined at sign-in.", Nested: ""},
|
||||
{Key: "no_passwords", Type: "bool", Doc: "PasswordSignIn off refuses passwords on the sign-in form: people come\nin through the provider only. Administrators still can, so a broken\nprovider does not lock everybody out.", Nested: ""},
|
||||
}},
|
||||
{Package: "config", Name: "Offline", Title: "Offline", Doc: "", Fields: []Field{
|
||||
{Key: "mirror", Type: "string", Doc: "Mirror is the mirror's file address, as nodes reach it:\nhttp://10.0.0.5:8081 — the port model files are served on, not the\nmanagement API. Set, the cluster is offline: nodes install engines\nfrom it and never from their publishers, releases come from it unless\nreleases names another place, and no registry is asked whether a\nmodel has newer weights. Python engines (vLLM, SGLang) come from PyPI,\nwhich no bundle carries; on an offline node they are installed by hand.", Nested: ""},
|
||||
{Key: "checksums", Type: "string", Doc: "Checksums is the file of engine checksums this control plane trusts,\nkept by llmgw bundle import and llmgw bundle trust; every engine file\na node fetches from the mirror is checked against it. Read again\nwhenever it changes. Empty is /etc/llmgw/offline.sha256.", Nested: ""},
|
||||
}},
|
||||
{Package: "discovery", Name: "Options", Title: "Options", Doc: "", Fields: []Field{
|
||||
{Key: "nvidia_smi", Type: "string", Doc: "NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.\nWorth configuring on nodes where the driver lives outside the default\npaths, which containerized driver installs often do.", Nested: ""},
|
||||
{Key: "ib_root", Type: "string", Doc: "IBRoot is the sysfs InfiniBand directory.", Nested: ""},
|
||||
@@ -323,6 +327,7 @@ var Reference = []Block{
|
||||
{Key: "monitoring", Type: "Monitoring", Doc: "Monitoring is where the dashboard reads its history from.", Nested: "config.Monitoring"},
|
||||
{Key: "tracing", Type: "obs.Tracing", Doc: "Tracing sends traces to an OpenTelemetry collector: each inference\nrequest, and on a control plane each placement, replica start and\nmodel job. Off unless an endpoint is set here or in\nOTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "obs.Tracing"},
|
||||
{Key: "mirror", Type: "Mirror", Doc: "Mirror points the control plane at a model mirror it can administer.\nEmpty means there is none, and /admin/mirror stays unmounted.", Nested: "config.Mirror"},
|
||||
{Key: "offline", Type: "Offline", Doc: "Offline is a cluster that reaches nothing outside itself: its nodes\ninstall engines, and upgrades fetch releases, from the mirror, out of\na bundle carried in (llmgw bundle create, llmgw bundle import).", Nested: "config.Offline"},
|
||||
{Key: "engines", Type: "map[string]string", Doc: "Engines are the engines every node installs, and at which version. The\nagents install them themselves — the right build for their hardware,\nchecked against its publisher's checksum and seen to run — so a node is\nnever trusted to already have what somebody once put there.\n\nUnset keeps DefaultEngines. A version replaces the default; an empty\none leaves that engine to be installed by hand.", Nested: ""},
|
||||
{Key: "engine_overrides", Type: "[]EngineOverride", Doc: "EngineOverrides give some nodes different engines from the rest: a\nvLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at\nall on nodes meant only for llama.cpp. Each picks nodes and lays its\nengines over the cluster's, in order, later ones last; an empty version\nremoves that engine from the nodes it picks.", Nested: ""},
|
||||
{Key: "backup_dir", Type: "string", Doc: "BackupDir is where `llmgw server backup` writes, and where the newest\narchive's age is read for llmgw_backup_last_success_timestamp_seconds.\nEmpty means /var/backups/llmgw-server.", Nested: ""},
|
||||
|
||||
@@ -202,6 +202,22 @@ func writeSource(file, source string) error {
|
||||
return os.Rename(tmp, sourcePath(file))
|
||||
}
|
||||
|
||||
// Record keeps a file's checksum and source beside it, as a download into
|
||||
// the mirror does: for a file put in place another way — an offline bundle's
|
||||
// — that should be offered to nodes with its checksum all the same.
|
||||
func Record(file, sha256, source string) error {
|
||||
if err := writeChecksum(file, sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
if source == "" {
|
||||
return nil
|
||||
}
|
||||
return writeSource(file, source)
|
||||
}
|
||||
|
||||
// Recorded is a file's checksum as kept beside it, "" for none.
|
||||
func Recorded(file string) string { return readChecksum(file) }
|
||||
|
||||
// readSource is where a file was fetched from, or "" for one fetched before
|
||||
// this was kept, or put in place by hand.
|
||||
func readSource(file string) string {
|
||||
|
||||
+79
-19
@@ -95,6 +95,11 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar
|
||||
// over plain HTTP, where anyone on the path can swap the bytes and, without a
|
||||
// signing key, the checksums with them. Loopback is allowed, for a mirror on
|
||||
// this machine and for tests.
|
||||
//
|
||||
// With a signing key built in, plain HTTP is allowed too: the checksums are
|
||||
// then checked against a key no server holds, so what carries them adds
|
||||
// nothing, and an offline site's mirror serves releases over the plain HTTP
|
||||
// it serves weights on.
|
||||
func CheckBase(base string) error {
|
||||
u, err := url.Parse(base)
|
||||
if err != nil || u.Host == "" {
|
||||
@@ -107,6 +112,9 @@ func CheckBase(base string) error {
|
||||
if ip := net.ParseIP(u.Hostname()); (ip != nil && ip.IsLoopback()) || u.Hostname() == "localhost" {
|
||||
return nil
|
||||
}
|
||||
if TrustedKey() != "" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("releases are fetched over https, and %s is not", base)
|
||||
}
|
||||
@@ -161,34 +169,86 @@ func (l *limitWriter) Write(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
// checksum reads the release's checksum for asset — after checking the
|
||||
// checksums' signature, when this build trusts a signing key. A release with
|
||||
// no signature is then refused rather than taken on its checksums: a missing
|
||||
// signature is exactly what replacing a release would look like.
|
||||
// checksums' signature, when this build trusts a signing key.
|
||||
func checksum(ctx context.Context, client *http.Client, root, asset string) (string, error) {
|
||||
body, err := get(ctx, client, root+"checksums.txt")
|
||||
c, err := fetchChecksums(ctx, client, root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return c.For(asset)
|
||||
}
|
||||
|
||||
// Checksums are a release's checksums.txt and its signature, as published.
|
||||
type Checksums struct {
|
||||
Text []byte
|
||||
// Signature is checksums.txt.sig; empty for a release read by a build
|
||||
// that trusts no key.
|
||||
Signature string
|
||||
}
|
||||
|
||||
// FetchChecksums reads a release's checksums and their signature, checked
|
||||
// as Install checks them: for a machine that keeps them — in an offline
|
||||
// bundle — to be checked again where they are used.
|
||||
func FetchChecksums(ctx context.Context, client *http.Client, base, version string) (Checksums, error) {
|
||||
if !Tag.MatchString(version) {
|
||||
return Checksums{}, fmt.Errorf("%q is not a release version", version)
|
||||
}
|
||||
if err := CheckBase(base); err != nil {
|
||||
return Checksums{}, err
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Minute}
|
||||
}
|
||||
return fetchChecksums(ctx, client, strings.TrimSuffix(base, "/")+"/"+version+"/")
|
||||
}
|
||||
|
||||
// fetchChecksums reads checksums.txt under root, and its signature when this
|
||||
// build trusts a signing key. A release with no signature is then refused
|
||||
// rather than taken on its checksums: a missing signature is exactly what
|
||||
// replacing a release would look like.
|
||||
func fetchChecksums(ctx context.Context, client *http.Client, root string) (Checksums, error) {
|
||||
body, err := get(ctx, client, root+"checksums.txt")
|
||||
if err != nil {
|
||||
return Checksums{}, err
|
||||
}
|
||||
defer body.Close()
|
||||
sums, err := io.ReadAll(io.LimitReader(body, 1<<20))
|
||||
if err != nil {
|
||||
return "", err
|
||||
return Checksums{}, err
|
||||
}
|
||||
if key := TrustedKey(); key != "" {
|
||||
sigBody, err := get(ctx, client, root+SignatureName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err)
|
||||
}
|
||||
sig, err := io.ReadAll(io.LimitReader(sigBody, 4096))
|
||||
sigBody.Close()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := Verify(key, sums, string(sig)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
c := Checksums{Text: sums}
|
||||
if TrustedKey() == "" {
|
||||
return c, nil
|
||||
}
|
||||
sc := bufio.NewScanner(bytes.NewReader(sums))
|
||||
sigBody, err := get(ctx, client, root+SignatureName)
|
||||
if err != nil {
|
||||
return Checksums{}, fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err)
|
||||
}
|
||||
sig, err := io.ReadAll(io.LimitReader(sigBody, 4096))
|
||||
sigBody.Close()
|
||||
if err != nil {
|
||||
return Checksums{}, err
|
||||
}
|
||||
c.Signature = string(sig)
|
||||
return c, c.Verify()
|
||||
}
|
||||
|
||||
// Verify checks the checksums against the key this build trusts. With none
|
||||
// they are taken as they are, as every release before signing was.
|
||||
func (c Checksums) Verify() error {
|
||||
key := TrustedKey()
|
||||
if key == "" {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(c.Signature) == "" {
|
||||
return fmt.Errorf("this build only installs signed releases, and these checksums carry no %s", SignatureName)
|
||||
}
|
||||
return Verify(key, c.Text, c.Signature)
|
||||
}
|
||||
|
||||
// For is the checksum the release gives for one asset.
|
||||
func (c Checksums) For(asset string) (string, error) {
|
||||
sc := bufio.NewScanner(bytes.NewReader(c.Text))
|
||||
for sc.Scan() {
|
||||
f := strings.Fields(sc.Text())
|
||||
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
|
||||
|
||||
@@ -71,6 +71,7 @@ func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) {
|
||||
unsigned(t)
|
||||
for base, ok := range map[string]bool{
|
||||
"https://gitea.example/releases/download": true,
|
||||
"http://127.0.0.1:8080/r": true,
|
||||
@@ -89,6 +90,60 @@ func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestASignedReleaseMayComeOverPlainHTTPFromAnOfflineMirror(t *testing.T) {
|
||||
pub, priv, err := GenerateKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("LLMGW_RELEASE_KEY", pub)
|
||||
if err := CheckBase("http://10.0.0.5:8081/_llmgw/releases"); err != nil {
|
||||
t.Fatalf("a mirror's plain HTTP was refused with a signing key trusted: %v", err)
|
||||
}
|
||||
|
||||
// And what comes over it is still held to the signature.
|
||||
body := []byte("the release")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := AssetName("v1.2.3", "linux", "amd64")
|
||||
sums := []byte(hex.EncodeToString(sum[:]) + " " + asset + "\n")
|
||||
sig, _ := Sign(priv, sums)
|
||||
_, other, _ := GenerateKey()
|
||||
forged, _ := Sign(other, sums)
|
||||
for _, tc := range []struct {
|
||||
sig string
|
||||
ok bool
|
||||
}{{sig, true}, {forged, false}, {"", false}} {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1.2.3/checksums.txt":
|
||||
_, _ = w.Write(sums)
|
||||
case "/v1.2.3/" + SignatureName:
|
||||
if tc.sig == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(tc.sig))
|
||||
case "/v1.2.3/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
c, err := FetchChecksums(t.Context(), srv.Client(), srv.URL, "v1.2.3")
|
||||
if (err == nil) != tc.ok {
|
||||
t.Errorf("signature %q: %v", tc.sig, err)
|
||||
}
|
||||
if err == nil {
|
||||
if got, _ := c.For(asset); got != hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("the checksum read is %s", got)
|
||||
}
|
||||
if err := c.Verify(); err != nil {
|
||||
t.Errorf("kept checksums do not verify again: %v", err)
|
||||
}
|
||||
}
|
||||
srv.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchGivesOnlyACheckedBinary(t *testing.T) {
|
||||
unsigned(t)
|
||||
body := []byte("for another machine")
|
||||
|
||||
Reference in new issue
Block a user