Merge C18 offline bundles and the Helm chart
Conflicts: server.go imports (both kept); agent.pb.go regenerated with buf from the merged proto. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
6f85c17fc5
51 files changed
+4177
-63
No files matched your search
@@ -88,7 +88,13 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) {
|
||||
// the installed build is used first — and quietly replace theirs.
|
||||
continue
|
||||
}
|
||||
wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()})
|
||||
w := provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()}
|
||||
if base := e.GetOfflineBase(); base != "" {
|
||||
// An offline cluster: from its mirror, checked against what the
|
||||
// control plane holds, never the publisher.
|
||||
w.Offline = &provision.Offline{Base: base, SHA256: maps.Clone(e.GetOfflineSha256())}
|
||||
}
|
||||
wants = append(wants, w)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.provWants = wants
|
||||
|
||||
@@ -34,6 +34,24 @@ type Want struct {
|
||||
// Variant is a build chosen for this node ("cu129"); empty leaves it to
|
||||
// the node's hardware.
|
||||
Variant string
|
||||
// Offline, when set, is where the engine comes from instead of its
|
||||
// publisher: a cluster with no way to the internet installs from its
|
||||
// mirror's copy of an offline bundle.
|
||||
Offline *Offline
|
||||
}
|
||||
|
||||
// Offline is an engine version's files on a cluster's own mirror.
|
||||
//
|
||||
// The files come from the mirror and their checksums from the control plane,
|
||||
// which read them from the bundle they were imported from. The mirror serves
|
||||
// plain HTTP to every node; a checksum taken from beside the file there would
|
||||
// prove only that the file and the checksum came from the same place.
|
||||
type Offline struct {
|
||||
// Base is where the version's files are: <mirror>/_llmgw/engines/<engine>/<version>.
|
||||
Base string
|
||||
// SHA256 is each file's checksum, by name, as the publisher gave it
|
||||
// when the bundle was made.
|
||||
SHA256 map[string]string
|
||||
}
|
||||
|
||||
// The states an engine's installation can be in.
|
||||
@@ -156,6 +174,16 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
|
||||
}
|
||||
return st
|
||||
}
|
||||
if w.Offline != nil && recipe.Repo == "" {
|
||||
// Said, not attempted: uv would spend minutes failing to reach PyPI,
|
||||
// and the failure would read like a network fault rather than the
|
||||
// design of an offline site.
|
||||
st.State = Unsupported
|
||||
st.Detail = "this cluster installs offline, and " + w.Engine + " comes from PyPI, which an offline bundle " +
|
||||
"does not carry: install it on the node by hand and name it in agent.yaml, or set engines." +
|
||||
w.Engine + ` to "" in server.yaml`
|
||||
return st
|
||||
}
|
||||
|
||||
// Installed before: whichever candidate is, best first, checked that it
|
||||
// still runs and trusted for nothing else.
|
||||
@@ -273,10 +301,15 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
return err
|
||||
}
|
||||
|
||||
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
|
||||
sums, err := in.checksums(ctx, r.Repo, w.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
var sums map[string]string
|
||||
if w.Offline != nil {
|
||||
sums = w.Offline.SHA256
|
||||
} else {
|
||||
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
|
||||
var err error
|
||||
if sums, err = in.checksums(ctx, r.Repo, w.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tmp := dir + ".partial"
|
||||
@@ -289,14 +322,22 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
got := map[string]string{}
|
||||
for _, asset := range b.Assets {
|
||||
want, ok := sums[asset]
|
||||
if !ok {
|
||||
switch {
|
||||
case !ok && w.Offline != nil:
|
||||
return fmt.Errorf("the offline bundle has no %s for %s %s: make one that has "+
|
||||
"(llmgw bundle create --engine %s=%s) and import it", asset, w.Engine, w.Version, w.Engine, w.Version)
|
||||
case !ok:
|
||||
// Refused rather than installed unchecked: this is a binary that
|
||||
// will run as root on every node.
|
||||
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
|
||||
w.Engine, w.Version, asset)
|
||||
}
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + w.Version + "/" + asset
|
||||
if w.Offline != nil {
|
||||
url = strings.TrimSuffix(w.Offline.Base, "/") + "/" + asset
|
||||
}
|
||||
archive := filepath.Join(tmp, asset+".download")
|
||||
if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
|
||||
if err := in.download(ctx, url, asset, archive, want, st); err != nil {
|
||||
return err
|
||||
}
|
||||
in.progress(st, "unpacking %s", asset)
|
||||
@@ -553,6 +594,26 @@ func (in *Installer) has(tool string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Published is the sha256 an engine's publisher gives for each file of a
|
||||
// version, and where each is downloaded from: what an offline bundle is made
|
||||
// of, read the way a node online reads it.
|
||||
func (in *Installer) Published(ctx context.Context, engine, version string) (sums map[string]string, url func(asset string) string, err error) {
|
||||
r, ok := Recipes[engine]
|
||||
if !ok || r.Repo == "" {
|
||||
return nil, nil, fmt.Errorf("%s is not published as release files", engine)
|
||||
}
|
||||
if !validVersion.MatchString(version) {
|
||||
return nil, nil, fmt.Errorf("%q is not a version of %s", version, engine)
|
||||
}
|
||||
sums, err = in.checksums(ctx, r.Repo, version)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return sums, func(asset string) string {
|
||||
return strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + version + "/" + asset
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checksums reads the sha256 GitHub publishes for each asset of a release.
|
||||
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
|
||||
url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
|
||||
@@ -594,8 +655,7 @@ func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[strin
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
|
||||
func (in *Installer) download(ctx context.Context, url, asset, dst, want string, st *Status) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
|
||||
// offlineMirror is a cluster's mirror holding a bundle's engine files, and
|
||||
// nothing that answers for GitHub.
|
||||
type offlineMirror struct {
|
||||
files map[string][]byte
|
||||
mu sync.Mutex
|
||||
paths []string
|
||||
}
|
||||
|
||||
func (m *offlineMirror) serve(t *testing.T) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
m.paths = append(m.paths, r.URL.Path)
|
||||
m.mu.Unlock()
|
||||
b, ok := m.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(b)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestAnOfflineNodeInstallsFromTheMirrorWithChecksumsFromTheControlPlane(t *testing.T) {
|
||||
rel := llamaRelease(t)
|
||||
asset := "llama-b1-bin-ubuntu-x64.tar.gz"
|
||||
m := &offlineMirror{files: map[string][]byte{"/_llmgw/engines/llamacpp/b1/" + asset: rel.assets[asset]}}
|
||||
srv := m.serve(t)
|
||||
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
|
||||
// GitHub is the mirror too, so a request for it would be seen and
|
||||
// answered 404: an offline install must not ask.
|
||||
in, _ := installer(t, srv, sys, ubuntu)
|
||||
|
||||
want := Want{Engine: "llamacpp", Version: "b1", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b1",
|
||||
SHA256: map[string]string{asset: strings.TrimPrefix(rel.digests[asset], "sha256:")},
|
||||
}}
|
||||
got := in.Converge(t.Context(), []Want{want})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("not installed from the mirror: %+v", got[0])
|
||||
}
|
||||
for _, p := range m.paths {
|
||||
if strings.HasPrefix(p, "/repos/") || strings.Contains(p, "/releases/download/") {
|
||||
t.Errorf("an offline install asked for %s", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A file on the mirror that is not what the bundle recorded is refused,
|
||||
// whatever the mirror says of it.
|
||||
m.files["/_llmgw/engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz"] = []byte("replaced on the mirror")
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b2", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b2",
|
||||
SHA256: map[string]string{"llama-b2-bin-ubuntu-x64.tar.gz": strings.Repeat("a", 64)},
|
||||
}}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") {
|
||||
t.Errorf("a file not matching the bundle's checksum was not refused: %+v", got[0])
|
||||
}
|
||||
|
||||
// One the bundle has no checksum for is not fetched at all, and the
|
||||
// operator is told which bundle would have it.
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b3", Offline: &Offline{
|
||||
Base: srv.URL + "/_llmgw/engines/llamacpp/b3",
|
||||
}}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "bundle create --engine llamacpp=b3") {
|
||||
t.Errorf("a version missing from the bundle did not say how to add it: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOfflineNodeIsToldPythonEnginesComeFromPyPI(t *testing.T) {
|
||||
gpu := ubuntu
|
||||
gpu.NVIDIADriver, gpu.MinComputeCapability = "580.65", 8.9
|
||||
in := &Installer{Root: t.TempDir(), Host: gpu, Run: (&fakeSystem{}).run}
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}})
|
||||
if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "PyPI") {
|
||||
t.Fatalf("vLLM offline: %+v", got[0])
|
||||
}
|
||||
|
||||
// A node it is not meant for still says nothing is wrong.
|
||||
in.Host = ubuntu
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}})
|
||||
if got[0].State != Skipped {
|
||||
t.Errorf("vLLM offline on a CPU node: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleCarriesEveryBuildTheNodesMightChoose(t *testing.T) {
|
||||
variants := func(arch string) []string {
|
||||
bs, err := Builds("llamacpp", "b1", arch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, b := range bs {
|
||||
out = append(out, b.Variant)
|
||||
}
|
||||
return out
|
||||
}
|
||||
if got := variants("amd64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "cuda-12.8", "vulkan"}) {
|
||||
t.Errorf("amd64 builds: %v", got)
|
||||
}
|
||||
if got := variants("arm64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "vulkan"}) {
|
||||
t.Errorf("arm64 builds: %v", got)
|
||||
}
|
||||
|
||||
// Each one a node would choose is among them, file for file.
|
||||
bundled := map[string][]string{}
|
||||
bs, _ := Builds("llamacpp", "b1", "amd64")
|
||||
for _, b := range bs {
|
||||
bundled[b.Variant] = b.Assets
|
||||
}
|
||||
for _, h := range []discovery.Host{
|
||||
ubuntu,
|
||||
{Arch: "amd64", NVIDIADriver: "550.1", MinComputeCapability: 8.6},
|
||||
{Arch: "amd64", NVIDIADriver: "470.1", MinComputeCapability: 6.1},
|
||||
} {
|
||||
chosen, err := llamaCppCandidates("b1", "", h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range chosen {
|
||||
if !slices.Equal(bundled[b.Variant], b.Assets) {
|
||||
t.Errorf("%+v chooses %s %v, which the bundle carries as %v", h, b.Variant, b.Assets, bundled[b.Variant])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := Builds("vllm", "0.30.0", "amd64"); err == nil || !strings.Contains(err.Error(), "PyPI") {
|
||||
t.Errorf("a Python engine was offered for a bundle: %v", err)
|
||||
}
|
||||
if _, err := Builds("llamacpp", "../b1", "amd64"); err == nil {
|
||||
t.Error("a version that is a path was accepted")
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,49 @@ func Installable() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// Builds are every build of an engine version published for an
|
||||
// architecture: what an offline bundle carries, for nodes it has not seen.
|
||||
// Read from the same table the nodes choose from, asked as a machine with
|
||||
// no GPU and as one with any NVIDIA GPU, so a bundle and the nodes that
|
||||
// install from it cannot disagree about which files a build is. Archive
|
||||
// engines only: a Python engine is a PyPI environment, which no bundle
|
||||
// carries.
|
||||
func Builds(engine, version, arch string) ([]Build, error) {
|
||||
r, ok := Recipes[engine]
|
||||
switch {
|
||||
case !ok:
|
||||
return nil, fmt.Errorf("this agent does not know how to install %s", engine)
|
||||
case r.Repo == "":
|
||||
return nil, fmt.Errorf("%s is installed from PyPI, which an offline bundle does not carry", engine)
|
||||
case !validVersion.MatchString(version):
|
||||
return nil, fmt.Errorf("%q is not a version of %s", version, engine)
|
||||
}
|
||||
var out []Build
|
||||
seen := map[string]bool{}
|
||||
var errs []string
|
||||
for _, h := range []discovery.Host{
|
||||
{Arch: arch},
|
||||
// Any card and the newest driver: every GPU build is a candidate.
|
||||
{Arch: arch, NVIDIADriver: "9999"},
|
||||
} {
|
||||
bs, err := r.Candidates(version, "", h)
|
||||
if err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
continue
|
||||
}
|
||||
for _, b := range bs {
|
||||
if !seen[b.Variant] {
|
||||
seen[b.Variant] = true
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("no build of %s %s for %s: %s", engine, version, arch, strings.Join(errs, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UVVersion is the uv that installs the Python engines.
|
||||
const UVVersion = "0.12.20"
|
||||
|
||||
|
||||
Reference in new issue
Block a user