Strata runs as an engine: a 125B MoE on one card
Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client. - engines/strata.go: the adapter. The agent starts llmgw strata-serve, which runs Strata's setup the way its desktop app does (STRATA_CONTROL_INSTALL: installs nothing, writes only under the replica's and the node's directories) on the fetched GGUF files, points the configuration it writes at the installed engine, and execs Strata's server. The key goes in STRATA_API_KEY, never on a command line. - A profile's args go to setup: --family, --size (setup's --model, which is the source everywhere else), --context, --parallel, ... Flags that move files or pick the engine or GPU are refused. - provision: the source pinned by digest in a Python environment, and the release's prebuilt engine unpacked beside it, checked against the digest GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9, driver 575). Python builds can now carry release assets. - Metrics are vLLM's names under /metrics/prometheus. - The catalog wants one card and a family, and a directory, not one .gguf. Opt-in: not in DefaultEngines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
eb9c448d68
commit
c0d5194ea0
19 files changed
+906
-4
No files matched your search
@@ -109,6 +109,11 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil
|
||||
}
|
||||
reqs = append(reqs, more...)
|
||||
}
|
||||
if len(b.Assets) > 0 {
|
||||
if err := in.unpackAssets(ctx, r, w, b, tmp, st); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if b.Wheel != nil {
|
||||
// The engine from its GitHub release, pinned to the digest GitHub
|
||||
// publishes for the file: the same trust as an archive, and uv
|
||||
@@ -372,6 +377,48 @@ func (in *Installer) unpackSource(ctx context.Context, w Want, b Build, env stri
|
||||
return requirementsIn(string(f), b.IndexPackages)
|
||||
}
|
||||
|
||||
// unpackAssets fetches a Python engine's release binaries, each checked
|
||||
// against the digest GitHub publishes for it, and unpacks them into the
|
||||
// environment's AssetDir.
|
||||
func (in *Installer) unpackAssets(ctx context.Context, r Recipe, w Want, b Build, env string, st *Status) error {
|
||||
tag := b.AssetTag
|
||||
if tag == "" {
|
||||
tag = w.Version
|
||||
}
|
||||
in.progress(st, "asking GitHub for the checksums of %s %s", r.Repo, tag)
|
||||
sums, err := in.checksums(ctx, r.Repo, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := filepath.Join(env, b.AssetDir)
|
||||
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, asset := range b.Assets {
|
||||
want, ok := sums[asset]
|
||||
if !ok {
|
||||
// Refused rather than installed unchecked: this is a binary that
|
||||
// will run on every node.
|
||||
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified", r.Repo, tag, asset)
|
||||
}
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + tag + "/" + asset
|
||||
archive := filepath.Join(env+".source", asset)
|
||||
if err := os.MkdirAll(filepath.Dir(archive), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in.download(ctx, url, asset, archive, want, st); err != nil {
|
||||
return err
|
||||
}
|
||||
in.progress(st, "unpacking %s", asset)
|
||||
err := untarGz(archive, dst)
|
||||
_ = os.RemoveAll(filepath.Dir(archive))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unpacking %s: %w", asset, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// requirementsIn reads a requirements file's lines. Options (-r, -e, --index-url
|
||||
// and the like) are refused: they would fetch from somewhere other than PyPI,
|
||||
// or install what the lock never saw.
|
||||
|
||||
Reference in new issue
Block a user