Found running it through the cluster: - Strata's server starts its engine in a process group of its own, so the agent saw a GPU replica on the CPU and its 22 GB as somebody else's memory. A process the shard started, at any remove, is now the shard's. - An adopted shard's kept compute was sent at once and never again, so a wrong one read again on adoption never reached the control plane: what is sent is sent again when it changes, and a GPU shard kept as the CPU is read again on adoption. - /metrics/prometheus is llama.cpp's names without the KV cache; vLLM's, with it, are /metrics?format=prometheus. The router now sees running 1, waiting 2 under load. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
145 lines
4.5 KiB
Go
145 lines
4.5 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// StrataConfig locates a Strata installation.
|
|
type StrataConfig struct {
|
|
// Dir is Strata's source directory, where setup.py and serve/ are.
|
|
Dir string `koanf:"dir"`
|
|
// Python is the interpreter of its environment; Dir's .venv when unset.
|
|
Python string `koanf:"python"`
|
|
// Engine is the directory of its prebuilt engine, where the strata
|
|
// binary and the CUDA libraries it ships are; Dir's engine/ when unset.
|
|
Engine string `koanf:"engine"`
|
|
}
|
|
|
|
// Strata runs Strata (github.com/Niko1221/Strata): a mixture-of-experts
|
|
// engine that keeps a model's experts in RAM and the ones used most in VRAM,
|
|
// so a 125B model serves from one 24 GB card. It runs a few model families
|
|
// only, each prepared from its GGUF files by Strata's own setup, so the agent
|
|
// starts llmgw strata-serve, which prepares the model and then becomes
|
|
// Strata's server.
|
|
type Strata struct {
|
|
cfg StrataConfig
|
|
installed atomic.Pointer[string]
|
|
}
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*Strata) Name() string { return "strata" }
|
|
|
|
// HealthPath answers 200 only once the model is loaded.
|
|
func (*Strata) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is Strata's Prometheus text, in vLLM's names; its /metrics is
|
|
// JSON for its own monitor unless asked for this format.
|
|
func (*Strata) MetricsPath() string { return "/metrics?format=prometheus" }
|
|
|
|
// ServesHTTP is true: a Strata replica is one process on one node.
|
|
func (*Strata) ServesHTTP(ShardSpec) bool { return true }
|
|
|
|
// UseInstalled points the engine at the environment the agent installed:
|
|
// its interpreter, with Strata's code beside its bin in strata/ and the
|
|
// engine in strata-engine/.
|
|
func (s *Strata) UseInstalled(python string) { s.installed.Store(&python) }
|
|
|
|
// Configured reports whether the configuration names a Strata.
|
|
func (s *Strata) Configured() bool { return s.cfg.Dir != "" }
|
|
|
|
// StrataSourceDir and StrataEngineDir are where an installed environment
|
|
// keeps Strata's code and its prebuilt engine, beside bin/.
|
|
const (
|
|
StrataSourceDir = "strata"
|
|
StrataEngineDir = "strata-engine"
|
|
)
|
|
|
|
func (s *Strata) env() string {
|
|
if p := s.installed.Load(); p != nil {
|
|
return filepath.Dir(filepath.Dir(*p))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (s *Strata) dir() string {
|
|
if e := s.env(); e != "" {
|
|
return filepath.Join(e, StrataSourceDir)
|
|
}
|
|
return s.cfg.Dir
|
|
}
|
|
|
|
func (s *Strata) engineDir() string {
|
|
if e := s.env(); e != "" {
|
|
return filepath.Join(e, StrataEngineDir)
|
|
}
|
|
if s.cfg.Engine != "" {
|
|
return s.cfg.Engine
|
|
}
|
|
return filepath.Join(s.cfg.Dir, "engine")
|
|
}
|
|
|
|
func (s *Strata) python() string {
|
|
if p := s.installed.Load(); p != nil {
|
|
return *p
|
|
}
|
|
if s.cfg.Python != "" {
|
|
return s.cfg.Python
|
|
}
|
|
return filepath.Join(s.cfg.Dir, ".venv", "bin", "python")
|
|
}
|
|
|
|
// Validate checks the profile: one card, on one node. Strata splits a model
|
|
// over a node's cards by layer, but that is not driven from here yet.
|
|
func (*Strata) Validate(p Profile, gpus []GPUInfo) error {
|
|
if p.Nodes > 1 || p.TP > 1 || p.PP > 1 || p.GPUsPerNode > 1 {
|
|
return fmt.Errorf("profile %q: strata runs on one GPU of one node", p.Name)
|
|
}
|
|
if err := requireComputeCapability(p, gpus, 7.0, "strata"); err != nil {
|
|
return err
|
|
}
|
|
return validateCommon(p, gpus)
|
|
}
|
|
|
|
// Commands is llmgw strata-serve, which prepares the model with Strata's
|
|
// setup and then execs Strata's server.
|
|
func (s *Strata) Commands(sh ShardSpec) ([]Command, error) {
|
|
dir := s.dir()
|
|
if dir == "" {
|
|
return nil, fmt.Errorf("strata: no Strata directory is configured (engines.strata.dir)")
|
|
}
|
|
bin, err := os.Executable()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
p := sh.Profile
|
|
args := []string{"strata-serve",
|
|
"--listen", net.JoinHostPort(sh.Host(), strconv.Itoa(sh.HTTPPort)),
|
|
"--model", p.ServedModelName,
|
|
"--weights", p.ModelPath,
|
|
// Relative to where engines run — their user's home — so they are
|
|
// theirs to write. The data is shared by the node's replicas: a
|
|
// prepared model and its draft layer are gigabytes, the same for
|
|
// every replica of it.
|
|
"--state", filepath.Join("strata", sh.ReplicaID),
|
|
"--data", filepath.Join("strata", "data"),
|
|
"--python", s.python(),
|
|
"--strata", dir,
|
|
"--engine", s.engineDir(),
|
|
"--",
|
|
}
|
|
args = append(args, p.Args...)
|
|
env := baseEnv(sh)
|
|
if sh.APIKey != "" {
|
|
// Strata's server reads its key from here, and strata-serve passes
|
|
// it on in the environment: not on a command line every user on the
|
|
// node can read.
|
|
env["STRATA_API_KEY"] = sh.APIKey
|
|
}
|
|
return []Command{{Name: "strata", Path: bin, Args: args, Env: materializeEnv(env)}}, nil
|
|
}
|