The startup timeout runs from when a node's download last moved, so a model of hundreds of gigabytes on a slow mirror is not failed while it arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO) on every node, and a layout given by hand keeps expert parallelism for a model of experts. docs/large-models.md: how a model is split, what each way needs of a card, and a checklist for bringing one up on a fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4fe9b46451
commit
8ffde2b80c
12 files changed
+259
-4
No files matched your search
@@ -69,6 +69,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
|
||||
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
|
||||
_ = f.MarkHidden("tp")
|
||||
f.StringToStringVar(&req.Env, "env", nil,
|
||||
"KEY=VALUE in the engine's environment on every node, e.g. NCCL_DEBUG=INFO; repeatable")
|
||||
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
|
||||
@@ -163,6 +163,10 @@ type Request struct {
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
// Env is set in the engine's environment, on every node of a replica:
|
||||
// NCCL_DEBUG=INFO to see how its cards talk, NCCL_IB_* to steer them.
|
||||
// Only the variables engines are tuned with; the catalog refuses others.
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
// IfAbsent adds the model only if nobody has meanwhile: the leader's own
|
||||
// serving of a judge a policy names gives way to the one init or an
|
||||
// operator added while it fetched, rather than replacing it.
|
||||
@@ -580,6 +584,8 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
o = p
|
||||
}
|
||||
}
|
||||
// A way the plan did not offer is still a model of experts.
|
||||
o.ExpertParallel = engine == "vllm" && plan.Experts > 1 && o.GPUs() > 1
|
||||
out = append(out, o)
|
||||
case len(opts) > 0:
|
||||
out = append(out, opts[0])
|
||||
@@ -614,7 +620,7 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
|
||||
var out []catalog.Profile
|
||||
for i, o := range layout {
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i, Env: req.Env}
|
||||
switch {
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
|
||||
@@ -475,3 +475,26 @@ func TestAJobAddingOnlyIfAbsentGivesWayToAModelAddedMeanwhile(t *testing.T) {
|
||||
t.Errorf("job %s, added %d: %+v", j.State, len(taken.added), j.Steps)
|
||||
}
|
||||
}
|
||||
|
||||
// A model of experts given its cards by hand spreads them still, and the
|
||||
// environment asked for reaches every profile.
|
||||
func TestAHandGivenLayoutOfExpertsKeepsExpertParallelAndTheEnvironment(t *testing.T) {
|
||||
plan := Plan{Repo: "Qwen/Qwen3-235B-A22B", Experts: 128}
|
||||
req := Request{Format: "safetensors", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, ContextLength: 8192,
|
||||
Env: map[string]string{"NCCL_DEBUG": "INFO"}}
|
||||
layout, err := chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profs := profilesFor(layout, req, 0, 0)
|
||||
p := profs[0]
|
||||
if p.TP != 8 || p.PP != 2 || p.Nodes != 2 || !slices.Contains(p.Args, "--enable-expert-parallel") || p.Env["NCCL_DEBUG"] != "INFO" {
|
||||
t.Errorf("profile %+v", p)
|
||||
}
|
||||
// A dense model given the same cards does not.
|
||||
plan.Experts = 0
|
||||
layout, _ = chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
|
||||
if slices.Contains(profilesFor(layout, req, 0, 0)[0].Args, "--enable-expert-parallel") {
|
||||
t.Error("a dense model was given expert parallelism")
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,9 @@ type Plan struct {
|
||||
// KVBytesPerToken is one token of context, the whole model: what an
|
||||
// option needs for another context is worked out from it. 0 is unknown.
|
||||
KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Experts is a mixture-of-experts model's routed experts; 0 for dense.
|
||||
Experts int `json:"experts,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Servable is false when the nodes' engine cannot run the model at all,
|
||||
// and Reason says why.
|
||||
Servable bool `json:"servable"`
|
||||
@@ -161,7 +163,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
p.Note = "the model's shape was not readable, so the context's memory is not counted"
|
||||
}
|
||||
shape.Params = p.Params
|
||||
p.KVBytesPerToken = shape.KVBytesPerToken
|
||||
p.KVBytesPerToken, p.Experts = shape.KVBytesPerToken, shape.Experts
|
||||
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
|
||||
if !p.Servable {
|
||||
p.Options = nil
|
||||
|
||||
@@ -582,7 +582,11 @@ func (r *Reconciler) expireStuckReplicas() {
|
||||
if rep.State != Starting && rep.State != Placing {
|
||||
continue
|
||||
}
|
||||
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
|
||||
since := rep.CreatedAt
|
||||
if rep.fetchedAt.After(since) {
|
||||
since = rep.fetchedAt
|
||||
}
|
||||
if now.Sub(since) < r.startupTimeout(rep) {
|
||||
continue
|
||||
}
|
||||
rep.State = Failed
|
||||
@@ -1328,6 +1332,9 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
|
||||
if rep.Shards[i].ID != shardID {
|
||||
continue
|
||||
}
|
||||
if prev := rep.Shards[i].Fetch; prev == nil || done > prev.BytesDone {
|
||||
rep.fetchedAt = r.now()
|
||||
}
|
||||
rep.Shards[i].Fetch = &FetchProgress{
|
||||
Source: source, BytesDone: done, BytesTotal: total,
|
||||
}
|
||||
|
||||
@@ -160,6 +160,11 @@ type Replica struct {
|
||||
AdoptedAt time.Time
|
||||
|
||||
CreatedAt time.Time
|
||||
// fetchedAt is when a shard's download last moved: the startup timeout
|
||||
// runs from then, so a model of hundreds of gigabytes on a slow mirror
|
||||
// is not failed while it is still arriving, and one whose download
|
||||
// stalls is failed a timeout after it stopped.
|
||||
fetchedAt time.Time
|
||||
ReadyAt time.Time
|
||||
// LastUsedAt is when a request was last routed to this replica, zero if
|
||||
// none has been since the router started.
|
||||
|
||||
@@ -406,3 +406,34 @@ func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) {
|
||||
t.Errorf("WantShards = %d, want 2", reps[0].WantShards)
|
||||
}
|
||||
}
|
||||
|
||||
// A model of hundreds of gigabytes on a slow mirror takes longer to arrive
|
||||
// than to load: the startup timeout runs from when its download last moved,
|
||||
// and a download that stops is failed a timeout after it stopped.
|
||||
func TestAReplicaStillDownloadingIsNotFailed(t *testing.T) {
|
||||
m := testModel(1, catalog.Profile{
|
||||
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 20 * time.Minute,
|
||||
})
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
h.r.Reconcile(context.Background())
|
||||
rep := h.r.Replicas()[0]
|
||||
shard := rep.Shards[0].ID
|
||||
|
||||
// Forty minutes of download, moving all the while.
|
||||
for i := int64(1); i <= 8; i++ {
|
||||
h.advance(5 * time.Minute)
|
||||
h.r.OnFetchProgress(rep.ID, shard, "mirror", i<<35, 400<<30)
|
||||
h.r.expireStuckReplicas()
|
||||
}
|
||||
if got := h.r.Replicas()[0].State; got == Failed {
|
||||
t.Fatal("a replica whose download kept moving was failed")
|
||||
}
|
||||
// The same figure again is not progress: stalled, it fails a timeout on.
|
||||
h.advance(10 * time.Minute)
|
||||
h.r.OnFetchProgress(rep.ID, shard, "mirror", 8<<35, 400<<30)
|
||||
h.advance(11 * time.Minute)
|
||||
h.r.expireStuckReplicas()
|
||||
if got := h.r.Replicas()[0].State; got != Failed {
|
||||
t.Errorf("a stalled download was never failed: %s", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user