The startup timeout runs from when a node's download last moved, so a model of hundreds of gigabytes on a slow mirror is not failed while it arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO) on every node, and a layout given by hand keeps expert parallelism for a model of experts. docs/large-models.md: how a model is split, what each way needs of a card, and a checklist for bringing one up on a fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4fe9b46451
commit
8ffde2b80c
12 files changed
+259
-4
No files matched your search
@@ -45,7 +45,9 @@ type Plan struct {
|
||||
// KVBytesPerToken is one token of context, the whole model: what an
|
||||
// option needs for another context is worked out from it. 0 is unknown.
|
||||
KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Experts is a mixture-of-experts model's routed experts; 0 for dense.
|
||||
Experts int `json:"experts,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Servable is false when the nodes' engine cannot run the model at all,
|
||||
// and Reason says why.
|
||||
Servable bool `json:"servable"`
|
||||
@@ -161,7 +163,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
p.Note = "the model's shape was not readable, so the context's memory is not counted"
|
||||
}
|
||||
shape.Params = p.Params
|
||||
p.KVBytesPerToken = shape.KVBytesPerToken
|
||||
p.KVBytesPerToken, p.Experts = shape.KVBytesPerToken, shape.Experts
|
||||
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
|
||||
if !p.Servable {
|
||||
p.Options = nil
|
||||
|
||||
Reference in new issue
Block a user