A served model can be moved to more cards, or fewer
llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
@@ -49,11 +49,13 @@ type ContextChange struct {
|
||||
}
|
||||
|
||||
// shape is what sizes a context: a token's KV cache, the context the model
|
||||
// was trained for, and its weights.
|
||||
// was trained for, and its weights; and its KV heads, which decide how many
|
||||
// cards tensor parallel can spread it over (0 unknown).
|
||||
type shape struct {
|
||||
kvPerToken int64
|
||||
trained int
|
||||
weights int64
|
||||
kvHeads int
|
||||
}
|
||||
|
||||
// SetContext changes a served model's context.
|
||||
@@ -135,12 +137,14 @@ func (s *Server) shapeOf(ctx context.Context, m catalog.Model) (shape, string) {
|
||||
return sh, "its header could not be read (" + err.Error() + "), so the memory it needs is not checked"
|
||||
}
|
||||
sh.kvPerToken, sh.trained = h.KVBytesPerToken(), int(h.ContextLength) //nolint:gosec // a header's context
|
||||
sh.kvHeads = int(h.HeadCountKV) //nolint:gosec // a header's head count
|
||||
case m.Origin != nil && m.Origin.Repo != "":
|
||||
mm, err := modelplan.FetchModel(ctx, m.Origin.Repo)
|
||||
if err != nil {
|
||||
return sh, "its config could not be read (" + err.Error() + "), so the memory it needs is not checked"
|
||||
}
|
||||
sh.kvPerToken, sh.trained = mm.KVBytesPerToken(modelplan.BF16), mm.MaxContext
|
||||
sh.kvHeads = mm.KVHeads
|
||||
default:
|
||||
return sh, "where it came from is not known, so the memory it needs is not checked"
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user