llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
@@ -112,6 +112,15 @@ func (c *updModels) SetContext(_ context.Context, n string, ctxLen, maxOut int,
|
||||
c.models[n] = m
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (c *updModels) SetProfiles(_ context.Context, n string, ps []catalog.Profile, _ string) (catalog.Model, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
m := c.models[n]
|
||||
m.Profiles = ps
|
||||
c.models[n] = m
|
||||
return m, nil
|
||||
}
|
||||
func (c *updModels) all() []catalog.Model {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
Reference in new issue
Block a user