A served model can be moved to more cards, or fewer

llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 16:40:49 +01:00
1 parent 51aeadae3e
commit e1cff0d4e3
16 files changed
+753 -13

No files matched your search

+113
View File
@@ -2964,6 +2964,58 @@
}
}
},
"/api/v1/models/{name}/gpus": {
"post": {
"operationId": "postModelsNameGpus",
"summary": "Spread a served model over more cards or fewer: its profile, checked against its KV heads, its cards' memory and the nodes, then its replicas restarted",
"description": "Tensor parallel for vLLM, SGLang and TensorRT-LLM, a layer split for llama.cpp, on one node. With no room beside the running replicas, in_place stops each first.",
"tags": [
"models"
],
"parameters": [
{
"name": "name",
"in": "path",
"required": true,
"description": "The model's name in the catalog.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/modelserve.GPUsRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/modelserve.GPUsChange"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/models/{name}/headroom": {
"get": {
"operationId": "getModelsNameHeadroom",
@@ -11133,6 +11185,67 @@
}
}
},
"modelserve.GPUsChange": {
"type": "object",
"description": "GPUsChange is what was changed, and the replicas started again on it.",
"properties": {
"model": {
"type": "string"
},
"profile": {
"type": "string"
},
"engine": {
"type": "string"
},
"gpu_class": {
"type": "string"
},
"before": {
"type": "integer"
},
"after": {
"type": "integer"
},
"args": {
"type": "array",
"items": {
"type": "string"
}
},
"per_card": {
"type": "string",
"description": "PerCard is what each card holds after, when it could be worked out."
},
"restarted": {
"type": "integer"
},
"in_place": {
"type": "boolean"
},
"note": {
"type": "string",
"description": "Note is what could not be checked, said once."
}
}
},
"modelserve.GPUsRequest": {
"type": "object",
"description": "GPUsRequest is how many cards a replica is to use.",
"properties": {
"gpus": {
"type": "integer"
},
"profile": {
"type": "string",
"description": "Profile is the one to change, when the model has more than one profile on cards."
},
"in_place": {
"type": "boolean",
"description": "InPlace stops each replica before its replacement starts: for when no node has the cards free beside it. It does not answer while the new one loads."
}
}
},
"modelserve.Job": {
"type": "object",
"description": "Job is one model being served, and where it stands.",