A served model's context can be changed, engine and memory with it

ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 15:50:04 +01:00
1 parent 1908bfb4b2
commit 986981ac81
14 files changed
+644 -1

No files matched your search

+128
View File
@@ -2719,6 +2719,57 @@
}
}
},
"/api/v1/models/{name}/context": {
"post": {
"operationId": "postModelsNameContext",
"summary": "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted",
"tags": [
"models"
],
"parameters": [
{
"name": "name",
"in": "path",
"required": true,
"description": "The model's name in the catalog.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/modelserve.ContextRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/modelserve.ContextChange"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/models/{name}/eval": {
"post": {
"operationId": "postModelsNameEval",
@@ -7749,6 +7800,13 @@
"name": {
"type": "string"
},
"context_length": {
"type": "integer",
"description": "ContextLength is what one request may use, MaxOutput what it may answer; 0 when the catalog does not say."
},
"max_output": {
"type": "integer"
},
"canary": {
"$ref": "#/components/schemas/catalog.Canary",
"description": "Canary is a share of its requests sent to a candidate; CanaryOf is set on the candidate, naming the model it is a canary of."
@@ -11005,6 +11063,76 @@
}
}
},
"modelserve.ContextChange": {
"type": "object",
"description": "ContextChange is what was changed, and the replicas started again on it.",
"properties": {
"model": {
"type": "string"
},
"before": {
"type": "integer"
},
"after": {
"type": "integer"
},
"max_output": {
"type": "integer"
},
"trained": {
"type": "integer"
},
"profiles": {
"type": "array",
"items": {
"$ref": "#/components/schemas/modelserve.ContextProfile"
}
},
"restarted": {
"type": "integer"
},
"note": {
"type": "string",
"description": "Note is what could not be checked, said once."
}
}
},
"modelserve.ContextProfile": {
"type": "object",
"description": "ContextProfile is what one profile needs before and after.",
"properties": {
"profile": {
"type": "string"
},
"engine": {
"type": "string"
},
"before": {
"type": "string"
},
"after": {
"type": "string"
},
"args": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"modelserve.ContextRequest": {
"type": "object",
"description": "ContextRequest is the context to change to: tokens one request may use, and for llama.cpp how many at once (its pool is the two multiplied).",
"properties": {
"tokens": {
"type": "integer"
},
"slots": {
"type": "integer"
}
}
},
"modelserve.Job": {
"type": "object",
"description": "Job is one model being served, and where it stands.",