A served model's context can be changed, engine and memory with it
ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1908bfb4b2
commit
986981ac81
14 files changed
+644
-1
No files matched your search
@@ -2719,6 +2719,57 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/context": {
|
||||
"post": {
|
||||
"operationId": "postModelsNameContext",
|
||||
"summary": "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The model's name in the catalog.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/modelserve.ContextRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/modelserve.ContextChange"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/eval": {
|
||||
"post": {
|
||||
"operationId": "postModelsNameEval",
|
||||
@@ -7749,6 +7800,13 @@
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"context_length": {
|
||||
"type": "integer",
|
||||
"description": "ContextLength is what one request may use, MaxOutput what it may answer; 0 when the catalog does not say."
|
||||
},
|
||||
"max_output": {
|
||||
"type": "integer"
|
||||
},
|
||||
"canary": {
|
||||
"$ref": "#/components/schemas/catalog.Canary",
|
||||
"description": "Canary is a share of its requests sent to a candidate; CanaryOf is set on the candidate, naming the model it is a canary of."
|
||||
@@ -11005,6 +11063,76 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.ContextChange": {
|
||||
"type": "object",
|
||||
"description": "ContextChange is what was changed, and the replicas started again on it.",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"before": {
|
||||
"type": "integer"
|
||||
},
|
||||
"after": {
|
||||
"type": "integer"
|
||||
},
|
||||
"max_output": {
|
||||
"type": "integer"
|
||||
},
|
||||
"trained": {
|
||||
"type": "integer"
|
||||
},
|
||||
"profiles": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/modelserve.ContextProfile"
|
||||
}
|
||||
},
|
||||
"restarted": {
|
||||
"type": "integer"
|
||||
},
|
||||
"note": {
|
||||
"type": "string",
|
||||
"description": "Note is what could not be checked, said once."
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.ContextProfile": {
|
||||
"type": "object",
|
||||
"description": "ContextProfile is what one profile needs before and after.",
|
||||
"properties": {
|
||||
"profile": {
|
||||
"type": "string"
|
||||
},
|
||||
"engine": {
|
||||
"type": "string"
|
||||
},
|
||||
"before": {
|
||||
"type": "string"
|
||||
},
|
||||
"after": {
|
||||
"type": "string"
|
||||
},
|
||||
"args": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.ContextRequest": {
|
||||
"type": "object",
|
||||
"description": "ContextRequest is the context to change to: tokens one request may use, and for llama.cpp how many at once (its pool is the two multiplied).",
|
||||
"properties": {
|
||||
"tokens": {
|
||||
"type": "integer"
|
||||
},
|
||||
"slots": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.Job": {
|
||||
"type": "object",
|
||||
"description": "Job is one model being served, and where it stands.",
|
||||
|
||||
Reference in new issue
Block a user