A served model's context can be changed, engine and memory with it
CI / test (push) Successful in 12m0s

ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 15:50:04 +01:00
1 parent 1908bfb4b2
commit 986981ac81
14 files changed
+644 -1

No files matched your search

+26
View File
@@ -205,6 +205,7 @@ func (a *API) Register(mux Mux) {
mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/context", a.setModelContext)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
a.registerCanary(mux)
a.registerEval(mux)
@@ -445,6 +446,10 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
// ModelView is a model as `ctl models` shows it.
type ModelView struct {
Name string `json:"name"`
// ContextLength is what one request may use, MaxOutput what it may
// answer; 0 when the catalog does not say.
ContextLength int `json:"context_length,omitempty"`
MaxOutput int `json:"max_output,omitempty"`
// Canary is a share of its requests sent to a candidate; CanaryOf is
// set on the candidate, naming the model it is a canary of.
Canary *catalog.Canary `json:"canary,omitempty"`
@@ -837,6 +842,7 @@ func (a *API) Models() []ModelView {
h := health[m.Name]
v := ModelView{
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput,
Min: m.Scaling.Min, Max: m.Scaling.Max,
Ready: ready[m.Name], Total: total[m.Name],
Fallbacks: m.Fallbacks,
@@ -1748,6 +1754,26 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, job)
}
// setModelContext changes a served model's context, its engines' arguments
// and memory with it, and starts its replicas again on it.
func (a *API) setModelContext(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot change a model's context")
return
}
var req modelserve.ContextRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
change, err := a.serve.SetContext(r.Context(), r.PathValue("name"), req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, change)
}
func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) {
out := make([]modelserve.UpdateInfo, 0)
if a.serve != nil {