A served model's context can be changed, engine and memory with it
CI / test (push) Successful in 12m0s
CI / test (push) Successful in 12m0s
ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1908bfb4b2
commit
986981ac81
14 files changed
+644
-1
No files matched your search
@@ -205,6 +205,7 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
|
||||
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/context", a.setModelContext)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
a.registerCanary(mux)
|
||||
a.registerEval(mux)
|
||||
@@ -445,6 +446,10 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
|
||||
// ModelView is a model as `ctl models` shows it.
|
||||
type ModelView struct {
|
||||
Name string `json:"name"`
|
||||
// ContextLength is what one request may use, MaxOutput what it may
|
||||
// answer; 0 when the catalog does not say.
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
MaxOutput int `json:"max_output,omitempty"`
|
||||
// Canary is a share of its requests sent to a candidate; CanaryOf is
|
||||
// set on the candidate, naming the model it is a canary of.
|
||||
Canary *catalog.Canary `json:"canary,omitempty"`
|
||||
@@ -837,6 +842,7 @@ func (a *API) Models() []ModelView {
|
||||
h := health[m.Name]
|
||||
v := ModelView{
|
||||
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
||||
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput,
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Ready: ready[m.Name], Total: total[m.Name],
|
||||
Fallbacks: m.Fallbacks,
|
||||
@@ -1748,6 +1754,26 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, job)
|
||||
}
|
||||
|
||||
// setModelContext changes a served model's context, its engines' arguments
|
||||
// and memory with it, and starts its replicas again on it.
|
||||
func (a *API) setModelContext(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot change a model's context")
|
||||
return
|
||||
}
|
||||
var req modelserve.ContextRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
change, err := a.serve.SetContext(r.Context(), r.PathValue("name"), req, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, change)
|
||||
}
|
||||
|
||||
func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) {
|
||||
out := make([]modelserve.UpdateInfo, 0)
|
||||
if a.serve != nil {
|
||||
|
||||
Reference in new issue
Block a user