From 986981ac812cd93a51c060263ba40d2059fdb370 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 29 Oct 2026 15:29:03 +0100 Subject: [PATCH] A served model's context can be changed, engine and memory with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context and the console's Context length…: the catalog, the engine's arguments and the KV memory change together, refused past the trained context or what the cards hold, then the replicas restart one by one. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 + docs/admin-api.openapi.json | 128 +++++++++++ docs/install.md | 19 ++ internal/cli/model.go | 58 ++++- internal/cli/tui_actions.go | 45 ++++ internal/control/adminapi/api.go | 26 +++ internal/control/adminapi/routes.go | 2 + internal/control/modelserve/context.go | 229 ++++++++++++++++++++ internal/control/modelserve/context_test.go | 90 ++++++++ internal/control/modelserve/updates.go | 1 + internal/control/modelserve/updates_test.go | 8 + internal/control/modelsvc/modelsvc.go | 11 + internal/gguf/gguf.go | 5 + internal/modelplan/options.go | 16 ++ 14 files changed, 644 insertions(+), 1 deletion(-) create mode 100644 internal/control/modelserve/context.go create mode 100644 internal/control/modelserve/context_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 699a114..df2a3cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ APIs change between minors without a deprecation period. ### Added +- **A served model's context, changed.** `llmgw ctl model context MODEL + TOKENS [--slots N]`, `POST /api/v1/models/{name}/context`, and Context + length… in the console's model menu. The catalog, the engine's arguments and + the memory change together, checked against what the model was trained for + and what its cards hold; its replicas are then started again one by one. The + models API shows `context_length` and `max_output`. + - **A second model mirror, kept in step.** In a three-machine install the setup screens put one on a database machine (`--second-mirror 1|2`). An install with one mirror gets one with `sudo llmgw mirror add-peer user@host`. - **In step:** each mirror reads the other every thirty seconds. It copies what it lacks, checksum checked, and diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index d0926f5..defc36f 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -2719,6 +2719,57 @@ } } }, + "/api/v1/models/{name}/context": { + "post": { + "operationId": "postModelsNameContext", + "summary": "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted", + "tags": [ + "models" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "description": "The model's name in the catalog.", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/modelserve.ContextRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/modelserve.ContextChange" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/models/{name}/eval": { "post": { "operationId": "postModelsNameEval", @@ -7749,6 +7800,13 @@ "name": { "type": "string" }, + "context_length": { + "type": "integer", + "description": "ContextLength is what one request may use, MaxOutput what it may answer; 0 when the catalog does not say." + }, + "max_output": { + "type": "integer" + }, "canary": { "$ref": "#/components/schemas/catalog.Canary", "description": "Canary is a share of its requests sent to a candidate; CanaryOf is set on the candidate, naming the model it is a canary of." @@ -11005,6 +11063,76 @@ } } }, + "modelserve.ContextChange": { + "type": "object", + "description": "ContextChange is what was changed, and the replicas started again on it.", + "properties": { + "model": { + "type": "string" + }, + "before": { + "type": "integer" + }, + "after": { + "type": "integer" + }, + "max_output": { + "type": "integer" + }, + "trained": { + "type": "integer" + }, + "profiles": { + "type": "array", + "items": { + "$ref": "#/components/schemas/modelserve.ContextProfile" + } + }, + "restarted": { + "type": "integer" + }, + "note": { + "type": "string", + "description": "Note is what could not be checked, said once." + } + } + }, + "modelserve.ContextProfile": { + "type": "object", + "description": "ContextProfile is what one profile needs before and after.", + "properties": { + "profile": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "modelserve.ContextRequest": { + "type": "object", + "description": "ContextRequest is the context to change to: tokens one request may use, and for llama.cpp how many at once (its pool is the two multiplied).", + "properties": { + "tokens": { + "type": "integer" + }, + "slots": { + "type": "integer" + } + } + }, "modelserve.Job": { "type": "object", "description": "Job is one model being served, and where it stands.", diff --git a/docs/install.md b/docs/install.md index 82181ac..5a62046 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1629,6 +1629,25 @@ Not yet: models declared in server.yaml cannot name a peer's model as a fallback (the catalog in the file is checked before the peers are read), and a peer behind a private CA needs that CA in the system's trust store. +### Changing a served model's context + +How much one request may use is set when a model is served. To change it +afterwards: + +``` +llmgw ctl model context qwen-7b 32768 # tokens a request +llmgw ctl model context qwen-7b 32768 --slots 4 # llama.cpp: four at once +``` + +The catalog, the engine's arguments (`--ctx-size` and `--parallel` for +llama.cpp, `--max-model-len` for vLLM, `--context-length` for SGLang) and +the memory its KV cache takes change together. A context longer than the +model was trained for is refused, and so is one its cards cannot hold — with +the figures, so you know how far to go. Its replicas are then started again on +it, one by one, a new one ready before the old one stops. The console has the +same under a model's menu, Context length…. A model declared in the cluster's +settings is changed there (`llmgw ctl config edit`). + ### Trying new weights on some of the traffic first When a model's repository publishes new weights, `llmgw ctl model update` diff --git a/internal/cli/model.go b/internal/cli/model.go index 634b5d0..60e8ad6 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -18,6 +18,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck" ) @@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { Use: "model", Short: "Add or remove models while the cluster is running", } - c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client), + c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client), newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), @@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command { c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable") return c } + +func newModelContextCmd(client func() *ctlClient) *cobra.Command { + var slots int + c := &cobra.Command{ + Use: "context ", + Short: "Change how much one request to a served model may use", + Long: "The context lives in three places: the catalog (what a request may use, and\n" + + "what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" + + "--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" + + "changes all three together. It refuses more than the model was trained for,\n" + + "and a context its cards would not hold, saying what each would need. Its\n" + + "replicas are then started again on it, each new one before the old stops.\n\n" + + "--slots is how many requests a llama.cpp replica answers at once; its pool is\n" + + "slots × tokens. Left out, it keeps what it has.", + Example: " llmgw ctl model context qwen3-coder-30b 65536\n" + + " llmgw ctl model context qwen2.5-7b 16384 --slots 4", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + tokens, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("the context is a whole number of tokens, not %q", args[1]) + } + body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots}) + var v modelserve.ContextChange + if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput) + if v.Trained > 0 { + fmt.Fprintf(out, " (trained for %d)", v.Trained) + } + fmt.Fprintln(out) + for _, p := range v.Profiles { + line := " " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ") + switch { + case p.Before != "" && p.After != "": + line += "; " + p.Before + " → " + p.After + case p.After != "": + line += "; " + p.After + } + fmt.Fprintln(out, line) + } + if v.Note != "" { + fmt.Fprintln(out, " note: "+v.Note) + } + if v.Restarted > 0 { + fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model) + } + return nil + }, + } + c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)") + return c +} diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index e786202..d03705b 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -332,6 +332,30 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog { }}) return nil }}, + {"Context length…", func(m *consoleModel) tea.Cmd { + m.openForm(&formDialog{title: "Context of " + md.Name, + note: "What one request may use. The engine's arguments and the memory change with it, checked against " + + "what the cards hold; its replicas are then started again on it, one by one.", + fields: []formField{ + textF("tokens", strconv.Itoa(md.ContextLength), "a request's prompt and answer together"), + textF("slots", "", "llama.cpp: requests at once; empty keeps it"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + tokens, err := strconv.Atoi(v[0]) + if err != nil || tokens < 256 { + return nil, fmt.Errorf("the context is a whole number of tokens, at least 256") + } + slots := 0 + if v[1] != "" { + if slots, err = strconv.Atoi(v[1]); err != nil || slots < 1 { + return nil, fmt.Errorf("slots is a whole number from 1") + } + } + return m.call("Context of "+md.Name, http.MethodPost, path+"/context", + modelserve.ContextRequest{Tokens: tokens, Slots: slots}, contextSays), nil + }}) + return nil + }}, {"Resources…", func(m *consoleModel) tea.Cmd { profiles := md.Profiles if len(profiles) == 0 { @@ -898,3 +922,24 @@ func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequ } return req } + +// contextSays is a context change in a sentence or two. +func contextSays(raw json.RawMessage) string { + var v modelserve.ContextChange + if json.Unmarshal(raw, &v) != nil { + return "Done." + } + s := fmt.Sprintf("%s: %d → %d tokens a request, answers up to %d.", v.Model, v.Before, v.After, v.MaxOutput) + for _, p := range v.Profiles { + if p.After != "" { + s += "\n" + p.Profile + ": " + p.After + } + } + if v.Note != "" { + s += "\n" + v.Note + } + if v.Restarted > 0 { + s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted) + } + return s +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index ad491cf..c280a8d 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -205,6 +205,7 @@ func (a *API) Register(mux Mux) { mux.HandleFunc("GET /api/v1/updates", a.listUpdates) mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) + mux.HandleFunc("POST /api/v1/models/{name}/context", a.setModelContext) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) a.registerCanary(mux) a.registerEval(mux) @@ -445,6 +446,10 @@ func loadView(l *inventory.Load, now time.Time) *LoadView { // ModelView is a model as `ctl models` shows it. type ModelView struct { Name string `json:"name"` + // ContextLength is what one request may use, MaxOutput what it may + // answer; 0 when the catalog does not say. + ContextLength int `json:"context_length,omitempty"` + MaxOutput int `json:"max_output,omitempty"` // Canary is a share of its requests sent to a candidate; CanaryOf is // set on the candidate, naming the model it is a canary of. Canary *catalog.Canary `json:"canary,omitempty"` @@ -837,6 +842,7 @@ func (a *API) Models() []ModelView { h := health[m.Name] v := ModelView{ Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source, + ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Min: m.Scaling.Min, Max: m.Scaling.Max, Ready: ready[m.Name], Total: total[m.Name], Fallbacks: m.Fallbacks, @@ -1748,6 +1754,26 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, job) } +// setModelContext changes a served model's context, its engines' arguments +// and memory with it, and starts its replicas again on it. +func (a *API) setModelContext(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "this control plane cannot change a model's context") + return + } + var req modelserve.ContextRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + change, err := a.serve.SetContext(r.Context(), r.PathValue("name"), req, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, change) +} + func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) { out := make([]modelserve.UpdateInfo, 0) if a.serve != nil { diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index f3e7e37..b68b28c 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -213,6 +213,8 @@ var Routes = []Route{ {Method: "POST", Path: "/api/v1/models/{name}/update", Summary: "Move a model to its repository's newer weights", Description: "With a quality suite, or the model's standing gate, the new weights are asked it first as a hidden canary.", Request: modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted}, + {Method: "POST", Path: "/api/v1/models/{name}/context", Summary: "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted", + Request: modelserve.ContextRequest{}, Response: modelserve.ContextChange{}}, {Method: "POST", Path: "/api/v1/models/{name}/rollback", Summary: "Go back to the weights the last update replaced", Response: Result{}}, {Method: "POST", Path: "/api/v1/models/{name}/origin", Summary: "Record the repository a model's weights came from", diff --git a/internal/control/modelserve/context.go b/internal/control/modelserve/context.go new file mode 100644 index 0000000..8f0c7ae --- /dev/null +++ b/internal/control/modelserve/context.go @@ -0,0 +1,229 @@ +package modelserve + +import ( + "context" + "errors" + "fmt" + "math" + "slices" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" +) + +// A model's context changed after it was served: what one request may use, +// in the three places it lives — the catalog, the engine's arguments, and +// the memory its KV cache takes — together, checked against what the cards +// hold, then its replicas started again on it one by one. + +// ContextRequest is the context to change to: tokens one request may use, +// and for llama.cpp how many at once (its pool is the two multiplied). +type ContextRequest struct { + Tokens int `json:"tokens"` + Slots int `json:"slots,omitempty"` +} + +// ContextProfile is what one profile needs before and after. +type ContextProfile struct { + Profile string `json:"profile"` + Engine string `json:"engine"` + Before string `json:"before,omitempty"` + After string `json:"after,omitempty"` + Args []string `json:"args"` +} + +// ContextChange is what was changed, and the replicas started again on it. +type ContextChange struct { + Model string `json:"model"` + Before int `json:"before"` + After int `json:"after"` + MaxOutput int `json:"max_output"` + Trained int `json:"trained,omitempty"` + Profiles []ContextProfile `json:"profiles"` + Restarted int `json:"restarted"` + // Note is what could not be checked, said once. + Note string `json:"note,omitempty"` +} + +// shape is what sizes a context: a token's KV cache, the context the model +// was trained for, and its weights. +type shape struct { + kvPerToken int64 + trained int + weights int64 +} + +// SetContext changes a served model's context. +func (s *Server) SetContext(ctx context.Context, name string, req ContextRequest, by string) (ContextChange, error) { + if s.deps.Current == nil || s.deps.Models == nil { + return ContextChange{}, errors.New("this control plane cannot change the catalog") + } + var m catalog.Model + found := false + for _, c := range s.deps.Current() { + if c.Name == name { + m, found = c, true + } + } + switch { + case !found: + return ContextChange{}, fmt.Errorf("no model %q in the catalog", name) + case s.deps.Models.Declared(name): + return ContextChange{}, fmt.Errorf("model %q is declared in the cluster's settings: change its context there (llmgw ctl config edit)", name) + case req.Tokens < 256: + return ContextChange{}, errors.New("a context of at least 256 tokens") + case req.Slots < 0 || req.Slots > 64: + return ContextChange{}, errors.New("slots are 1 to 64") + } + sh, note := s.shapeOf(ctx, m) + if sh.trained > 0 && req.Tokens > sh.trained { + return ContextChange{}, fmt.Errorf("%s was trained for %d tokens; a longer context answers worse, and llama.cpp or vLLM refuses it", + name, sh.trained) + } + out := ContextChange{Model: name, Before: m.ContextLength, After: req.Tokens, Trained: sh.trained, Note: note} + profiles := slices.Clone(m.Profiles) + for i, p := range profiles { + next, change, err := resizeProfile(p, req, sh) + if err != nil { + return ContextChange{}, err + } + profiles[i] = next + out.Profiles = append(out.Profiles, change) + } + out.MaxOutput = m.MaxOutput + if m.MaxOutput == 0 || m.MaxOutput == maxOutput(m.ContextLength) { + out.MaxOutput = maxOutput(req.Tokens) + } + out.MaxOutput = min(out.MaxOutput, req.Tokens) + if _, err := s.deps.Models.SetContext(ctx, name, req.Tokens, out.MaxOutput, profiles, by); err != nil { + return ContextChange{}, err + } + // Started again on it, a new replica before the old one stops. + var ids []string + if s.deps.Fleet != nil { + for _, r := range s.deps.Fleet.Replicas() { + if r.Model == name { + ids = append(ids, r.ID) + } + } + } + if len(ids) > 0 && s.deps.Restart != nil { + if err := s.deps.Restart(ids...); err != nil { + return out, fmt.Errorf("the context is changed, and its replicas could not be started again on it: %w", err) + } + out.Restarted = len(ids) + } + return out, nil +} + +// shapeOf reads what sizes a model's context: a GGUF file's header from the +// mirror, else the original weights' config from their repository. +func (s *Server) shapeOf(ctx context.Context, m catalog.Model) (shape, string) { + var sh shape + if m.Origin != nil { + for _, f := range m.Origin.Files { + sh.weights += f.Bytes + } + } + switch { + case strings.HasSuffix(strings.ToLower(m.Source), ".gguf"): + h, err := gguf.FromURL(ctx, nil, m.Source) + if err != nil { + return sh, "its header could not be read (" + err.Error() + "), so the memory it needs is not checked" + } + sh.kvPerToken, sh.trained = h.KVBytesPerToken(), int(h.ContextLength) //nolint:gosec // a header's context + case m.Origin != nil && m.Origin.Repo != "": + mm, err := modelplan.FetchModel(ctx, m.Origin.Repo) + if err != nil { + return sh, "its config could not be read (" + err.Error() + "), so the memory it needs is not checked" + } + sh.kvPerToken, sh.trained = mm.KVBytesPerToken(modelplan.BF16), mm.MaxContext + default: + return sh, "where it came from is not known, so the memory it needs is not checked" + } + if sh.weights == 0 { + return sh, "its files' sizes are not known, so the memory it needs is not checked" + } + return sh, "" +} + +// resizeProfile is a profile with the context asked for: its engine's +// arguments, and its memory where it is on CPUs, refused where its cards +// would not hold it. +func resizeProfile(p catalog.Profile, req ContextRequest, sh shape) (catalog.Profile, ContextProfile, error) { + change := ContextProfile{Profile: p.Name, Engine: p.Engine} + var pool int + switch p.Engine { + case "llamacpp": + slots := req.Slots + if slots == 0 { + slots = max(argInt(p.Args, "--parallel"), 1) + } + pool = req.Tokens * slots + p.Args = setArg(setArg(p.Args, "--ctx-size", strconv.Itoa(pool)), "--parallel", strconv.Itoa(slots)) + case "vllm": + pool = req.Tokens + p.Args = setArg(p.Args, "--max-model-len", strconv.Itoa(pool)) + case "sglang": + pool = req.Tokens + p.Args = setArg(p.Args, "--context-length", strconv.Itoa(pool)) + default: + return p, change, fmt.Errorf("a %s profile (%s) does not take a context this way", p.Engine, p.Name) + } + change.Args = p.Args + if sh.kvPerToken == 0 || sh.weights == 0 { + return p, change, nil + } + if p.CPUOnly() { + need := sh.weights + int64(pool)*sh.kvPerToken + int64(1<<30) + sh.weights/20 + gb := int(math.Ceil(modelplan.GB(need))) + change.Before, change.After = fmt.Sprintf("%d GB of RAM", p.MinMemoryGB), fmt.Sprintf("%d GB of RAM", gb) + p.MinMemoryGB = gb + return p, change, nil + } + cards := max(p.GPUsPerNode, p.TP, 1) * max(p.Nodes, p.PP, 1) + need := modelplan.CardNeed(p.Engine, sh.weights, int64(pool)*sh.kvPerToken, cards) + change.After = fmt.Sprintf("%.1f GB a card of %d", need, cards) + if budget := modelplan.CardBudget(p.GPUClass); budget > 0 { + change.After += fmt.Sprintf(", of %.0f usable", budget) + if need > budget { + return p, change, fmt.Errorf("with %d tokens the %s profile needs %.1f GB on each of its %d %s cards, more than the %.0f GB they hold: "+ + "a shorter context, or the model on more cards", req.Tokens, p.Name, need, cards, p.GPUClass, budget) + } + } + return p, change, nil +} + +// setArg sets a flag's value in arguments, or adds it. +func setArg(args []string, flag, value string) []string { + out := slices.Clone(args) + for i := 0; i < len(out); i++ { + if out[i] == flag && i+1 < len(out) { + out[i+1] = value + return out + } + if v, ok := strings.CutPrefix(out[i], flag+"="); ok && v != "" { + out[i] = flag + "=" + value + return out + } + } + return append(out, flag, value) +} + +// argInt is a flag's whole-number value, 0 when absent. +func argInt(args []string, flag string) int { + for i, a := range args { + if a == flag && i+1 < len(args) { + n, _ := strconv.Atoi(args[i+1]) + return n + } + if v, ok := strings.CutPrefix(a, flag+"="); ok { + n, _ := strconv.Atoi(v) + return n + } + } + return 0 +} diff --git a/internal/control/modelserve/context_test.go b/internal/control/modelserve/context_test.go new file mode 100644 index 0000000..b5338d3 --- /dev/null +++ b/internal/control/modelserve/context_test.go @@ -0,0 +1,90 @@ +package modelserve + +import ( + "context" + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +func TestAContextChangeResizesTheEngineAndTheMemoryAndRestartsTheReplicas(t *testing.T) { + s, _, models, _, fleet := updSetup(t) + m := models.models["qwen"] + m.ContextLength, m.MaxOutput = 4096, maxOutput(4096) + m.Origin = &catalog.Origin{Repo: "o/r", Files: []catalog.OriginFile{{Path: "q4.gguf", Bytes: 2 << 30}}} + m.Profiles[0].Args = []string{"--ctx-size", "8192", "--parallel", "2"} + models.models["qwen"] = m + + got, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 32768}, "admin") + if err != nil { + t.Fatal(err) + } + if got.Before != 4096 || got.After != 32768 || got.Restarted != 2 || got.Note != "" { + t.Fatalf("%+v", got) + } + after := models.models["qwen"] + if after.ContextLength != 32768 || after.MaxOutput != maxOutput(32768) { + t.Errorf("the catalog says %d tokens, %d out", after.ContextLength, after.MaxOutput) + } + // Two slots kept: the pool is both. + if a := after.Profiles[0].Args; !slices.Equal(a, []string{"--ctx-size", "65536", "--parallel", "2"}) { + t.Errorf("args %v", a) + } + // 2 GiB of weights, 64k tokens at 28 KiB each, an engine's share. + if gb := after.Profiles[0].MinMemoryGB; gb < 4 || gb > 6 { + t.Errorf("memory %d GB", gb) + } + for _, r := range fleet.Replicas() { + if !strings.Contains(r.ID, "-new") { + t.Errorf("%s was not started again", r.ID) + } + } +} + +func TestAContextTheCardsCannotHoldIsRefusedWithTheFigures(t *testing.T) { + sh := shape{kvPerToken: 128 << 10, weights: 8 << 30} + p := catalog.Profile{Name: "gpu", Engine: "vllm", GPUClass: "t4-16g", GPUsPerNode: 1} + if _, _, err := resizeProfile(p, ContextRequest{Tokens: 131072}, sh); err == nil || !strings.Contains(err.Error(), "GB") { + t.Fatalf("not refused: %v", err) + } + next, change, err := resizeProfile(p, ContextRequest{Tokens: 4096}, sh) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(next.Args, []string{"--max-model-len", "4096"}) || !strings.Contains(change.After, "a card of 1") { + t.Errorf("%v %+v", next.Args, change) + } +} + +func TestAContextIsRefusedForADeclaredModelOrPastWhatItWasTrainedFor(t *testing.T) { + s, _, models, _, _ := updSetup(t) + models.declared["qwen"] = true + if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 8192}, "admin"); err == nil { + t.Error("a declared model's context was changed") + } + models.declared["qwen"] = false + if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 100}, "admin"); err == nil { + t.Error("100 tokens taken") + } + if _, err := s.SetContext(context.Background(), "nope", ContextRequest{Tokens: 8192}, "admin"); err == nil { + t.Error("no such model, and no error") + } +} + +func TestEngineArgumentsAreSetInEitherSpelling(t *testing.T) { + if a := setArg([]string{"--ctx-size=4096", "-ngl", "99"}, "--ctx-size", "8192"); !slices.Equal(a, []string{"--ctx-size=8192", "-ngl", "99"}) { + t.Error(a) + } + if a := setArg([]string{"-ngl", "99"}, "--parallel", "4"); !slices.Equal(a, []string{"-ngl", "99", "--parallel", "4"}) { + t.Error(a) + } + if argInt([]string{"--parallel=3"}, "--parallel") != 3 || argInt(nil, "--parallel") != 0 { + t.Error("argInt") + } + p := catalog.Profile{Name: "x", Engine: "comfyui"} + if _, _, err := resizeProfile(p, ContextRequest{Tokens: 4096}, shape{}); err == nil { + t.Error("a comfyui profile took a context") + } +} diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index 57e0363..4f5b1fb 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -34,6 +34,7 @@ type Models interface { Declared(name string) bool SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error + SetContext(ctx context.Context, name string, contextLength, maxOutput int, profiles []catalog.Profile, by string) (catalog.Model, error) // For a canary: the candidate added beside the model and taken out // again, and the share of requests it is given. Add(ctx context.Context, m catalog.Model, by string) error diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go index f723caa..12a2cc1 100644 --- a/internal/control/modelserve/updates_test.go +++ b/internal/control/modelserve/updates_test.go @@ -104,6 +104,14 @@ func (c *updModels) SetCanary(_ context.Context, n string, cn *catalog.Canary, _ c.models[n] = m return m, nil } +func (c *updModels) SetContext(_ context.Context, n string, ctxLen, maxOut int, ps []catalog.Profile, _ string) (catalog.Model, error) { + c.mu.Lock() + defer c.mu.Unlock() + m := c.models[n] + m.ContextLength, m.MaxOutput, m.Profiles = ctxLen, maxOut, ps + c.models[n] = m + return m, nil +} func (c *updModels) all() []catalog.Model { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 47fca5c..1526cd9 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -269,6 +269,17 @@ func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, // SetProfiles replaces a model's profiles, as a model added with them // would have: for a model whose engines are to run otherwise, its args among // it. Running replicas keep what they were started with until restarted. +// SetContext changes how much one request may use and what it may answer, +// with the profiles that carry it: their engines' arguments and memory. +func (s *Service) SetContext(ctx context.Context, name string, contextLength, maxOutput int, + profiles []catalog.Profile, by string, +) (catalog.Model, error) { + return s.update(ctx, name, by, "change its context", func(m *catalog.Model) error { + m.ContextLength, m.MaxOutput, m.Profiles = contextLength, maxOutput, profiles + return nil + }) +} + func (s *Service) SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error) { if len(profiles) == 0 { return catalog.Model{}, errors.New("a model needs a profile") diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go index 8d4257a..02b4671 100644 --- a/internal/gguf/gguf.go +++ b/internal/gguf/gguf.go @@ -41,6 +41,9 @@ type Header struct { // them; otherwise a head is EmbeddingLength / HeadCount. KeyLength uint64 ValueLength uint64 + // ContextLength is the context the model was trained for + // (.context_length); 0 when the file does not say. + ContextLength uint64 // Embedding is whether the model pools its tokens into one vector // (.pooling_type set and not none): an embedding model, which // llama.cpp serves with --embedding and does not chat. @@ -271,6 +274,8 @@ func Read(r io.Reader) (Header, error) { h.ValueLength = num case "pooling_type": h.Embedding = num > 0 + case "context_length": + h.ContextLength = num } } // The tokenizer's arrays — megabytes of vocabulary — come after the diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index deb295e..4e1c84d 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -500,3 +500,19 @@ func MemFromClass(class string) int { } return gb } + +// CardNeed is what one card of a replica holds, in GB: the weights and a +// context's KV cache divided over its cards, and the engine's own. +func CardNeed(engine string, weights, kv int64, cards int) float64 { + cards = max(cards, 1) + per := weights / int64(cards) + over := overheadBytes(per) + if engine == "llamacpp" { + over = ggufOverhead(per) + } + return GB(per + kv/int64(cards) + over) +} + +// CardBudget is what a card of a class may be planned to hold, in GB; 0 for +// a class whose memory its name does not say. +func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gpuShare }