A served model's context can be changed, engine and memory with it

ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -29,6 +29,13 @@ APIs change between minors without a deprecation period.
 
 ### Added
 
+- **A served model's context, changed.** `llmgw ctl model context MODEL
+  TOKENS [--slots N]`, `POST /api/v1/models/{name}/context`, and Context
+  length… in the console's model menu. The catalog, the engine's arguments and
+  the memory change together, checked against what the model was trained for
+  and what its cards hold; its replicas are then started again one by one. The
+  models API shows `context_length` and `max_output`.
+
 - **A second model mirror, kept in step.** In a three-machine install the setup screens put one on a database machine
   (`--second-mirror 1|2`). An install with one mirror gets one with `sudo llmgw mirror add-peer user@host`.
   - **In step:** each mirror reads the other every thirty seconds. It copies what it lacks, checksum checked, and

docs/admin-api.openapi.json

@@ -2719,6 +2719,57 @@
         }
       }
     },
+    "/api/v1/models/{name}/context": {
+      "post": {
+        "operationId": "postModelsNameContext",
+        "summary": "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted",
+        "tags": [
+          "models"
+        ],
+        "parameters": [
+          {
+            "name": "name",
+            "in": "path",
+            "required": true,
+            "description": "The model's name in the catalog.",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "$ref": "#/components/parameters/Actor"
+          }
+        ],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/modelserve.ContextRequest"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/modelserve.ContextChange"
+                }
+              }
+            }
+          },
+          "401": {
+            "$ref": "#/components/responses/Unauthorized"
+          },
+          "default": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/api/v1/models/{name}/eval": {
       "post": {
         "operationId": "postModelsNameEval",
@@ -7749,6 +7800,13 @@
           "name": {
             "type": "string"
           },
+          "context_length": {
+            "type": "integer",
+            "description": "ContextLength is what one request may use, MaxOutput what it may answer; 0 when the catalog does not say."
+          },
+          "max_output": {
+            "type": "integer"
+          },
           "canary": {
             "$ref": "#/components/schemas/catalog.Canary",
             "description": "Canary is a share of its requests sent to a candidate; CanaryOf is set on the candidate, naming the model it is a canary of."
@@ -11005,6 +11063,76 @@
           }
         }
       },
+      "modelserve.ContextChange": {
+        "type": "object",
+        "description": "ContextChange is what was changed, and the replicas started again on it.",
+        "properties": {
+          "model": {
+            "type": "string"
+          },
+          "before": {
+            "type": "integer"
+          },
+          "after": {
+            "type": "integer"
+          },
+          "max_output": {
+            "type": "integer"
+          },
+          "trained": {
+            "type": "integer"
+          },
+          "profiles": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/modelserve.ContextProfile"
+            }
+          },
+          "restarted": {
+            "type": "integer"
+          },
+          "note": {
+            "type": "string",
+            "description": "Note is what could not be checked, said once."
+          }
+        }
+      },
+      "modelserve.ContextProfile": {
+        "type": "object",
+        "description": "ContextProfile is what one profile needs before and after.",
+        "properties": {
+          "profile": {
+            "type": "string"
+          },
+          "engine": {
+            "type": "string"
+          },
+          "before": {
+            "type": "string"
+          },
+          "after": {
+            "type": "string"
+          },
+          "args": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "modelserve.ContextRequest": {
+        "type": "object",
+        "description": "ContextRequest is the context to change to: tokens one request may use, and for llama.cpp how many at once (its pool is the two multiplied).",
+        "properties": {
+          "tokens": {
+            "type": "integer"
+          },
+          "slots": {
+            "type": "integer"
+          }
+        }
+      },
       "modelserve.Job": {
         "type": "object",
         "description": "Job is one model being served, and where it stands.",

docs/install.md

@@ -1629,6 +1629,25 @@ Not yet: models declared in server.yaml cannot name a peer's model as a
 fallback (the catalog in the file is checked before the peers are read), and
 a peer behind a private CA needs that CA in the system's trust store.
 
+### Changing a served model's context
+
+How much one request may use is set when a model is served. To change it
+afterwards:
+
+```
+llmgw ctl model context qwen-7b 32768             # tokens a request
+llmgw ctl model context qwen-7b 32768 --slots 4   # llama.cpp: four at once
+```
+
+The catalog, the engine's arguments (`--ctx-size` and `--parallel` for
+llama.cpp, `--max-model-len` for vLLM, `--context-length` for SGLang) and
+the memory its KV cache takes change together. A context longer than the
+model was trained for is refused, and so is one its cards cannot hold — with
+the figures, so you know how far to go. Its replicas are then started again on
+it, one by one, a new one ready before the old one stops. The console has the
+same under a model's menu, Context length…. A model declared in the cluster's
+settings is changed there (`llmgw ctl config edit`).
+
 ### Trying new weights on some of the traffic first
 
 When a model's repository publishes new weights, `llmgw ctl model update`

internal/cli/model.go

@@ -18,6 +18,7 @@ import (
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
 )
 
@@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 		Use:   "model",
 		Short: "Add or remove models while the cluster is running",
 	}
-	c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
+	c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
 		newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
 		newTargetCmd(client, asJSON), newAdaptersCmd(client),
 		newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
 	c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
 	return c
 }
+
+func newModelContextCmd(client func() *ctlClient) *cobra.Command {
+	var slots int
+	c := &cobra.Command{
+		Use:   "context <model> <tokens>",
+		Short: "Change how much one request to a served model may use",
+		Long: "The context lives in three places: the catalog (what a request may use, and\n" +
+			"what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" +
+			"--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" +
+			"changes all three together. It refuses more than the model was trained for,\n" +
+			"and a context its cards would not hold, saying what each would need. Its\n" +
+			"replicas are then started again on it, each new one before the old stops.\n\n" +
+			"--slots is how many requests a llama.cpp replica answers at once; its pool is\n" +
+			"slots × tokens. Left out, it keeps what it has.",
+		Example: "  llmgw ctl model context qwen3-coder-30b 65536\n" +
+			"  llmgw ctl model context qwen2.5-7b 16384 --slots 4",
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			tokens, err := strconv.Atoi(args[1])
+			if err != nil {
+				return fmt.Errorf("the context is a whole number of tokens, not %q", args[1])
+			}
+			body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots})
+			var v modelserve.ContextChange
+			if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil {
+				return err
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput)
+			if v.Trained > 0 {
+				fmt.Fprintf(out, " (trained for %d)", v.Trained)
+			}
+			fmt.Fprintln(out)
+			for _, p := range v.Profiles {
+				line := "  " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ")
+				switch {
+				case p.Before != "" && p.After != "":
+					line += "; " + p.Before + " → " + p.After
+				case p.After != "":
+					line += "; " + p.After
+				}
+				fmt.Fprintln(out, line)
+			}
+			if v.Note != "" {
+				fmt.Fprintln(out, "  note: "+v.Note)
+			}
+			if v.Restarted > 0 {
+				fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
+			}
+			return nil
+		},
+	}
+	c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
+	return c
+}

internal/cli/tui_actions.go

@@ -332,6 +332,30 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
 				}})
 			return nil
 		}},
+		{"Context length…", func(m *consoleModel) tea.Cmd {
+			m.openForm(&formDialog{title: "Context of " + md.Name,
+				note: "What one request may use. The engine's arguments and the memory change with it, checked against " +
+					"what the cards hold; its replicas are then started again on it, one by one.",
+				fields: []formField{
+					textF("tokens", strconv.Itoa(md.ContextLength), "a request's prompt and answer together"),
+					textF("slots", "", "llama.cpp: requests at once; empty keeps it"),
+				},
+				submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
+					tokens, err := strconv.Atoi(v[0])
+					if err != nil || tokens < 256 {
+						return nil, fmt.Errorf("the context is a whole number of tokens, at least 256")
+					}
+					slots := 0
+					if v[1] != "" {
+						if slots, err = strconv.Atoi(v[1]); err != nil || slots < 1 {
+							return nil, fmt.Errorf("slots is a whole number from 1")
+						}
+					}
+					return m.call("Context of "+md.Name, http.MethodPost, path+"/context",
+						modelserve.ContextRequest{Tokens: tokens, Slots: slots}, contextSays), nil
+				}})
+			return nil
+		}},
 		{"Resources…", func(m *consoleModel) tea.Cmd {
 			profiles := md.Profiles
 			if len(profiles) == 0 {
@@ -898,3 +922,24 @@ func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequ
 	}
 	return req
 }
+
+// contextSays is a context change in a sentence or two.
+func contextSays(raw json.RawMessage) string {
+	var v modelserve.ContextChange
+	if json.Unmarshal(raw, &v) != nil {
+		return "Done."
+	}
+	s := fmt.Sprintf("%s: %d → %d tokens a request, answers up to %d.", v.Model, v.Before, v.After, v.MaxOutput)
+	for _, p := range v.Profiles {
+		if p.After != "" {
+			s += "\n" + p.Profile + ": " + p.After
+		}
+	}
+	if v.Note != "" {
+		s += "\n" + v.Note
+	}
+	if v.Restarted > 0 {
+		s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
+	}
+	return s
+}

internal/control/adminapi/api.go

@@ -205,6 +205,7 @@ func (a *API) Register(mux Mux) {
 	mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
 	mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
 	mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
+	mux.HandleFunc("POST /api/v1/models/{name}/context", a.setModelContext)
 	mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
 	a.registerCanary(mux)
 	a.registerEval(mux)
@@ -445,6 +446,10 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
 // ModelView is a model as `ctl models` shows it.
 type ModelView struct {
 	Name string `json:"name"`
+	// ContextLength is what one request may use, MaxOutput what it may
+	// answer; 0 when the catalog does not say.
+	ContextLength int `json:"context_length,omitempty"`
+	MaxOutput     int `json:"max_output,omitempty"`
 	// Canary is a share of its requests sent to a candidate; CanaryOf is
 	// set on the candidate, naming the model it is a canary of.
 	Canary   *catalog.Canary `json:"canary,omitempty"`
@@ -837,6 +842,7 @@ func (a *API) Models() []ModelView {
 		h := health[m.Name]
 		v := ModelView{
 			Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
+			ContextLength: m.ContextLength, MaxOutput: m.MaxOutput,
 			Min: m.Scaling.Min, Max: m.Scaling.Max,
 			Ready: ready[m.Name], Total: total[m.Name],
 			Fallbacks: m.Fallbacks,
@@ -1748,6 +1754,26 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, job)
 }
 
+// setModelContext changes a served model's context, its engines' arguments
+// and memory with it, and starts its replicas again on it.
+func (a *API) setModelContext(w http.ResponseWriter, r *http.Request) {
+	if a.serve == nil {
+		writeError(w, http.StatusNotImplemented, "this control plane cannot change a model's context")
+		return
+	}
+	var req modelserve.ContextRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	change, err := a.serve.SetContext(r.Context(), r.PathValue("name"), req, actorOf(r))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, change)
+}
+
 func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) {
 	out := make([]modelserve.UpdateInfo, 0)
 	if a.serve != nil {

internal/control/adminapi/routes.go

@@ -213,6 +213,8 @@ var Routes = []Route{
 	{Method: "POST", Path: "/api/v1/models/{name}/update", Summary: "Move a model to its repository's newer weights",
 		Description: "With a quality suite, or the model's standing gate, the new weights are asked it first as a hidden canary.",
 		Request:     modelserve.Change{}, OptionalBody: true, Response: modelserve.Job{}, Status: http.StatusAccepted},
+	{Method: "POST", Path: "/api/v1/models/{name}/context", Summary: "Change a served model's context: the catalog, its engines' arguments and memory, then its replicas restarted",
+		Request: modelserve.ContextRequest{}, Response: modelserve.ContextChange{}},
 	{Method: "POST", Path: "/api/v1/models/{name}/rollback", Summary: "Go back to the weights the last update replaced",
 		Response: Result{}},
 	{Method: "POST", Path: "/api/v1/models/{name}/origin", Summary: "Record the repository a model's weights came from",

internal/control/modelserve/context.go

@@ -0,0 +1,229 @@
+package modelserve
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"slices"
+	"strconv"
+	"strings"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
+)
+
+// A model's context changed after it was served: what one request may use,
+// in the three places it lives — the catalog, the engine's arguments, and
+// the memory its KV cache takes — together, checked against what the cards
+// hold, then its replicas started again on it one by one.
+
+// ContextRequest is the context to change to: tokens one request may use,
+// and for llama.cpp how many at once (its pool is the two multiplied).
+type ContextRequest struct {
+	Tokens int `json:"tokens"`
+	Slots  int `json:"slots,omitempty"`
+}
+
+// ContextProfile is what one profile needs before and after.
+type ContextProfile struct {
+	Profile string   `json:"profile"`
+	Engine  string   `json:"engine"`
+	Before  string   `json:"before,omitempty"`
+	After   string   `json:"after,omitempty"`
+	Args    []string `json:"args"`
+}
+
+// ContextChange is what was changed, and the replicas started again on it.
+type ContextChange struct {
+	Model     string           `json:"model"`
+	Before    int              `json:"before"`
+	After     int              `json:"after"`
+	MaxOutput int              `json:"max_output"`
+	Trained   int              `json:"trained,omitempty"`
+	Profiles  []ContextProfile `json:"profiles"`
+	Restarted int              `json:"restarted"`
+	// Note is what could not be checked, said once.
+	Note string `json:"note,omitempty"`
+}
+
+// shape is what sizes a context: a token's KV cache, the context the model
+// was trained for, and its weights.
+type shape struct {
+	kvPerToken int64
+	trained    int
+	weights    int64
+}
+
+// SetContext changes a served model's context.
+func (s *Server) SetContext(ctx context.Context, name string, req ContextRequest, by string) (ContextChange, error) {
+	if s.deps.Current == nil || s.deps.Models == nil {
+		return ContextChange{}, errors.New("this control plane cannot change the catalog")
+	}
+	var m catalog.Model
+	found := false
+	for _, c := range s.deps.Current() {
+		if c.Name == name {
+			m, found = c, true
+		}
+	}
+	switch {
+	case !found:
+		return ContextChange{}, fmt.Errorf("no model %q in the catalog", name)
+	case s.deps.Models.Declared(name):
+		return ContextChange{}, fmt.Errorf("model %q is declared in the cluster's settings: change its context there (llmgw ctl config edit)", name)
+	case req.Tokens < 256:
+		return ContextChange{}, errors.New("a context of at least 256 tokens")
+	case req.Slots < 0 || req.Slots > 64:
+		return ContextChange{}, errors.New("slots are 1 to 64")
+	}
+	sh, note := s.shapeOf(ctx, m)
+	if sh.trained > 0 && req.Tokens > sh.trained {
+		return ContextChange{}, fmt.Errorf("%s was trained for %d tokens; a longer context answers worse, and llama.cpp or vLLM refuses it",
+			name, sh.trained)
+	}
+	out := ContextChange{Model: name, Before: m.ContextLength, After: req.Tokens, Trained: sh.trained, Note: note}
+	profiles := slices.Clone(m.Profiles)
+	for i, p := range profiles {
+		next, change, err := resizeProfile(p, req, sh)
+		if err != nil {
+			return ContextChange{}, err
+		}
+		profiles[i] = next
+		out.Profiles = append(out.Profiles, change)
+	}
+	out.MaxOutput = m.MaxOutput
+	if m.MaxOutput == 0 || m.MaxOutput == maxOutput(m.ContextLength) {
+		out.MaxOutput = maxOutput(req.Tokens)
+	}
+	out.MaxOutput = min(out.MaxOutput, req.Tokens)
+	if _, err := s.deps.Models.SetContext(ctx, name, req.Tokens, out.MaxOutput, profiles, by); err != nil {
+		return ContextChange{}, err
+	}
+	// Started again on it, a new replica before the old one stops.
+	var ids []string
+	if s.deps.Fleet != nil {
+		for _, r := range s.deps.Fleet.Replicas() {
+			if r.Model == name {
+				ids = append(ids, r.ID)
+			}
+		}
+	}
+	if len(ids) > 0 && s.deps.Restart != nil {
+		if err := s.deps.Restart(ids...); err != nil {
+			return out, fmt.Errorf("the context is changed, and its replicas could not be started again on it: %w", err)
+		}
+		out.Restarted = len(ids)
+	}
+	return out, nil
+}
+
+// shapeOf reads what sizes a model's context: a GGUF file's header from the
+// mirror, else the original weights' config from their repository.
+func (s *Server) shapeOf(ctx context.Context, m catalog.Model) (shape, string) {
+	var sh shape
+	if m.Origin != nil {
+		for _, f := range m.Origin.Files {
+			sh.weights += f.Bytes
+		}
+	}
+	switch {
+	case strings.HasSuffix(strings.ToLower(m.Source), ".gguf"):
+		h, err := gguf.FromURL(ctx, nil, m.Source)
+		if err != nil {
+			return sh, "its header could not be read (" + err.Error() + "), so the memory it needs is not checked"
+		}
+		sh.kvPerToken, sh.trained = h.KVBytesPerToken(), int(h.ContextLength) //nolint:gosec // a header's context
+	case m.Origin != nil && m.Origin.Repo != "":
+		mm, err := modelplan.FetchModel(ctx, m.Origin.Repo)
+		if err != nil {
+			return sh, "its config could not be read (" + err.Error() + "), so the memory it needs is not checked"
+		}
+		sh.kvPerToken, sh.trained = mm.KVBytesPerToken(modelplan.BF16), mm.MaxContext
+	default:
+		return sh, "where it came from is not known, so the memory it needs is not checked"
+	}
+	if sh.weights == 0 {
+		return sh, "its files' sizes are not known, so the memory it needs is not checked"
+	}
+	return sh, ""
+}
+
+// resizeProfile is a profile with the context asked for: its engine's
+// arguments, and its memory where it is on CPUs, refused where its cards
+// would not hold it.
+func resizeProfile(p catalog.Profile, req ContextRequest, sh shape) (catalog.Profile, ContextProfile, error) {
+	change := ContextProfile{Profile: p.Name, Engine: p.Engine}
+	var pool int
+	switch p.Engine {
+	case "llamacpp":
+		slots := req.Slots
+		if slots == 0 {
+			slots = max(argInt(p.Args, "--parallel"), 1)
+		}
+		pool = req.Tokens * slots
+		p.Args = setArg(setArg(p.Args, "--ctx-size", strconv.Itoa(pool)), "--parallel", strconv.Itoa(slots))
+	case "vllm":
+		pool = req.Tokens
+		p.Args = setArg(p.Args, "--max-model-len", strconv.Itoa(pool))
+	case "sglang":
+		pool = req.Tokens
+		p.Args = setArg(p.Args, "--context-length", strconv.Itoa(pool))
+	default:
+		return p, change, fmt.Errorf("a %s profile (%s) does not take a context this way", p.Engine, p.Name)
+	}
+	change.Args = p.Args
+	if sh.kvPerToken == 0 || sh.weights == 0 {
+		return p, change, nil
+	}
+	if p.CPUOnly() {
+		need := sh.weights + int64(pool)*sh.kvPerToken + int64(1<<30) + sh.weights/20
+		gb := int(math.Ceil(modelplan.GB(need)))
+		change.Before, change.After = fmt.Sprintf("%d GB of RAM", p.MinMemoryGB), fmt.Sprintf("%d GB of RAM", gb)
+		p.MinMemoryGB = gb
+		return p, change, nil
+	}
+	cards := max(p.GPUsPerNode, p.TP, 1) * max(p.Nodes, p.PP, 1)
+	need := modelplan.CardNeed(p.Engine, sh.weights, int64(pool)*sh.kvPerToken, cards)
+	change.After = fmt.Sprintf("%.1f GB a card of %d", need, cards)
+	if budget := modelplan.CardBudget(p.GPUClass); budget > 0 {
+		change.After += fmt.Sprintf(", of %.0f usable", budget)
+		if need > budget {
+			return p, change, fmt.Errorf("with %d tokens the %s profile needs %.1f GB on each of its %d %s cards, more than the %.0f GB they hold: "+
+				"a shorter context, or the model on more cards", req.Tokens, p.Name, need, cards, p.GPUClass, budget)
+		}
+	}
+	return p, change, nil
+}
+
+// setArg sets a flag's value in arguments, or adds it.
+func setArg(args []string, flag, value string) []string {
+	out := slices.Clone(args)
+	for i := 0; i < len(out); i++ {
+		if out[i] == flag && i+1 < len(out) {
+			out[i+1] = value
+			return out
+		}
+		if v, ok := strings.CutPrefix(out[i], flag+"="); ok && v != "" {
+			out[i] = flag + "=" + value
+			return out
+		}
+	}
+	return append(out, flag, value)
+}
+
+// argInt is a flag's whole-number value, 0 when absent.
+func argInt(args []string, flag string) int {
+	for i, a := range args {
+		if a == flag && i+1 < len(args) {
+			n, _ := strconv.Atoi(args[i+1])
+			return n
+		}
+		if v, ok := strings.CutPrefix(a, flag+"="); ok {
+			n, _ := strconv.Atoi(v)
+			return n
+		}
+	}
+	return 0
+}

internal/control/modelserve/context_test.go

@@ -0,0 +1,90 @@
+package modelserve
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+)
+
+func TestAContextChangeResizesTheEngineAndTheMemoryAndRestartsTheReplicas(t *testing.T) {
+	s, _, models, _, fleet := updSetup(t)
+	m := models.models["qwen"]
+	m.ContextLength, m.MaxOutput = 4096, maxOutput(4096)
+	m.Origin = &catalog.Origin{Repo: "o/r", Files: []catalog.OriginFile{{Path: "q4.gguf", Bytes: 2 << 30}}}
+	m.Profiles[0].Args = []string{"--ctx-size", "8192", "--parallel", "2"}
+	models.models["qwen"] = m
+
+	got, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 32768}, "admin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Before != 4096 || got.After != 32768 || got.Restarted != 2 || got.Note != "" {
+		t.Fatalf("%+v", got)
+	}
+	after := models.models["qwen"]
+	if after.ContextLength != 32768 || after.MaxOutput != maxOutput(32768) {
+		t.Errorf("the catalog says %d tokens, %d out", after.ContextLength, after.MaxOutput)
+	}
+	// Two slots kept: the pool is both.
+	if a := after.Profiles[0].Args; !slices.Equal(a, []string{"--ctx-size", "65536", "--parallel", "2"}) {
+		t.Errorf("args %v", a)
+	}
+	// 2 GiB of weights, 64k tokens at 28 KiB each, an engine's share.
+	if gb := after.Profiles[0].MinMemoryGB; gb < 4 || gb > 6 {
+		t.Errorf("memory %d GB", gb)
+	}
+	for _, r := range fleet.Replicas() {
+		if !strings.Contains(r.ID, "-new") {
+			t.Errorf("%s was not started again", r.ID)
+		}
+	}
+}
+
+func TestAContextTheCardsCannotHoldIsRefusedWithTheFigures(t *testing.T) {
+	sh := shape{kvPerToken: 128 << 10, weights: 8 << 30}
+	p := catalog.Profile{Name: "gpu", Engine: "vllm", GPUClass: "t4-16g", GPUsPerNode: 1}
+	if _, _, err := resizeProfile(p, ContextRequest{Tokens: 131072}, sh); err == nil || !strings.Contains(err.Error(), "GB") {
+		t.Fatalf("not refused: %v", err)
+	}
+	next, change, err := resizeProfile(p, ContextRequest{Tokens: 4096}, sh)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(next.Args, []string{"--max-model-len", "4096"}) || !strings.Contains(change.After, "a card of 1") {
+		t.Errorf("%v %+v", next.Args, change)
+	}
+}
+
+func TestAContextIsRefusedForADeclaredModelOrPastWhatItWasTrainedFor(t *testing.T) {
+	s, _, models, _, _ := updSetup(t)
+	models.declared["qwen"] = true
+	if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 8192}, "admin"); err == nil {
+		t.Error("a declared model's context was changed")
+	}
+	models.declared["qwen"] = false
+	if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 100}, "admin"); err == nil {
+		t.Error("100 tokens taken")
+	}
+	if _, err := s.SetContext(context.Background(), "nope", ContextRequest{Tokens: 8192}, "admin"); err == nil {
+		t.Error("no such model, and no error")
+	}
+}
+
+func TestEngineArgumentsAreSetInEitherSpelling(t *testing.T) {
+	if a := setArg([]string{"--ctx-size=4096", "-ngl", "99"}, "--ctx-size", "8192"); !slices.Equal(a, []string{"--ctx-size=8192", "-ngl", "99"}) {
+		t.Error(a)
+	}
+	if a := setArg([]string{"-ngl", "99"}, "--parallel", "4"); !slices.Equal(a, []string{"-ngl", "99", "--parallel", "4"}) {
+		t.Error(a)
+	}
+	if argInt([]string{"--parallel=3"}, "--parallel") != 3 || argInt(nil, "--parallel") != 0 {
+		t.Error("argInt")
+	}
+	p := catalog.Profile{Name: "x", Engine: "comfyui"}
+	if _, _, err := resizeProfile(p, ContextRequest{Tokens: 4096}, shape{}); err == nil {
+		t.Error("a comfyui profile took a context")
+	}
+}

internal/control/modelserve/updates.go

@@ -34,6 +34,7 @@ type Models interface {
 	Declared(name string) bool
 	SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error)
 	SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error
+	SetContext(ctx context.Context, name string, contextLength, maxOutput int, profiles []catalog.Profile, by string) (catalog.Model, error)
 	// For a canary: the candidate added beside the model and taken out
 	// again, and the share of requests it is given.
 	Add(ctx context.Context, m catalog.Model, by string) error

internal/control/modelserve/updates_test.go

@@ -104,6 +104,14 @@ func (c *updModels) SetCanary(_ context.Context, n string, cn *catalog.Canary, _
 	c.models[n] = m
 	return m, nil
 }
+func (c *updModels) SetContext(_ context.Context, n string, ctxLen, maxOut int, ps []catalog.Profile, _ string) (catalog.Model, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	m := c.models[n]
+	m.ContextLength, m.MaxOutput, m.Profiles = ctxLen, maxOut, ps
+	c.models[n] = m
+	return m, nil
+}
 func (c *updModels) all() []catalog.Model {
 	c.mu.Lock()
 	defer c.mu.Unlock()

internal/control/modelsvc/modelsvc.go

@@ -269,6 +269,17 @@ func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB,
 // SetProfiles replaces a model's profiles, as a model added with them
 // would have: for a model whose engines are to run otherwise, its args among
 // it. Running replicas keep what they were started with until restarted.
+// SetContext changes how much one request may use and what it may answer,
+// with the profiles that carry it: their engines' arguments and memory.
+func (s *Service) SetContext(ctx context.Context, name string, contextLength, maxOutput int,
+	profiles []catalog.Profile, by string,
+) (catalog.Model, error) {
+	return s.update(ctx, name, by, "change its context", func(m *catalog.Model) error {
+		m.ContextLength, m.MaxOutput, m.Profiles = contextLength, maxOutput, profiles
+		return nil
+	})
+}
+
 func (s *Service) SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error) {
 	if len(profiles) == 0 {
 		return catalog.Model{}, errors.New("a model needs a profile")

internal/gguf/gguf.go

@@ -41,6 +41,9 @@ type Header struct {
 	// them; otherwise a head is EmbeddingLength / HeadCount.
 	KeyLength   uint64
 	ValueLength uint64
+	// ContextLength is the context the model was trained for
+	// (<arch>.context_length); 0 when the file does not say.
+	ContextLength uint64
 	// Embedding is whether the model pools its tokens into one vector
 	// (<arch>.pooling_type set and not none): an embedding model, which
 	// llama.cpp serves with --embedding and does not chat.
@@ -271,6 +274,8 @@ func Read(r io.Reader) (Header, error) {
 				h.ValueLength = num
 			case "pooling_type":
 				h.Embedding = num > 0
+			case "context_length":
+				h.ContextLength = num
 			}
 		}
 		// The tokenizer's arrays — megabytes of vocabulary — come after the

internal/modelplan/options.go

@@ -500,3 +500,19 @@ func MemFromClass(class string) int {
 	}
 	return gb
 }
+
+// CardNeed is what one card of a replica holds, in GB: the weights and a
+// context's KV cache divided over its cards, and the engine's own.
+func CardNeed(engine string, weights, kv int64, cards int) float64 {
+	cards = max(cards, 1)
+	per := weights / int64(cards)
+	over := overheadBytes(per)
+	if engine == "llamacpp" {
+		over = ggufOverhead(per)
+	}
+	return GB(per + kv/int64(cards) + over)
+}
+
+// CardBudget is what a card of a class may be planned to hold, in GB; 0 for
+// a class whose memory its name does not say.
+func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gpuShare }