A served model's context can be changed, engine and memory with it

ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 15:50:04 +01:00
1 parent 1908bfb4b2
commit 986981ac81
14 files changed
+644 -1

No files matched your search

+57 -1
View File
@@ -18,6 +18,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
)
@@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
return c
}
func newModelContextCmd(client func() *ctlClient) *cobra.Command {
var slots int
c := &cobra.Command{
Use: "context <model> <tokens>",
Short: "Change how much one request to a served model may use",
Long: "The context lives in three places: the catalog (what a request may use, and\n" +
"what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" +
"--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" +
"changes all three together. It refuses more than the model was trained for,\n" +
"and a context its cards would not hold, saying what each would need. Its\n" +
"replicas are then started again on it, each new one before the old stops.\n\n" +
"--slots is how many requests a llama.cpp replica answers at once; its pool is\n" +
"slots × tokens. Left out, it keeps what it has.",
Example: " llmgw ctl model context qwen3-coder-30b 65536\n" +
" llmgw ctl model context qwen2.5-7b 16384 --slots 4",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
tokens, err := strconv.Atoi(args[1])
if err != nil {
return fmt.Errorf("the context is a whole number of tokens, not %q", args[1])
}
body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots})
var v modelserve.ContextChange
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput)
if v.Trained > 0 {
fmt.Fprintf(out, " (trained for %d)", v.Trained)
}
fmt.Fprintln(out)
for _, p := range v.Profiles {
line := " " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ")
switch {
case p.Before != "" && p.After != "":
line += "; " + p.Before + " → " + p.After
case p.After != "":
line += "; " + p.After
}
fmt.Fprintln(out, line)
}
if v.Note != "" {
fmt.Fprintln(out, " note: "+v.Note)
}
if v.Restarted > 0 {
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
}
return nil
},
}
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
return c
}