A served model's context can be changed, engine and memory with it
CI / test (push) Successful in 12m0s

ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 15:50:04 +01:00
1 parent 1908bfb4b2
commit 986981ac81
14 files changed
+644 -1

No files matched your search

+57 -1
View File
@@ -18,6 +18,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
)
@@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
return c
}
func newModelContextCmd(client func() *ctlClient) *cobra.Command {
var slots int
c := &cobra.Command{
Use: "context <model> <tokens>",
Short: "Change how much one request to a served model may use",
Long: "The context lives in three places: the catalog (what a request may use, and\n" +
"what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" +
"--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" +
"changes all three together. It refuses more than the model was trained for,\n" +
"and a context its cards would not hold, saying what each would need. Its\n" +
"replicas are then started again on it, each new one before the old stops.\n\n" +
"--slots is how many requests a llama.cpp replica answers at once; its pool is\n" +
"slots × tokens. Left out, it keeps what it has.",
Example: " llmgw ctl model context qwen3-coder-30b 65536\n" +
" llmgw ctl model context qwen2.5-7b 16384 --slots 4",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
tokens, err := strconv.Atoi(args[1])
if err != nil {
return fmt.Errorf("the context is a whole number of tokens, not %q", args[1])
}
body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots})
var v modelserve.ContextChange
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput)
if v.Trained > 0 {
fmt.Fprintf(out, " (trained for %d)", v.Trained)
}
fmt.Fprintln(out)
for _, p := range v.Profiles {
line := " " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ")
switch {
case p.Before != "" && p.After != "":
line += "; " + p.Before + " → " + p.After
case p.After != "":
line += "; " + p.After
}
fmt.Fprintln(out, line)
}
if v.Note != "" {
fmt.Fprintln(out, " note: "+v.Note)
}
if v.Restarted > 0 {
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
}
return nil
},
}
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
return c
}
+45
View File
@@ -332,6 +332,30 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
}})
return nil
}},
{"Context length…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Context of " + md.Name,
note: "What one request may use. The engine's arguments and the memory change with it, checked against " +
"what the cards hold; its replicas are then started again on it, one by one.",
fields: []formField{
textF("tokens", strconv.Itoa(md.ContextLength), "a request's prompt and answer together"),
textF("slots", "", "llama.cpp: requests at once; empty keeps it"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
tokens, err := strconv.Atoi(v[0])
if err != nil || tokens < 256 {
return nil, fmt.Errorf("the context is a whole number of tokens, at least 256")
}
slots := 0
if v[1] != "" {
if slots, err = strconv.Atoi(v[1]); err != nil || slots < 1 {
return nil, fmt.Errorf("slots is a whole number from 1")
}
}
return m.call("Context of "+md.Name, http.MethodPost, path+"/context",
modelserve.ContextRequest{Tokens: tokens, Slots: slots}, contextSays), nil
}})
return nil
}},
{"Resources…", func(m *consoleModel) tea.Cmd {
profiles := md.Profiles
if len(profiles) == 0 {
@@ -898,3 +922,24 @@ func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequ
}
return req
}
// contextSays is a context change in a sentence or two.
func contextSays(raw json.RawMessage) string {
var v modelserve.ContextChange
if json.Unmarshal(raw, &v) != nil {
return "Done."
}
s := fmt.Sprintf("%s: %d → %d tokens a request, answers up to %d.", v.Model, v.Before, v.After, v.MaxOutput)
for _, p := range v.Profiles {
if p.After != "" {
s += "\n" + p.Profile + ": " + p.After
}
}
if v.Note != "" {
s += "\n" + v.Note
}
if v.Restarted > 0 {
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
}
return s
}