A served model's context can be changed, engine and memory with it
ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1908bfb4b2
commit
986981ac81
14 files changed
+644
-1
No files matched your search
+57
-1
@@ -18,6 +18,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
|
||||
return c
|
||||
}
|
||||
|
||||
func newModelContextCmd(client func() *ctlClient) *cobra.Command {
|
||||
var slots int
|
||||
c := &cobra.Command{
|
||||
Use: "context <model> <tokens>",
|
||||
Short: "Change how much one request to a served model may use",
|
||||
Long: "The context lives in three places: the catalog (what a request may use, and\n" +
|
||||
"what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" +
|
||||
"--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" +
|
||||
"changes all three together. It refuses more than the model was trained for,\n" +
|
||||
"and a context its cards would not hold, saying what each would need. Its\n" +
|
||||
"replicas are then started again on it, each new one before the old stops.\n\n" +
|
||||
"--slots is how many requests a llama.cpp replica answers at once; its pool is\n" +
|
||||
"slots × tokens. Left out, it keeps what it has.",
|
||||
Example: " llmgw ctl model context qwen3-coder-30b 65536\n" +
|
||||
" llmgw ctl model context qwen2.5-7b 16384 --slots 4",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
tokens, err := strconv.Atoi(args[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("the context is a whole number of tokens, not %q", args[1])
|
||||
}
|
||||
body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots})
|
||||
var v modelserve.ContextChange
|
||||
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput)
|
||||
if v.Trained > 0 {
|
||||
fmt.Fprintf(out, " (trained for %d)", v.Trained)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
for _, p := range v.Profiles {
|
||||
line := " " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ")
|
||||
switch {
|
||||
case p.Before != "" && p.After != "":
|
||||
line += "; " + p.Before + " → " + p.After
|
||||
case p.After != "":
|
||||
line += "; " + p.After
|
||||
}
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
if v.Note != "" {
|
||||
fmt.Fprintln(out, " note: "+v.Note)
|
||||
}
|
||||
if v.Restarted > 0 {
|
||||
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
|
||||
return c
|
||||
}
|
||||
@@ -332,6 +332,30 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Context length…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Context of " + md.Name,
|
||||
note: "What one request may use. The engine's arguments and the memory change with it, checked against " +
|
||||
"what the cards hold; its replicas are then started again on it, one by one.",
|
||||
fields: []formField{
|
||||
textF("tokens", strconv.Itoa(md.ContextLength), "a request's prompt and answer together"),
|
||||
textF("slots", "", "llama.cpp: requests at once; empty keeps it"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
tokens, err := strconv.Atoi(v[0])
|
||||
if err != nil || tokens < 256 {
|
||||
return nil, fmt.Errorf("the context is a whole number of tokens, at least 256")
|
||||
}
|
||||
slots := 0
|
||||
if v[1] != "" {
|
||||
if slots, err = strconv.Atoi(v[1]); err != nil || slots < 1 {
|
||||
return nil, fmt.Errorf("slots is a whole number from 1")
|
||||
}
|
||||
}
|
||||
return m.call("Context of "+md.Name, http.MethodPost, path+"/context",
|
||||
modelserve.ContextRequest{Tokens: tokens, Slots: slots}, contextSays), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Resources…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := md.Profiles
|
||||
if len(profiles) == 0 {
|
||||
@@ -898,3 +922,24 @@ func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequ
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// contextSays is a context change in a sentence or two.
|
||||
func contextSays(raw json.RawMessage) string {
|
||||
var v modelserve.ContextChange
|
||||
if json.Unmarshal(raw, &v) != nil {
|
||||
return "Done."
|
||||
}
|
||||
s := fmt.Sprintf("%s: %d → %d tokens a request, answers up to %d.", v.Model, v.Before, v.After, v.MaxOutput)
|
||||
for _, p := range v.Profiles {
|
||||
if p.After != "" {
|
||||
s += "\n" + p.Profile + ": " + p.After
|
||||
}
|
||||
}
|
||||
if v.Note != "" {
|
||||
s += "\n" + v.Note
|
||||
}
|
||||
if v.Restarted > 0 {
|
||||
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user