A served model's context can be changed, engine and memory with it
ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1908bfb4b2
commit
986981ac81
14 files changed
+644
-1
No files matched your search
+57
-1
@@ -18,6 +18,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -386,3 +387,58 @@ func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
|
||||
return c
|
||||
}
|
||||
|
||||
func newModelContextCmd(client func() *ctlClient) *cobra.Command {
|
||||
var slots int
|
||||
c := &cobra.Command{
|
||||
Use: "context <model> <tokens>",
|
||||
Short: "Change how much one request to a served model may use",
|
||||
Long: "The context lives in three places: the catalog (what a request may use, and\n" +
|
||||
"what it may answer), the engine's arguments (llama.cpp's --ctx-size and\n" +
|
||||
"--parallel, vLLM's --max-model-len), and the memory the KV cache takes. This\n" +
|
||||
"changes all three together. It refuses more than the model was trained for,\n" +
|
||||
"and a context its cards would not hold, saying what each would need. Its\n" +
|
||||
"replicas are then started again on it, each new one before the old stops.\n\n" +
|
||||
"--slots is how many requests a llama.cpp replica answers at once; its pool is\n" +
|
||||
"slots × tokens. Left out, it keeps what it has.",
|
||||
Example: " llmgw ctl model context qwen3-coder-30b 65536\n" +
|
||||
" llmgw ctl model context qwen2.5-7b 16384 --slots 4",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
tokens, err := strconv.Atoi(args[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("the context is a whole number of tokens, not %q", args[1])
|
||||
}
|
||||
body, _ := json.Marshal(modelserve.ContextRequest{Tokens: tokens, Slots: slots})
|
||||
var v modelserve.ContextChange
|
||||
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/context", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %d → %d tokens a request, answers up to %d", v.Model, v.Before, v.After, v.MaxOutput)
|
||||
if v.Trained > 0 {
|
||||
fmt.Fprintf(out, " (trained for %d)", v.Trained)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
for _, p := range v.Profiles {
|
||||
line := " " + p.Profile + " (" + p.Engine + "): " + strings.Join(p.Args, " ")
|
||||
switch {
|
||||
case p.Before != "" && p.After != "":
|
||||
line += "; " + p.Before + " → " + p.After
|
||||
case p.After != "":
|
||||
line += "; " + p.After
|
||||
}
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
if v.Note != "" {
|
||||
fmt.Fprintln(out, " note: "+v.Note)
|
||||
}
|
||||
if v.Restarted > 0 {
|
||||
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
|
||||
return c
|
||||
}
|
||||
Reference in new issue
Block a user