Ilker ManapandClaude Opus 5.5
986981ac81
ctl model context MODEL TOKENS [--slots N], POST /api/v1/models/{name}/context
and the console's Context length…: the catalog, the engine's arguments and
the KV memory change together, refused past the trained context or what the
cards hold, then the replicas restart one by one.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
91 lines
3.3 KiB
Go
91 lines
3.3 KiB
Go
package modelserve
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
func TestAContextChangeResizesTheEngineAndTheMemoryAndRestartsTheReplicas(t *testing.T) {
|
|
s, _, models, _, fleet := updSetup(t)
|
|
m := models.models["qwen"]
|
|
m.ContextLength, m.MaxOutput = 4096, maxOutput(4096)
|
|
m.Origin = &catalog.Origin{Repo: "o/r", Files: []catalog.OriginFile{{Path: "q4.gguf", Bytes: 2 << 30}}}
|
|
m.Profiles[0].Args = []string{"--ctx-size", "8192", "--parallel", "2"}
|
|
models.models["qwen"] = m
|
|
|
|
got, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 32768}, "admin")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Before != 4096 || got.After != 32768 || got.Restarted != 2 || got.Note != "" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
after := models.models["qwen"]
|
|
if after.ContextLength != 32768 || after.MaxOutput != maxOutput(32768) {
|
|
t.Errorf("the catalog says %d tokens, %d out", after.ContextLength, after.MaxOutput)
|
|
}
|
|
// Two slots kept: the pool is both.
|
|
if a := after.Profiles[0].Args; !slices.Equal(a, []string{"--ctx-size", "65536", "--parallel", "2"}) {
|
|
t.Errorf("args %v", a)
|
|
}
|
|
// 2 GiB of weights, 64k tokens at 28 KiB each, an engine's share.
|
|
if gb := after.Profiles[0].MinMemoryGB; gb < 4 || gb > 6 {
|
|
t.Errorf("memory %d GB", gb)
|
|
}
|
|
for _, r := range fleet.Replicas() {
|
|
if !strings.Contains(r.ID, "-new") {
|
|
t.Errorf("%s was not started again", r.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAContextTheCardsCannotHoldIsRefusedWithTheFigures(t *testing.T) {
|
|
sh := shape{kvPerToken: 128 << 10, weights: 8 << 30}
|
|
p := catalog.Profile{Name: "gpu", Engine: "vllm", GPUClass: "t4-16g", GPUsPerNode: 1}
|
|
if _, _, err := resizeProfile(p, ContextRequest{Tokens: 131072}, sh); err == nil || !strings.Contains(err.Error(), "GB") {
|
|
t.Fatalf("not refused: %v", err)
|
|
}
|
|
next, change, err := resizeProfile(p, ContextRequest{Tokens: 4096}, sh)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Equal(next.Args, []string{"--max-model-len", "4096"}) || !strings.Contains(change.After, "a card of 1") {
|
|
t.Errorf("%v %+v", next.Args, change)
|
|
}
|
|
}
|
|
|
|
func TestAContextIsRefusedForADeclaredModelOrPastWhatItWasTrainedFor(t *testing.T) {
|
|
s, _, models, _, _ := updSetup(t)
|
|
models.declared["qwen"] = true
|
|
if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 8192}, "admin"); err == nil {
|
|
t.Error("a declared model's context was changed")
|
|
}
|
|
models.declared["qwen"] = false
|
|
if _, err := s.SetContext(context.Background(), "qwen", ContextRequest{Tokens: 100}, "admin"); err == nil {
|
|
t.Error("100 tokens taken")
|
|
}
|
|
if _, err := s.SetContext(context.Background(), "nope", ContextRequest{Tokens: 8192}, "admin"); err == nil {
|
|
t.Error("no such model, and no error")
|
|
}
|
|
}
|
|
|
|
func TestEngineArgumentsAreSetInEitherSpelling(t *testing.T) {
|
|
if a := setArg([]string{"--ctx-size=4096", "-ngl", "99"}, "--ctx-size", "8192"); !slices.Equal(a, []string{"--ctx-size=8192", "-ngl", "99"}) {
|
|
t.Error(a)
|
|
}
|
|
if a := setArg([]string{"-ngl", "99"}, "--parallel", "4"); !slices.Equal(a, []string{"-ngl", "99", "--parallel", "4"}) {
|
|
t.Error(a)
|
|
}
|
|
if argInt([]string{"--parallel=3"}, "--parallel") != 3 || argInt(nil, "--parallel") != 0 {
|
|
t.Error("argInt")
|
|
}
|
|
p := catalog.Profile{Name: "x", Engine: "comfyui"}
|
|
if _, _, err := resizeProfile(p, ContextRequest{Tokens: 4096}, shape{}); err == nil {
|
|
t.Error("a comfyui profile took a context")
|
|
}
|
|
}
|