diff --git a/CHANGELOG.md b/CHANGELOG.md index 850e6a1..32e99ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,15 @@ APIs change between minors without a deprecation period. ### Fixed +- **The serve forms start at the longest context that fits.** The mirror + page's form began every GGUF model at 4096 tokens, and the console's at + 4096 or the plan's figure, whatever the model was trained for. Both now + start at what it was trained for, as long as its KV cache fits where the + form places it beside the weights, else at the most that does, in whole + thousands; the mirror page's llama.cpp slots are as many of that, up to + four, as the room left holds, and its memory figure is for that pool. A + plan says what the model was trained for (`trained`). Strata stays at + its 32768; vLLM's form is unchanged. - **`ctl weights` offered models in use for deletion.** A model of several files (a set, its address ending in `/`) is fetched into a directory named from the address without the slash; the sweep named it from the address diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 6c20685..cc16492 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -11745,6 +11745,10 @@ "context": { "type": "integer" }, + "trained": { + "type": "integer", + "description": "Trained is the context the model was trained for, the most a request may use; 0 when neither its header nor its config says." + }, "kv_bytes_per_token": { "type": "integer", "format": "int64", diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 57312bf..ab9ba21 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -213,10 +213,17 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact "before it is served.", modelplan.GB(p.KVBytesPerToken*1000)) } note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello." + // The longest context the first way holds, up to what the model was + // trained for: a choice of another way is checked again on submitting. ctxDefault := "4096" if a.Format == "safetensors" { ctxDefault = strconv.Itoa(p.Context) } + if len(opts) > 0 { + if most := opts[0].MostContext(p.KVBytesPerToken, p.Trained); most > 0 { + ctxDefault = strconv.Itoa(most) + } + } if len(opts) > 0 && opts[0].Engine == "strata" { ctxDefault = "32768" } @@ -249,6 +256,11 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact Description: v[3], Format: a.Format} if a.Format == "gguf" { req.File = a.Name + // A context longer than the default pool is the pool: one + // slot of it, where the server would refuse the request. + if ctxLen > modelserve.DefaultCtxPool { + req.CtxPool = ctxLen + } } o := opts[slices.Index(where, v[4])] // Asked for a context other than planned: does it still fit? diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 074e62f..79a57a3 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -296,7 +296,7 @@ func (s *Server) Start(req Request) (Job, error) { } req.sized = req.CtxPool > 0 || req.ContextLength > 0 if req.CtxPool <= 0 { - req.CtxPool = 16384 + req.CtxPool = DefaultCtxPool } if req.ContextLength <= 0 { req.ContextLength = 4096 @@ -731,6 +731,10 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat return out } +// DefaultCtxPool is llama.cpp's pool when a request names none: a context +// asked for longer than it needs a pool of its own. +const DefaultCtxPool = 16384 + // llamaArgs size llama.cpp's pool and its slots together. The pool is shared // by the slots, so slots of the context the catalog promises are as many as // the pool holds: the engine's default of four turned a 32768 pool into diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index a81d797..fa2c11a 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -43,6 +43,9 @@ type Plan struct { License string `json:"license,omitempty"` Params int64 `json:"params,omitempty"` Context int `json:"context"` + // Trained is the context the model was trained for, the most a request + // may use; 0 when neither its header nor its config says. + Trained int `json:"trained,omitempty"` // KVBytesPerToken is one token of context, the whole model: what an // option needs for another context is worked out from it. 0 is unknown. KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"` @@ -149,6 +152,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, p.Servable, p.Reason = false, v.Reason } shape.KVBytesPerToken = v.Header.KVBytesPerToken() + p.Trained = int(v.Header.ContextLength) //nolint:gosec // a header's context length shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count shape.ActiveShare = v.Header.ActiveShare() if p.Arch == "" { @@ -167,6 +171,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, m, err := modelplan.FetchModel(ctx, repo.ID) if err == nil { shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) + p.Trained = m.MaxContext shape.KVHeads = m.KVHeads shape.Experts = m.Experts if p.Params == 0 { diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index 2f583f1..db03b30 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -173,6 +173,29 @@ func (o Option) WithContext(kvPerToken int64, tokens int) Option { return o } +// MostContext is the longest context, up to trained, this option still holds +// for one request, in whole thousands of tokens below trained: trained when +// the KV cache's size is unknown, 0 when trained is, for Strata (its context +// is its own), or when not even 1024 tokens fit. +func (o Option) MostContext(kvPerToken int64, trained int) int { + switch { + case trained <= 0 || o.Engine == "strata": + return 0 + case kvPerToken <= 0 || o.WithContext(kvPerToken, trained).Fits(): + return trained + } + lo, hi := 0, trained/1024+1 // lo fits (or is none), hi does not + for hi-lo > 1 { + mid := (lo + hi) / 2 + if o.WithContext(kvPerToken, mid*1024).Fits() { + lo = mid + } else { + hi = mid + } + } + return lo * 1024 +} + // MemoryText says what it needs, part by part. func (o Option) MemoryText() string { if o.Engine == "strata" { diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go index 768d594..c02aa17 100644 --- a/internal/modelplan/options_test.go +++ b/internal/modelplan/options_test.go @@ -193,3 +193,23 @@ func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) { t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText()) } } + +func TestTheMostContextAnOptionHolds(t *testing.T) { + // 4 GB of weights on a 24 GB card, a token's cache 1 MiB. + o := Option{Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1, + WeightsGB: 4, EngineGB: 1, NeedGB: 5, HaveGB: 21.6} + const kv = 1 << 20 + if got := o.MostContext(kv, 8192); got != 8192 { + t.Errorf("a context that fits whole: %d, want what it was trained for", got) + } + got := o.MostContext(kv, 131072) + if got <= 0 || got%1024 != 0 || !o.WithContext(kv, got).Fits() || o.WithContext(kv, got+1024).Fits() { + t.Errorf("the most that fits is %d", got) + } + if got := o.MostContext(0, 32768); got != 32768 { + t.Errorf("an unknown cache size: %d, want what it was trained for", got) + } + if got := (Option{Engine: "strata"}).MostContext(kv, 32768); got != 0 { + t.Errorf("Strata's context is its own: %d", got) + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 4f4102b..6fc30eb 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -224,7 +224,10 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string, if weights <= 0 { weights = st.BytesDone } - if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 { + if c, par := mostContext(header, weights, f.Placements, f.GPUs, fleet); c > 0 { + f.Context, f.Parallel, f.CtxPool = c, par, c*par + } + if b := header.MemoryBytes(weights, f.CtxPool); b > 0 { kvHeads := header.HeadCountKV if kvHeads == 0 { kvHeads = header.HeadCount @@ -235,8 +238,8 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string, // flat "half again" could not see. f.MinMemoryGB = int((b + 1<<30 - 1) >> 30) f.MemoryBasis = l.T("serve.basis.shape", - l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30), 1), - l.Int(defaultCtxPool), header.BlockCount, kvHeads, + l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(f.CtxPool)*header.KVBytesPerToken())/(1<<30), 1), + l.Int(int64(f.CtxPool)), header.BlockCount, kvHeads, l.Num(float64(int64(1<<30)+weights/20)/(1<<30), 1)) } if fleet != nil { @@ -552,6 +555,46 @@ func modeOf(sc catalog.Scaling) string { // --parallel divides it among slots; it does not multiply it. const defaultCtxPool = 16384 +// mostContext is the longest context the serve form starts with: what the +// model was trained for, as long as its KV cache fits where the form places +// it beside the weights, else the most that does; and as many slots of it, +// up to defaultParallel, as the room left holds. 0 when the header does not +// say what it was trained for, or the weights alone do not fit. +// +// The longest, because a context is the one thing about a served model a +// client runs into without being told: a 4096 default on a model trained for +// 40960 cut documents short that the model could have read whole. +func mostContext(h gguf.Header, weights int64, places []placeChoice, gpus int, fleet Source) (tokens, slots int) { + trained := int(h.ContextLength) //nolint:gosec // a header's context length + if trained <= 0 { + return 0, 0 + } + pool := 0 + for _, pc := range places { + if !pc.Selected { + continue + } + if class, ok := strings.CutPrefix(pc.Value, "gpu:"); ok { + pool = h.PoolThatFits(weights, gguf.ClassMemGB(class), gpus) + } else { + pool = h.PoolThatFits(weights, biggestNode(fleet), 1) + } + } + switch { + case pool < 0: + return 0, 0 + case pool == 0: + // Nothing to measure it against: what it was trained for, as one slot + // would hold it; the checks on submitting still see the card. + return trained, 1 + } + tokens = min(trained, pool/1024*1024) + if tokens < 1024 { + return 0, 0 + } + return tokens, max(1, min(defaultParallel, pool/tokens)) +} + // defaultParallel is the requests a llama.cpp replica serves at once when // the form is not changed: the agent's own default. const defaultParallel = 4 diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index b9f1519..b0f5bda 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -670,3 +670,31 @@ func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) { t.Fatal("ticking the box should add it") } } + +func TestTheServeFormStartsAtTheLongestContextThatFits(t *testing.T) { + // An 8B's shape: 36 layers, 8 KV heads of 128, so 144 KiB a token. + h := gguf.Header{BlockCount: 36, HeadCount: 32, HeadCountKV: 8, KeyLength: 128, ValueLength: 128, + ContextLength: 40960} + card := []placeChoice{{Value: "gpu:6000-24g", Selected: true}, {Value: "cpu"}} + cpu := []placeChoice{{Value: "gpu:6000-24g"}, {Value: "cpu", Selected: true}} + for _, c := range []struct { + name string + weights int64 + places []placeChoice + header gguf.Header + tokens, slots int + }{ + // All it was trained for, and two slots of it in the room left. + {"on a card with room", 5 << 30, card, h, 40960, 2}, + // The weights take most of the card: the most that fits, one slot. + {"on a card nearly full", 18 << 30, card, h, 18432, 1}, + {"on the CPU of a 64 GB node", 5 << 30, cpu, h, 40960, 4}, + {"weights larger than the card", 30 << 30, card, h, 0, 0}, + {"a header that does not say", 5 << 30, card, gguf.Header{BlockCount: 36, HeadCountKV: 8, HeadCount: 32, KeyLength: 128}, 0, 0}, + } { + tokens, slots := mostContext(c.header, c.weights, c.places, 1, fakeFleet{memoryGB: 64}) + if tokens != c.tokens || slots != c.slots { + t.Errorf("%s: %d tokens × %d slots, want %d × %d", c.name, tokens, slots, c.tokens, c.slots) + } + } +}