A model that does not fit its card is not added to fail; weights in use are not removed
CI / test (push) Failing after 30m3s

Apertus 70B with a 65536-token pool, and then 8B with 131072, on a 24 GB
card loaded their weights and failed to make their context on every start,
the 8B taking the card from the 70B each time only to fail. ctl model serve
now shrinks a pool the card cannot hold beside the weights (KV cache from
the GGUF's shape, 90% of the card, a gigabyte for the engine), saying so,
and refuses weights that do not fit at all; the portal's serve form refuses
such a pool and says the largest that fits.

The mirror page removed the 70B's weights while the catalog still had the
model, whose replicas went on loading the node's copy with nothing said.
Removing weights a catalog model loads is refused, naming the model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 00:22:01 +02:00
1 parent bda988b78f
commit c22528c26e
9 files changed
+220 -1

No files matched your search

+41
View File
@@ -490,6 +490,20 @@ func (s *Server) run(j *Job) {
// third busy.
req.ContextLength, req.CtxPool = embedSlotTokens, embedSlotTokens*embedSlots
}
// On one card, the weights and the context's cache have to fit it: a
// 70B with a 65536-token pool on a 24 GB card loaded its weights and then
// failed to make its context, every time, as did an 8B with 131072.
if len(layout) > 0 && layout[0].Class != "" && !layout[0].CPU {
fitted, why, err := fitPoolToCard(req, header, set.Bytes, layout[0].Class, max(layout[0].GPUsPerNode, 1))
if err != nil {
s.fail(j, "check", err)
return
}
if why != "" {
s.step(j, "check", Running, why)
req = fitted
}
}
j.Request = req
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
@@ -1071,3 +1085,30 @@ func (s *Server) embeds(name string) bool {
}
return false
}
// fitPoolToCard shrinks a context pool the cards cannot hold to one they
// can, saying so, and refuses weights that do not fit at all. The slots stay
// as many as asked while each keeps 2048 tokens, then fewer.
func fitPoolToCard(req Request, h gguf.Header, weights int64, class string, gpus int) (Request, string, error) {
maxTok := h.PoolThatFits(weights, gguf.ClassMemGB(class), gpus)
switch {
case maxTok == 0 || req.CtxPool <= maxTok:
return req, "", nil
case maxTok < 0:
return req, "", fmt.Errorf("its weights (%.1f GB) do not fit %d card(s) of %s with room to run: "+
"choose a smaller quantisation, or more cards", gb(weights), gpus, class)
}
slots := max(1, req.CtxPool/max(req.ContextLength, 1))
for slots > 1 && maxTok/slots < 2048 {
slots--
}
per := (maxTok / slots) / 1024 * 1024
if per < 1024 {
return req, "", fmt.Errorf("its weights (%.1f GB) leave room on %s for a context of %d tokens only",
gb(weights), class, maxTok)
}
was := req.CtxPool
req.ContextLength, req.CtxPool = per, per*slots
return req, fmt.Sprintf("a %d-token pool does not fit %s beside %.1f GB of weights: %d slots of %d tokens instead",
was, class, gb(weights), slots, per), nil
}
@@ -5,6 +5,7 @@ import (
"context"
"encoding/binary"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"net/http"
"net/http/httptest"
"slices"
@@ -412,3 +413,18 @@ func TestAnEmbeddingModelIsServedForEmbedding(t *testing.T) {
t.Errorf("a chat model got %q", args)
}
}
func TestAPoolTheCardCannotHoldIsShrunkNotStarted(t *testing.T) {
h := gguf.Header{Architecture: "apertus", BlockCount: 32, HeadCount: 32, HeadCountKV: 8, EmbeddingLength: 4096}
req := Request{CtxPool: 131072, ContextLength: 32768}
got, why, err := fitPoolToCard(req, h, int64(8.5*(1<<30)), "6000-24g", 1)
if err != nil || why == "" || got.CtxPool > 105000 || got.CtxPool/got.ContextLength != 4 {
t.Errorf("fitted %+v %q %v", got, why, err)
}
if _, _, err := fitPoolToCard(req, h, int64(30*(1<<30)), "6000-24g", 1); err == nil {
t.Error("weights larger than the card were not refused")
}
if got, why, _ := fitPoolToCard(Request{CtxPool: 16384, ContextLength: 4096}, h, int64(8.5*(1<<30)), "6000-24g", 1); why != "" || got.CtxPool != 16384 {
t.Errorf("a pool that fits was changed: %+v %q", got, why)
}
}