A model that does not fit its card is not added to fail; weights in use are not removed
CI / test (push) Failing after 30m3s
CI / test (push) Failing after 30m3s
Apertus 70B with a 65536-token pool, and then 8B with 131072, on a 24 GB card loaded their weights and failed to make their context on every start, the 8B taking the card from the 70B each time only to fail. ctl model serve now shrinks a pool the card cannot hold beside the weights (KV cache from the GGUF's shape, 90% of the card, a gigabyte for the engine), saying so, and refuses weights that do not fit at all; the portal's serve form refuses such a pool and says the largest that fits. The mirror page removed the 70B's weights while the catalog still had the model, whose replicas went on loading the node's copy with nothing said. Removing weights a catalog model loads is refused, naming the model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bda988b78f
commit
c22528c26e
9 files changed
+220
-1
No files matched your search
@@ -490,6 +490,20 @@ func (s *Server) run(j *Job) {
|
||||
// third busy.
|
||||
req.ContextLength, req.CtxPool = embedSlotTokens, embedSlotTokens*embedSlots
|
||||
}
|
||||
// On one card, the weights and the context's cache have to fit it: a
|
||||
// 70B with a 65536-token pool on a 24 GB card loaded its weights and then
|
||||
// failed to make its context, every time, as did an 8B with 131072.
|
||||
if len(layout) > 0 && layout[0].Class != "" && !layout[0].CPU {
|
||||
fitted, why, err := fitPoolToCard(req, header, set.Bytes, layout[0].Class, max(layout[0].GPUsPerNode, 1))
|
||||
if err != nil {
|
||||
s.fail(j, "check", err)
|
||||
return
|
||||
}
|
||||
if why != "" {
|
||||
s.step(j, "check", Running, why)
|
||||
req = fitted
|
||||
}
|
||||
}
|
||||
j.Request = req
|
||||
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
|
||||
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
|
||||
@@ -1071,3 +1085,30 @@ func (s *Server) embeds(name string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fitPoolToCard shrinks a context pool the cards cannot hold to one they
|
||||
// can, saying so, and refuses weights that do not fit at all. The slots stay
|
||||
// as many as asked while each keeps 2048 tokens, then fewer.
|
||||
func fitPoolToCard(req Request, h gguf.Header, weights int64, class string, gpus int) (Request, string, error) {
|
||||
maxTok := h.PoolThatFits(weights, gguf.ClassMemGB(class), gpus)
|
||||
switch {
|
||||
case maxTok == 0 || req.CtxPool <= maxTok:
|
||||
return req, "", nil
|
||||
case maxTok < 0:
|
||||
return req, "", fmt.Errorf("its weights (%.1f GB) do not fit %d card(s) of %s with room to run: "+
|
||||
"choose a smaller quantisation, or more cards", gb(weights), gpus, class)
|
||||
}
|
||||
slots := max(1, req.CtxPool/max(req.ContextLength, 1))
|
||||
for slots > 1 && maxTok/slots < 2048 {
|
||||
slots--
|
||||
}
|
||||
per := (maxTok / slots) / 1024 * 1024
|
||||
if per < 1024 {
|
||||
return req, "", fmt.Errorf("its weights (%.1f GB) leave room on %s for a context of %d tokens only",
|
||||
gb(weights), class, maxTok)
|
||||
}
|
||||
was := req.CtxPool
|
||||
req.ContextLength, req.CtxPool = per, per*slots
|
||||
return req, fmt.Sprintf("a %d-token pool does not fit %s beside %.1f GB of weights: %d slots of %d tokens instead",
|
||||
was, class, gb(weights), slots, per), nil
|
||||
}
|
||||
Reference in new issue
Block a user