A model that does not fit its card is not added to fail; weights in use are not removed
Apertus 70B with a 65536-token pool, and then 8B with 131072, on a 24 GB card loaded their weights and failed to make their context on every start, the 8B taking the card from the 70B each time only to fail. ctl model serve now shrinks a pool the card cannot hold beside the weights (KV cache from the GGUF's shape, 90% of the card, a gigabyte for the engine), saying so, and refuses weights that do not fit at all; the portal's serve form refuses such a pool and says the largest that fits. The mirror page removed the 70B's weights while the catalog still had the model, whose replicas went on loading the node's copy with nothing said. Removing weights a catalog model loads is refused, naming the model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bda988b78f
commit
c22528c26e
9 files changed
+220
-1
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -342,7 +343,14 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
// between the listing and the click, or a mirror that has gone
|
||||
// down since, both end as a replica failing on a node minutes
|
||||
// later with an error naming the node. Asking costs one HEAD.
|
||||
if _, err := sourcecheck.Check(ctx, model.Source); err != nil {
|
||||
size, err := sourcecheck.Check(ctx, model.Source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The weights and the context's cache have to fit the card:
|
||||
// an 8B asked for a 131072-token pool on 24 GB loaded its
|
||||
// weights and failed to make its context, again and again.
|
||||
if err := p.fitsCard(ctx, l, model, size); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := d.Catalog.Add(ctx, model, admin.Email); err != nil {
|
||||
@@ -365,6 +373,14 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
|
||||
func(ctx context.Context, _ store.User, r *http.Request) (string, error) {
|
||||
model, ver := r.PathValue("model"), r.PathValue("version")
|
||||
// Weights a catalog model still loads are not taken from under
|
||||
// it: removed from the mirror alone, the model stayed in the
|
||||
// catalog and its replicas went on loading the copy on the node.
|
||||
if users := p.mirrorUsers(ctx, model, ver); len(users) > 0 {
|
||||
return "", fmt.Errorf("%s/%s is the weights of %s in the catalog: remove %s first "+
|
||||
"(the models page, or llmgw ctl model remove %s), then the weights",
|
||||
model, ver, strings.Join(users, ", "), strings.Join(users, ", "), users[0])
|
||||
}
|
||||
removed, err := m.Remove(ctx, model, ver)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -937,3 +953,52 @@ func sharedPrefix(ids []string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mirrorUsers are the catalog models whose weights are the mirror's
|
||||
// model/version.
|
||||
func (p *AccountPages) mirrorUsers(ctx context.Context, model, ver string) []string {
|
||||
if p.catalog == nil {
|
||||
return nil
|
||||
}
|
||||
srcs, err := p.catalog.Sources(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
part := "/" + model + "/" + ver + "/"
|
||||
var out []string
|
||||
for src, ms := range srcs {
|
||||
if !strings.Contains(src, part) {
|
||||
continue
|
||||
}
|
||||
for _, m := range ms {
|
||||
if !slices.Contains(out, m.Name) {
|
||||
out = append(out, m.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// fitsCard refuses a model whose weights and context pool do not fit the
|
||||
// cards its profile names, saying the largest pool that would.
|
||||
func (p *AccountPages) fitsCard(ctx context.Context, l *Lang, m catalog.Model, size int64) error {
|
||||
if len(m.Profiles) == 0 || m.Profiles[0].GPUClass == "" || size <= 0 {
|
||||
return nil
|
||||
}
|
||||
pr := m.Profiles[0]
|
||||
v := p.checkSource(ctx, m.Source)
|
||||
cards := max(pr.GPUsPerNode, pr.TP, 1)
|
||||
most := v.Header.PoolThatFits(size, gguf.ClassMemGB(pr.GPUClass), cards)
|
||||
pool := argInt(pr.Args, "--ctx-size")
|
||||
if pool <= 0 {
|
||||
pool = defaultCtxPool
|
||||
}
|
||||
switch {
|
||||
case most < 0:
|
||||
return errors.New(l.T("serve.err.weights_too_big", l.Num(float64(size)/(1<<30), 1), pr.GPUClass))
|
||||
case most > 0 && pool > most:
|
||||
return errors.New(l.T("serve.err.pool_too_big", l.Int(int64(pool)), pr.GPUClass, l.Int(int64(most/1024*1024))))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user