Let the mirror's serve form place a model on a GPU
The form offered no placement and named every GGUF profile cpu. It now lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU count and parallel requests, names the profile from the placement, shows what the model will run as, and refuses the CPU while GPUs are free unless that is ticked. Refs #2 (item 1). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbd5521f4f
commit
cba3e329fc
5 files changed
+364
-11
No files matched your search
@@ -317,6 +317,16 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The CPU while GPUs sit free is a choice to make on purpose:
|
||||
// made by default, it was a model running at a tenth of the
|
||||
// speed with nothing saying why.
|
||||
if model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" {
|
||||
if n := freeGPUs(p.fleet); n > 0 {
|
||||
return "", fmt.Errorf("this fleet has %s free and %s was set to run on the CPU; "+
|
||||
"choose a GPU placement, or tick \"run on the CPU anyway\"",
|
||||
plural(n, "GPU", "GPUs"), model.Name)
|
||||
}
|
||||
}
|
||||
// The same check `llmgw ctl model add` makes. This page builds
|
||||
// the source out of a row the mirror reported, so it is right
|
||||
// far more often than something typed — but a file removed
|
||||
@@ -332,8 +342,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
// Placed on the next pass, not now: saying it is serving would
|
||||
// be wrong for however long the weights take to reach a node.
|
||||
return fmt.Sprintf(
|
||||
"%s added to the catalog, %s; it is placed on the next pass",
|
||||
model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil
|
||||
"%s added to the catalog, %s: %s; it is placed on the next pass",
|
||||
model.Name, plural(model.Scaling.Min, "replica", "replicas"), describeServe(model)), nil
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user