Running the planner against the real fleet produced twenty-six rows for a 32B, of which thirteen were duplicates: two fabrics of identical cards, and every single-node fit listed once for each. A replica on one node does not span anything, so which fabric its node happens to be attached to changes nothing about whether the model fits. A multi-node fit is still listed per fabric, because there the fabric is the constraint and which one has room is the thing the operator needs told. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
300e298053
commit
88a5949f7c
2 files changed
+70
No files matched your search
@@ -72,6 +72,7 @@ func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit {
|
||||
}
|
||||
}
|
||||
|
||||
out = dedupe(out)
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
if ga, gb := a.TP, b.TP; ga != gb {
|
||||
@@ -87,6 +88,39 @@ func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit {
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupe drops fits that differ only by a fabric that does not matter.
|
||||
//
|
||||
// A cluster with two fabrics of identical cards produced every single-node fit
|
||||
// twice, once per fabric, which is pure noise: a replica on one node does not
|
||||
// span anything, so which fabric its node is attached to changes nothing about
|
||||
// whether the model fits. Only a multi-node fit is fabric-specific, because
|
||||
// that is the case where the fabric is the constraint.
|
||||
//
|
||||
// Found by running it against a real fleet: a 32B produced twenty-six rows of
|
||||
// which thirteen were duplicates.
|
||||
func dedupe(in []Fit) []Fit {
|
||||
type key struct {
|
||||
precision string
|
||||
class string
|
||||
tp, nodes int
|
||||
fabric string
|
||||
}
|
||||
seen := map[key]struct{}{}
|
||||
out := in[:0]
|
||||
for _, f := range in {
|
||||
k := key{f.Precision.Name, f.Class, f.TP, f.Nodes, ""}
|
||||
if f.Nodes > 1 {
|
||||
k.fabric = f.Fabric
|
||||
}
|
||||
if _, dup := seen[k]; dup {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
|
||||
if h.Class == "" {
|
||||
return cpuFit(h, d, p, context)
|
||||
|
||||
@@ -148,3 +148,39 @@ func TestNothingFitsIsAnEmptyAnswerNotAWrongOne(t *testing.T) {
|
||||
t.Errorf("70B bf16 fitted a single 16 GB card: %s", fits[0].Describe())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdenticalHardwareOnTwoFabricsIsNotOfferedTwice(t *testing.T) {
|
||||
// A single-node replica does not span anything, so which fabric its node is
|
||||
// attached to changes nothing about whether the model fits. Listing it once
|
||||
// per fabric is noise: on a real fleet a 32B produced twenty-six rows of
|
||||
// which thirteen were duplicates.
|
||||
hw := []Hardware{h100Node(2, 4, "ib-a"), h100Node(2, 4, "ib-b")}
|
||||
fits := Fits(coder14B, hw, 8192, []Precision{BF16})
|
||||
|
||||
seen := map[int]int{}
|
||||
for _, f := range fits {
|
||||
if f.Nodes == 1 {
|
||||
seen[f.TP]++
|
||||
}
|
||||
}
|
||||
for tp, n := range seen {
|
||||
if n > 1 {
|
||||
t.Errorf("single-node tp=%d offered %d times", tp, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMultiNodeFitIsStillOfferedPerFabric(t *testing.T) {
|
||||
// The other half: a replica that spans nodes is fabric-specific, and which
|
||||
// fabric has room is exactly what the operator needs told.
|
||||
hw := []Hardware{h100Node(2, 2, "ib-a"), h100Node(2, 2, "ib-b")}
|
||||
fabrics := map[string]bool{}
|
||||
for _, f := range Fits(llama70B, hw, 8192, []Precision{Q4GGUF}) {
|
||||
if f.Nodes > 1 {
|
||||
fabrics[f.Fabric] = true
|
||||
}
|
||||
}
|
||||
if len(fabrics) < 2 {
|
||||
t.Errorf("multi-node fits were offered for %d fabrics, want both: %v", len(fabrics), fabrics)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user