package catalog import ( "strings" "testing" ) func TestAProfileOnAMIGClassRunsOnExactlyOneInstance(t *testing.T) { // One instance, by default or said outright, is a profile like any other. for _, p := range []Profile{ {Name: "vllm", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb"}, {Name: "llama", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 1, TP: 1}, } { c, err := New([]Model{model("m", p)}) if err != nil { t.Fatalf("%s on one MIG instance: %v", p.Name, err) } m, _ := c.Resolve("m") if pl := m.PlacementProfiles()[0]; pl.GPUsPerNode != 1 || pl.Nodes != 1 { t.Errorf("%s: placement asked for %+v", p.Name, pl) } } // More than one is refused where the model is defined: a process opens // one MIG device, and instances have no NVLink between them, so the // replica would be placed and then fail at start. for _, p := range []Profile{ {Name: "tp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2}, {Name: "pp2", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 1, PP: 2, GPUsPerNode: 2}, {Name: "nodes", Engine: "vllm", GPUClass: "a100-80g-mig-3g.40gb", TP: 2, Nodes: 2, GPUsPerNode: 1}, {Name: "layers", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", GPUsPerNode: 2}, {Name: "old-tp", Engine: "llamacpp", GPUClass: "a100-80g-mig-3g.40gb", TP: 2}, } { _, err := New([]Model{model("m", p)}) if err == nil || !strings.Contains(err.Error(), "MIG instance") { t.Errorf("%s: %v", p.Name, err) } } }