diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index d34f2d7..9226d84 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -296,14 +296,28 @@ var quantBits = map[string]float64{ } // qualityCap is the precision past which a bigger file is not a better -// model worth its memory: eight bits keep what sixteen do, near enough. -const qualityCap = 8.6 +// model worth its memory: eight bits keep what sixteen do, near enough. On a +// CPU the cap is lower: there every bit is read from memory for every token, +// so a file twice the size answers at half the speed, and a Q4_K_M or Q5 is +// the usual balance. +const ( + qualityCap = 8.6 + cpuQualityCap = 5.8 +) +// quality scores an option's precision: rising to the cap, falling past it. func quality(o Option) float64 { if o.BitsPerWeight == 0 { return 0 } - return min(o.BitsPerWeight, qualityCap) + limit := qualityCap + if o.CPU { + limit = cpuQualityCap + } + if o.BitsPerWeight <= limit { + return o.BitsPerWeight + } + return limit - (o.BitsPerWeight-limit)*0.25 } // Rank orders options best first: what can start now; on a GPU before the diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go index 4551b1b..bed2e59 100644 --- a/internal/modelplan/options_test.go +++ b/internal/modelplan/options_test.go @@ -97,3 +97,18 @@ func TestQuantisationsAreReadFromNames(t *testing.T) { } } } + +func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) { + nodes := []PoolNode{{ID: "c", HostMemGB: 256, HostFreeGB: 200}} + arts := []Artifact{ + {Name: "q4", Format: "gguf", Bytes: 9 * gib}, {Name: "q5", Format: "gguf", Bytes: 10 * gib}, + {Name: "q8", Format: "gguf", Bytes: 15 * gib}, {Name: "f16", Format: "gguf", Bytes: 28 * gib}, + } + if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q5" { + t.Errorf("CPU: %s first (%s)", o[0].Artifact, strings.Join(labels(o), "; ")) + } + nodes = []PoolNode{{ID: "g", Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}} + if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q8" { + t.Errorf("GPU: %s first", o[0].Artifact) + } +}