On a CPU prefer a middling quantisation, on a card eight bits; past them a bigger file ranks lower

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:05:55 +02:00
1 parent 54db7aecb7
commit 0ea3008f64
2 files changed
+32 -3

No files matched your search

+17 -3
View File
@@ -296,14 +296,28 @@ var quantBits = map[string]float64{
}
// qualityCap is the precision past which a bigger file is not a better
// model worth its memory: eight bits keep what sixteen do, near enough.
const qualityCap = 8.6
// model worth its memory: eight bits keep what sixteen do, near enough. On a
// CPU the cap is lower: there every bit is read from memory for every token,
// so a file twice the size answers at half the speed, and a Q4_K_M or Q5 is
// the usual balance.
const (
qualityCap = 8.6
cpuQualityCap = 5.8
)
// quality scores an option's precision: rising to the cap, falling past it.
func quality(o Option) float64 {
if o.BitsPerWeight == 0 {
return 0
}
return min(o.BitsPerWeight, qualityCap)
limit := qualityCap
if o.CPU {
limit = cpuQualityCap
}
if o.BitsPerWeight <= limit {
return o.BitsPerWeight
}
return limit - (o.BitsPerWeight-limit)*0.25
}
// Rank orders options best first: what can start now; on a GPU before the
+15
View File
@@ -97,3 +97,18 @@ func TestQuantisationsAreReadFromNames(t *testing.T) {
}
}
}
func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) {
nodes := []PoolNode{{ID: "c", HostMemGB: 256, HostFreeGB: 200}}
arts := []Artifact{
{Name: "q4", Format: "gguf", Bytes: 9 * gib}, {Name: "q5", Format: "gguf", Bytes: 10 * gib},
{Name: "q8", Format: "gguf", Bytes: 15 * gib}, {Name: "f16", Format: "gguf", Bytes: 28 * gib},
}
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q5" {
t.Errorf("CPU: %s first (%s)", o[0].Artifact, strings.Join(labels(o), "; "))
}
nodes = []PoolNode{{ID: "g", Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}}
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q8" {
t.Errorf("GPU: %s first", o[0].Artifact)
}
}