On a CPU prefer a middling quantisation, on a card eight bits; past them a bigger file ranks lower
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
54db7aecb7
commit
0ea3008f64
2 files changed
+32
-3
No files matched your search
@@ -296,14 +296,28 @@ var quantBits = map[string]float64{
|
||||
}
|
||||
|
||||
// qualityCap is the precision past which a bigger file is not a better
|
||||
// model worth its memory: eight bits keep what sixteen do, near enough.
|
||||
const qualityCap = 8.6
|
||||
// model worth its memory: eight bits keep what sixteen do, near enough. On a
|
||||
// CPU the cap is lower: there every bit is read from memory for every token,
|
||||
// so a file twice the size answers at half the speed, and a Q4_K_M or Q5 is
|
||||
// the usual balance.
|
||||
const (
|
||||
qualityCap = 8.6
|
||||
cpuQualityCap = 5.8
|
||||
)
|
||||
|
||||
// quality scores an option's precision: rising to the cap, falling past it.
|
||||
func quality(o Option) float64 {
|
||||
if o.BitsPerWeight == 0 {
|
||||
return 0
|
||||
}
|
||||
return min(o.BitsPerWeight, qualityCap)
|
||||
limit := qualityCap
|
||||
if o.CPU {
|
||||
limit = cpuQualityCap
|
||||
}
|
||||
if o.BitsPerWeight <= limit {
|
||||
return o.BitsPerWeight
|
||||
}
|
||||
return limit - (o.BitsPerWeight-limit)*0.25
|
||||
}
|
||||
|
||||
// Rank orders options best first: what can start now; on a GPU before the
|
||||
|
||||
@@ -97,3 +97,18 @@ func TestQuantisationsAreReadFromNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) {
|
||||
nodes := []PoolNode{{ID: "c", HostMemGB: 256, HostFreeGB: 200}}
|
||||
arts := []Artifact{
|
||||
{Name: "q4", Format: "gguf", Bytes: 9 * gib}, {Name: "q5", Format: "gguf", Bytes: 10 * gib},
|
||||
{Name: "q8", Format: "gguf", Bytes: 15 * gib}, {Name: "f16", Format: "gguf", Bytes: 28 * gib},
|
||||
}
|
||||
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q5" {
|
||||
t.Errorf("CPU: %s first (%s)", o[0].Artifact, strings.Join(labels(o), "; "))
|
||||
}
|
||||
nodes = []PoolNode{{ID: "g", Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}}
|
||||
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q8" {
|
||||
t.Errorf("GPU: %s first", o[0].Artifact)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user