On a CPU prefer a middling quantisation, on a card eight bits; past them a bigger file ranks lower

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:05:55 +02:00
1 parent 54db7aecb7
commit 0ea3008f64
2 files changed
+32 -3

No files matched your search

+15
View File
@@ -97,3 +97,18 @@ func TestQuantisationsAreReadFromNames(t *testing.T) {
}
}
}
func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) {
nodes := []PoolNode{{ID: "c", HostMemGB: 256, HostFreeGB: 200}}
arts := []Artifact{
{Name: "q4", Format: "gguf", Bytes: 9 * gib}, {Name: "q5", Format: "gguf", Bytes: 10 * gib},
{Name: "q8", Format: "gguf", Bytes: 15 * gib}, {Name: "f16", Format: "gguf", Bytes: 28 * gib},
}
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q5" {
t.Errorf("CPU: %s first (%s)", o[0].Artifact, strings.Join(labels(o), "; "))
}
nodes = []PoolNode{{ID: "g", Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}}
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q8" {
t.Errorf("GPU: %s first", o[0].Artifact)
}
}