On a CPU prefer a middling quantisation, on a card eight bits; past them a bigger file ranks lower
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
54db7aecb7
commit
0ea3008f64
2 files changed
+32
-3
No files matched your search
@@ -97,3 +97,18 @@ func TestQuantisationsAreReadFromNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) {
|
||||
nodes := []PoolNode{{ID: "c", HostMemGB: 256, HostFreeGB: 200}}
|
||||
arts := []Artifact{
|
||||
{Name: "q4", Format: "gguf", Bytes: 9 * gib}, {Name: "q5", Format: "gguf", Bytes: 10 * gib},
|
||||
{Name: "q8", Format: "gguf", Bytes: 15 * gib}, {Name: "f16", Format: "gguf", Bytes: 28 * gib},
|
||||
}
|
||||
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q5" {
|
||||
t.Errorf("CPU: %s first (%s)", o[0].Artifact, strings.Join(labels(o), "; "))
|
||||
}
|
||||
nodes = []PoolNode{{ID: "g", Class: "a100-80g", GPUMemGB: 80, GPUs: 1, FreeGPUs: 1}}
|
||||
if o := Options(arts, shape14B, nodes, 16384); o[0].Artifact != "q8" {
|
||||
t.Errorf("GPU: %s first", o[0].Artifact)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user