Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a45068461
commit
84444e71c8
20 files changed
+388
-7
No files matched your search
@@ -1206,9 +1206,12 @@ type Host struct {
|
||||
CpuFlags []string `protobuf:"bytes,5,rep,name=cpu_flags,json=cpuFlags,proto3" json:"cpu_flags,omitempty"`
|
||||
// nvidia_driver is the driver version, empty when there is none. A card
|
||||
// without a driver is not a GPU anything can use.
|
||||
NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"`
|
||||
// mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when
|
||||
// the agent started: what decides a model's speed on the CPU. 0 unknown.
|
||||
MemBandwidthGbs float64 `protobuf:"fixed64,7,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Host) Reset() {
|
||||
@@ -1283,6 +1286,13 @@ func (x *Host) GetNvidiaDriver() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Host) GetMemBandwidthGbs() float64 {
|
||||
if x != nil {
|
||||
return x.MemBandwidthGbs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Storage is one filesystem a node keeps weights on.
|
||||
//
|
||||
// Per filesystem rather than per configured directory: free space belongs to
|
||||
@@ -5190,7 +5200,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb1\x01\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xdd\x01\n" +
|
||||
"\x04Host\x12\x13\n" +
|
||||
"\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -5199,7 +5209,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"osCodename\x12\x12\n" +
|
||||
"\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" +
|
||||
"\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" +
|
||||
"\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\"u\n" +
|
||||
"\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" +
|
||||
"\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\"u\n" +
|
||||
"\aStorage\x12\x12\n" +
|
||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
|
||||
"\n" +
|
||||
|
||||
@@ -238,6 +238,9 @@ message Host {
|
||||
// nvidia_driver is the driver version, empty when there is none. A card
|
||||
// without a driver is not a GPU anything can use.
|
||||
string nvidia_driver = 6;
|
||||
// mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when
|
||||
// the agent started: what decides a model's speed on the CPU. 0 unknown.
|
||||
double mem_bandwidth_gbs = 7;
|
||||
}
|
||||
|
||||
// Storage is one filesystem a node keeps weights on.
|
||||
|
||||
Reference in new issue
Block a user