A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement
llama.cpp's start-up output is read as a shard becomes ready: the devices it used, its backend, and the layers it offloaded. That goes up with READY in a new ShardStatus.compute, is kept on the shard, and is shown by the admin API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose engine says it found no usable GPU or offloaded no layer is stopped and failed, even where nvidia-smi cannot be asked. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f285aa0548
commit
0bd0d881c5
15 files changed
+517
-126
No files matched your search
@@ -518,6 +518,30 @@ type ShardView struct {
|
||||
// node's last heartbeat; zero when not measured.
|
||||
CPUCores float64 `json:"cpu_cores,omitempty"`
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
// Compute is where the engine said it put the model as it became
|
||||
// ready: absent when it did not say.
|
||||
Compute *ComputeView `json:"compute,omitempty"`
|
||||
}
|
||||
|
||||
// ComputeView is where an engine runs its model, by its own account.
|
||||
type ComputeView struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
|
||||
Backend string `json:"backend"`
|
||||
Devices []string `json:"devices,omitempty"`
|
||||
// LayersOffloaded of LayersTotal went to the devices; absent when unsaid.
|
||||
LayersOffloaded int `json:"layers_offloaded,omitempty"`
|
||||
LayersTotal int `json:"layers_total,omitempty"`
|
||||
}
|
||||
|
||||
// Short is the compute in a few characters: "CUDA 49/49", "CPU".
|
||||
func (c *ComputeView) Short() string {
|
||||
if c == nil {
|
||||
return "-"
|
||||
}
|
||||
if c.LayersTotal > 0 && c.Backend != "CPU" {
|
||||
return fmt.Sprintf("%s %d/%d", c.Backend, c.LayersOffloaded, c.LayersTotal)
|
||||
}
|
||||
return c.Backend
|
||||
}
|
||||
|
||||
// FetchView is a download in flight.
|
||||
@@ -829,6 +853,10 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
sv.Fetch.Percent = &p
|
||||
}
|
||||
}
|
||||
if c := s.Compute; c != nil {
|
||||
sv.Compute = &ComputeView{Backend: c.Backend, Devices: c.Devices,
|
||||
LayersOffloaded: c.Offloaded, LayersTotal: c.Total}
|
||||
}
|
||||
if l := loads[s.NodeID]; l != nil {
|
||||
if use, ok := l.Shards[s.ID]; ok {
|
||||
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
|
||||
|
||||
@@ -36,6 +36,8 @@ type StatusSink interface {
|
||||
// still starting, and treating each report as a state change would reset
|
||||
// the timers that decide when a slow start has become a failed one.
|
||||
OnFetchProgress(replicaID, shardID, source string, done, total int64)
|
||||
// OnShardCompute records where a shard's engine said it runs the model.
|
||||
OnShardCompute(replicaID, shardID string, c reconciler.Compute)
|
||||
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
}
|
||||
|
||||
@@ -449,6 +451,12 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
}
|
||||
st := StateFromWire(m.Shard.GetState())
|
||||
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
|
||||
if c := m.Shard.GetCompute(); c != nil {
|
||||
s.status.OnShardCompute(m.Shard.GetReplicaId(), m.Shard.GetShardId(), reconciler.Compute{
|
||||
Backend: c.GetBackend(), Devices: c.GetDevices(),
|
||||
Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal()),
|
||||
})
|
||||
}
|
||||
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
|
||||
s.log.Error("shard failed",
|
||||
"node", nodeID, "shard", m.Shard.GetShardId(),
|
||||
|
||||
@@ -1246,6 +1246,22 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
|
||||
}
|
||||
}
|
||||
|
||||
// OnShardCompute records where a shard's engine said it put the model.
|
||||
func (r *Reconciler) OnShardCompute(replicaID, shardID string, c Compute) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
rep, ok := r.replicas[replicaID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := range rep.Shards {
|
||||
if rep.Shards[i].ID == shardID {
|
||||
cc := c
|
||||
rep.Shards[i].Compute = &cc
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnShardStatus records a status report from an agent.
|
||||
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
||||
r.markShard(replicaID, shardID, st, errMsg)
|
||||
|
||||
@@ -72,6 +72,20 @@ type Shard struct {
|
||||
// shard reaches any other state, so a finished download does not sit in
|
||||
// the listing looking like one still running.
|
||||
Fetch *FetchProgress
|
||||
|
||||
// Compute is where the engine said it put the model as it became ready:
|
||||
// nil when it did not say.
|
||||
Compute *Compute
|
||||
}
|
||||
|
||||
// Compute is where an engine runs its model, by its own account.
|
||||
type Compute struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
|
||||
Backend string
|
||||
// Devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
Devices []string
|
||||
// Offloaded of Total layers went to the devices; 0 and 0 when unsaid.
|
||||
Offloaded, Total int
|
||||
}
|
||||
|
||||
// FetchProgress is how far a shard's weights have got.
|
||||
|
||||
Reference in new issue
Block a user