The agent has always logged a percentage as it fetches weights, with a comment saying a replica pulling a hundred gigabytes and one that is wedged look identical from the control plane. They still did: the log is on the node, and the node is not where anyone is looking when a replica has been starting for ten minutes. Progress now rides on the shard status and appears under the replica in `ctl replicas` — a line rather than a column, because most replicas are not downloading anything and a column would be empty for all of them. It is deliberately not a state transition. Treating each chunk as one would re-mark the shard as starting on every report and reset the timer that decides when a slow start has become a failed one, so an hour-long download would keep a doomed replica alive indefinitely. It arrives on its own method for that reason, and a real transition clears it — otherwise a finished download sits in the listing at ninety-nine percent under a shard that has been serving for an hour. Watchers are keyed by source rather than by shard, because a fetch is single-flight: eight shards starting together produce one download and all eight are waiting on it. No percentage is invented when the server sent no length. One that stops moving is worse than a byte count that keeps rising, so the field is absent and the listing says "so far". Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
100f10ac17
commit
a35e8796d2
11 files changed
+447
-60
No files matched your search
@@ -131,6 +131,22 @@ type ShardView struct {
|
||||
GPUs []int `json:"gpus"`
|
||||
State string `json:"state"`
|
||||
Err string `json:"error,omitempty"`
|
||||
// Fetch is the download this shard is waiting on, absent when it is not
|
||||
// waiting on one. It is what tells a replica pulling a hundred gigabytes
|
||||
// apart from one that is wedged — which look identical without it.
|
||||
Fetch *FetchView `json:"fetch,omitempty"`
|
||||
}
|
||||
|
||||
// FetchView is a download in flight.
|
||||
type FetchView struct {
|
||||
Source string `json:"source"`
|
||||
BytesDone int64 `json:"bytes_done"`
|
||||
// BytesTotal is 0 when the server did not say how large the file is.
|
||||
BytesTotal int64 `json:"bytes_total,omitempty"`
|
||||
// Percent is absent rather than zero when the total is unknown: an
|
||||
// invented percentage that stops moving is worse than a byte count that
|
||||
// keeps rising.
|
||||
Percent *int `json:"percent,omitempty"`
|
||||
}
|
||||
|
||||
// ExplainView is why a model could not be placed.
|
||||
@@ -262,10 +278,19 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
v.IdleSec = int64(time.Since(rep.LastUsedAt).Seconds())
|
||||
}
|
||||
for _, s := range rep.Shards {
|
||||
v.Shards = append(v.Shards, ShardView{
|
||||
sv := ShardView{
|
||||
ID: s.ID, Node: s.NodeID, Rank: s.Rank,
|
||||
GPUs: s.GPUIndices, State: s.State.String(), Err: s.Err,
|
||||
})
|
||||
}
|
||||
if f := s.Fetch; f != nil {
|
||||
sv.Fetch = &FetchView{
|
||||
Source: f.Source, BytesDone: f.BytesDone, BytesTotal: f.BytesTotal,
|
||||
}
|
||||
if p, ok := f.Percent(); ok {
|
||||
sv.Fetch.Percent = &p
|
||||
}
|
||||
}
|
||||
v.Shards = append(v.Shards, sv)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,10 @@ import (
|
||||
// agents report they are already running. The reconciler implements it.
|
||||
type StatusSink interface {
|
||||
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
|
||||
// OnFetchProgress is separate because it is not a transition: the shard is
|
||||
// still starting, and treating each report as a state change would reset
|
||||
// the timers that decide when a slow start has become a failed one.
|
||||
OnFetchProgress(replicaID, shardID, source string, done, total int64)
|
||||
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
}
|
||||
|
||||
@@ -233,6 +237,14 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
s.applyHeartbeat(nodeID, m.Heartbeat)
|
||||
|
||||
case *llmgwv1.AgentMessage_Shard:
|
||||
// Progress rather than a transition. Falling through to OnShardStatus
|
||||
// would re-mark the shard as starting on every chunk, which is how a
|
||||
// download that takes an hour would keep a start timer at zero.
|
||||
if f := m.Shard.GetFetch(); f != nil {
|
||||
s.status.OnFetchProgress(m.Shard.GetReplicaId(), m.Shard.GetShardId(),
|
||||
f.GetSource(), f.GetBytesDone(), f.GetBytesTotal())
|
||||
return
|
||||
}
|
||||
st := StateFromWire(m.Shard.GetState())
|
||||
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
|
||||
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
func TestProgressIsVisibleWithoutBeingATransition(t *testing.T) {
|
||||
// The failure this replaces: a replica pulling a hundred gigabytes and one
|
||||
// that is wedged look identical from the control plane — both are a shard
|
||||
// that has been starting for ten minutes.
|
||||
//
|
||||
// And progress must not count as a state change. Treating each chunk as
|
||||
// one would reset the timer that decides when a slow start has become a
|
||||
// failed one, so a download would keep a replica alive forever.
|
||||
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 || len(reps[0].Shards) == 0 {
|
||||
t.Fatalf("expected a placed replica with shards, got %+v", reps)
|
||||
}
|
||||
rep, shard := reps[0].ID, reps[0].Shards[0].ID
|
||||
|
||||
h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 3<<30, 12<<30)
|
||||
|
||||
got := h.r.Replicas()[0].Shards[0]
|
||||
if got.Fetch == nil {
|
||||
t.Fatal("the download is not visible on the shard")
|
||||
}
|
||||
if got.Fetch.BytesDone != 3<<30 || got.Fetch.BytesTotal != 12<<30 {
|
||||
t.Errorf("progress came back as %+v", got.Fetch)
|
||||
}
|
||||
if p, ok := got.Fetch.Percent(); !ok || p != 25 {
|
||||
t.Errorf("percent = %d (known %v), want 25", p, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownTotalHasNoPercentage(t *testing.T) {
|
||||
// A server that sends no Content-Length is common. An invented percentage
|
||||
// that stops moving is worse than a byte count that keeps rising.
|
||||
f := FetchProgress{Source: "https://x/y.gguf", BytesDone: 500}
|
||||
if _, ok := f.Percent(); ok {
|
||||
t.Error("a percentage was invented from an unknown total")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATransitionClearsTheDownload(t *testing.T) {
|
||||
// Left in place it would sit in the listing at ninety-nine percent under
|
||||
// a shard that has been serving for an hour.
|
||||
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
reps := h.r.Replicas()
|
||||
rep, shard := reps[0].ID, reps[0].Shards[0].ID
|
||||
|
||||
h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 1, 2)
|
||||
h.r.OnShardStatus(rep, shard, supervisor.Ready, "")
|
||||
|
||||
if got := h.r.Replicas()[0].Shards[0]; got.Fetch != nil {
|
||||
t.Errorf("a finished shard still shows a download: %+v", got.Fetch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressForAReplicaThatIsGoneIsIgnored(t *testing.T) {
|
||||
// A download that finishes after its replica was torn down should not
|
||||
// resurrect anything or panic.
|
||||
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
||||
h.r.OnFetchProgress("no-such-replica", "no-such-shard", "x", 1, 2)
|
||||
}
|
||||
@@ -1235,6 +1235,29 @@ type AdoptedShard struct {
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// OnFetchProgress records how far a shard's download has got.
|
||||
//
|
||||
// Separate from OnShardStatus because it is not a transition: the shard is
|
||||
// still starting, and treating each progress report as a state change would
|
||||
// reset the timers that decide when a slow start becomes a failed one.
|
||||
func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
rep, ok := r.replicas[replicaID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := range rep.Shards {
|
||||
if rep.Shards[i].ID != shardID {
|
||||
continue
|
||||
}
|
||||
rep.Shards[i].Fetch = &FetchProgress{
|
||||
Source: source, BytesDone: done, BytesTotal: total,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnShardStatus records a status report from an agent.
|
||||
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
||||
r.markShard(replicaID, shardID, st, errMsg)
|
||||
@@ -1257,6 +1280,10 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
|
||||
if errMsg != "" {
|
||||
rep.Shards[i].Err = errMsg
|
||||
}
|
||||
// A real transition means the download is over, one way or the other.
|
||||
// Left in place it would sit in the listing at ninety-nine percent
|
||||
// under a shard that has been serving for an hour.
|
||||
rep.Shards[i].Fetch = nil
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,35 @@ type Shard struct {
|
||||
HTTPPort int
|
||||
State supervisor.State
|
||||
Err string
|
||||
|
||||
// Fetch is the download this shard is waiting on, if any. Cleared when the
|
||||
// shard reaches any other state, so a finished download does not sit in
|
||||
// the listing looking like one still running.
|
||||
Fetch *FetchProgress
|
||||
}
|
||||
|
||||
// FetchProgress is how far a shard's weights have got.
|
||||
//
|
||||
// Carried so an operator can tell a replica pulling a hundred gigabytes from
|
||||
// one that is wedged. From outside they are the same thing — a shard that has
|
||||
// been starting for ten minutes — and the difference is whether to wait or to
|
||||
// go and look.
|
||||
type FetchProgress struct {
|
||||
Source string
|
||||
BytesDone int64
|
||||
// BytesTotal is 0 when the server did not say. A percentage would then be
|
||||
// invented, and an invented one that stops moving is worse than a byte
|
||||
// count that keeps rising.
|
||||
BytesTotal int64
|
||||
}
|
||||
|
||||
// Percent returns progress as a percentage, and false when the total is
|
||||
// unknown.
|
||||
func (f FetchProgress) Percent() (int, bool) {
|
||||
if f.BytesTotal <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return int(f.BytesDone * 100 / f.BytesTotal), true
|
||||
}
|
||||
|
||||
// Leader reports whether this shard serves HTTP for the replica.
|
||||
|
||||
Reference in new issue
Block a user