The agent has always logged a percentage as it fetches weights, with a comment saying a replica pulling a hundred gigabytes and one that is wedged look identical from the control plane. They still did: the log is on the node, and the node is not where anyone is looking when a replica has been starting for ten minutes. Progress now rides on the shard status and appears under the replica in `ctl replicas` — a line rather than a column, because most replicas are not downloading anything and a column would be empty for all of them. It is deliberately not a state transition. Treating each chunk as one would re-mark the shard as starting on every report and reset the timer that decides when a slow start has become a failed one, so an hour-long download would keep a doomed replica alive indefinitely. It arrives on its own method for that reason, and a real transition clears it — otherwise a finished download sits in the listing at ninety-nine percent under a shard that has been serving for an hour. Watchers are keyed by source rather than by shard, because a fetch is single-flight: eight shards starting together produce one download and all eight are waiting on it. No percentage is invented when the server sent no length. One that stops moving is worse than a byte count that keeps rising, so the field is absent and the listing says "so far". Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
100f10ac17
commit
a35e8796d2
11 files changed
+447
-60
No files matched your search
+75
-9
@@ -15,6 +15,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -219,6 +220,12 @@ type Agent struct {
|
||||
// heartbeat carries the list only when it changed. Touched only by the
|
||||
// heartbeat goroutine, which is the only thing that reads or writes it.
|
||||
lastWeights string
|
||||
|
||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||
// download reaches every shard that asked for it. A fetch is single-flight
|
||||
// — eight shards starting together produce one download — and all eight
|
||||
// are waiting, so all eight should say so.
|
||||
fetchWatchers map[string][]func(weights.Progress)
|
||||
}
|
||||
|
||||
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||
@@ -456,12 +463,20 @@ func (a *Agent) weights() *weights.Fetcher {
|
||||
CacheDir: cache,
|
||||
Log: a.log,
|
||||
OnProgress: func(p weights.Progress) {
|
||||
// Logged rather than silent: a replica that is downloading a
|
||||
// hundred gigabytes and one that is wedged look identical from the
|
||||
// control plane, and the difference matters to whoever is waiting.
|
||||
a.log.Info("fetching weights",
|
||||
"source", p.Source, "percent", p.Percent(),
|
||||
"bytes", p.BytesDone, "total", p.BytesTotal)
|
||||
// And upward, to whoever is watching from the control plane. The
|
||||
// log line above is on the node, which is where nobody is looking
|
||||
// when a replica has been starting for ten minutes.
|
||||
a.mu.Lock()
|
||||
watchers := slices.Clone(a.fetchWatchers[p.Source])
|
||||
a.mu.Unlock()
|
||||
for _, w := range watchers {
|
||||
if w != nil {
|
||||
w(p)
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
return a.fetcher
|
||||
@@ -763,6 +778,42 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
//
|
||||
// It runs in its own goroutine because model loading takes minutes, and the
|
||||
// stream has to keep carrying heartbeats and other commands meanwhile.
|
||||
// watchFetch registers interest in a source's download progress and returns
|
||||
// the function that stops watching.
|
||||
//
|
||||
// Keyed by source rather than by shard because a fetch is single-flight: eight
|
||||
// shards starting together produce one download, and every one of them is
|
||||
// waiting on it.
|
||||
func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func()) {
|
||||
a.mu.Lock()
|
||||
if a.fetchWatchers == nil {
|
||||
a.fetchWatchers = map[string][]func(weights.Progress){}
|
||||
}
|
||||
a.fetchWatchers[source] = append(a.fetchWatchers[source], fn)
|
||||
i := len(a.fetchWatchers[source]) - 1
|
||||
a.mu.Unlock()
|
||||
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
w := a.fetchWatchers[source]
|
||||
if i < len(w) {
|
||||
// Cleared rather than removed, so the indexes other watchers
|
||||
// hold stay valid. The slice is dropped when the last one goes.
|
||||
w[i] = nil
|
||||
}
|
||||
for _, f := range w {
|
||||
if f != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
delete(a.fetchWatchers, source)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
@@ -770,16 +821,18 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
a.shardReplica[shardID] = replicaID
|
||||
a.mu.Unlock()
|
||||
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
}}}
|
||||
send := func(s *llmgwv1.ShardStatus) {
|
||||
s.NodeId, s.ReplicaId, s.ShardId = a.cfg.NodeID, replicaID, shardID
|
||||
select {
|
||||
case out <- msg:
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: s}}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
send(&llmgwv1.ShardStatus{
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
})
|
||||
}
|
||||
|
||||
spec, timeout := agentsvc.ShardSpecFromStart(m)
|
||||
|
||||
@@ -808,10 +861,23 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// arrives minutes into a load, in its own words, on a node nobody is
|
||||
// watching — and because eight shards starting together would otherwise
|
||||
// each fetch the same file.
|
||||
//
|
||||
// Progress is reported upward while it runs, so the control plane can tell
|
||||
// a download from a hang.
|
||||
stop := a.watchFetch(spec.Profile.ModelPath, func(p weights.Progress) {
|
||||
send(&llmgwv1.ShardStatus{
|
||||
State: agentsvc.StateToWire(supervisor.Starting),
|
||||
Fetch: &llmgwv1.FetchProgress{
|
||||
Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal,
|
||||
},
|
||||
})
|
||||
})
|
||||
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil {
|
||||
stop()
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
} else {
|
||||
stop()
|
||||
spec.Profile.ModelPath = path
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ type statusRecorder struct {
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
fetches []fetchReport
|
||||
}
|
||||
|
||||
type status struct {
|
||||
@@ -55,6 +56,25 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
|
||||
}
|
||||
}
|
||||
|
||||
// fetches records download progress, so a test can assert that a fetch is
|
||||
// visible from the control plane rather than only in the node's log.
|
||||
func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.fetches = append(r.fetches, fetchReport{replicaID, shardID, source, done, total})
|
||||
}
|
||||
|
||||
func (r *statusRecorder) fetchReports() []fetchReport {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]fetchReport(nil), r.fetches...)
|
||||
}
|
||||
|
||||
type fetchReport struct {
|
||||
replicaID, shardID, source string
|
||||
done, total int64
|
||||
}
|
||||
|
||||
func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
Reference in new issue
Block a user