Adoption set a shard to Starting and started a liveness poll, and that was all. Nothing health-checked it, so nothing ever moved it to Ready. The control plane then waited out the profile's startup timeout and killed a perfectly healthy engine so it could start an identical one. An agent restart exists precisely so that engines are not interrupted. It was costing every model on the node a full reload, minutes of it, which is the failure adoption was written to prevent. Measured on node05 while deploying today's other changes: the engine answered /health the whole time and the replica sat in Starting for three minutes until the timeout reaped it. I had seen this earlier in the session and worked around it by killing the engine to force a clean restart, which made the symptom go away and left the cause in place. An adopted shard now takes the same two paths a freshly started one does: a rank that serves HTTP is health-checked, a worker is judged by staying alive. Run in the background, because recovery happens before the agent connects and a slow check must not delay registration. An engine that will not answer is left for the control plane to replace rather than killed locally, which would race the decision it already has a timeout for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d71d785598
commit
1431426db0
2 files changed
+104
-1
No files matched your search
+57
-1
@@ -47,6 +47,13 @@ type Config struct {
|
||||
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
||||
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
||||
|
||||
// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
|
||||
// health check before the agent stops waiting on it.
|
||||
//
|
||||
// Shorter than a startup timeout on purpose: this engine is already
|
||||
// running, so it is not loading a model, it is either serving or it is not.
|
||||
AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
|
||||
|
||||
// TLS authenticates both ends of the control-plane stream. Leaving it unset
|
||||
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
||||
TLS tlsconf.Config `koanf:"tls"`
|
||||
@@ -65,6 +72,7 @@ func DefaultConfig() Config {
|
||||
HeartbeatInterval: 5 * time.Second,
|
||||
ReconnectBase: time.Second,
|
||||
ReconnectMax: 30 * time.Second,
|
||||
AdoptReadyTimeout: 60 * time.Second,
|
||||
Superv: supervisor.DefaultConfig(),
|
||||
// NVML gives NVLink topology, so prefer it when this build has it.
|
||||
Discovery: discovery.Options{PreferNVML: true},
|
||||
@@ -94,6 +102,9 @@ func (c *Config) normalize() error {
|
||||
if c.ReconnectBase <= 0 {
|
||||
c.ReconnectBase = d.ReconnectBase
|
||||
}
|
||||
if c.AdoptReadyTimeout <= 0 {
|
||||
c.AdoptReadyTimeout = d.AdoptReadyTimeout
|
||||
}
|
||||
if c.ReconnectMax < c.ReconnectBase {
|
||||
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
|
||||
}
|
||||
@@ -262,11 +273,56 @@ func (a *Agent) recoverOrphans() {
|
||||
a.mu.Unlock()
|
||||
a.log.Info("adopted a running engine",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
|
||||
_ = sh
|
||||
go a.settleAdopted(sh, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// settleAdopted brings an adopted engine to Ready, or lets it fail.
|
||||
//
|
||||
// Adoption stopped at "the process is alive", which left the shard in Starting
|
||||
// with nothing that would ever move it. The control plane then waited out the
|
||||
// profile's startup timeout and killed a perfectly healthy engine so it could
|
||||
// start an identical one — so an agent restart, which exists precisely so that
|
||||
// engines are not interrupted, cost every model on the node a full reload.
|
||||
//
|
||||
// Measured on node05: the engine answered /health the whole time, and the
|
||||
// replica sat in Starting for three minutes until the timeout reaped it.
|
||||
//
|
||||
// Run in the background because recovery happens before the agent connects, and
|
||||
// a slow health check must not delay registration — the control plane needs to
|
||||
// hear about these shards promptly whatever state they are in.
|
||||
func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), a.cfg.AdoptReadyTimeout)
|
||||
defer cancel()
|
||||
|
||||
eng, err := a.reg.Get(o.Spec.Profile.Engine)
|
||||
if err != nil {
|
||||
a.log.Warn("cannot judge an adopted engine; leaving it to the control plane",
|
||||
"shard", o.ShardID, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
// The same two paths a freshly started shard takes: a rank that serves HTTP
|
||||
// is health-checked, a worker is judged by staying alive.
|
||||
if eng.ServesHTTP(o.Spec) {
|
||||
base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort)
|
||||
err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout)
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
}
|
||||
if err != nil {
|
||||
// Not killed here. An adopted engine that will not answer is the
|
||||
// control plane's decision to replace, and it has the timeout for it;
|
||||
// killing it locally would race that.
|
||||
a.log.Warn("an adopted engine did not become ready",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID, "err", err)
|
||||
return
|
||||
}
|
||||
a.log.Info("adopted engine is serving",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID)
|
||||
}
|
||||
|
||||
// runningShards describes what this agent is supervising, for registration.
|
||||
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
var out []*llmgwv1.RunningShard
|
||||
|
||||
@@ -739,3 +739,50 @@ func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
|
||||
t.Errorf("waited %v for a process that had already exited", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedShardCanBecomeReady(t *testing.T) {
|
||||
// Adoption stopped at "the process is alive", leaving the shard in Starting
|
||||
// with nothing that would ever move it. The control plane then waited out
|
||||
// the startup timeout and killed a perfectly healthy engine to start an
|
||||
// identical one — so an agent restart, which exists precisely so engines
|
||||
// are not interrupted, cost every model on the node a full reload.
|
||||
//
|
||||
// Seen on node05: the engine answered /health throughout and the replica
|
||||
// sat in Starting for three minutes until the timeout reaped it.
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20201}
|
||||
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
health := &scriptedHealth{}
|
||||
s2, err := New(Config{
|
||||
WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second,
|
||||
HealthInterval: 10 * time.Millisecond,
|
||||
}, health, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, err := s2.FindOrphans()
|
||||
if err != nil || len(orphans) != 1 {
|
||||
t.Fatalf("orphan not found: %+v (%v)", orphans, err)
|
||||
}
|
||||
adopted, err := s2.Adopt(orphans[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := adopted.Status().State; got != Starting {
|
||||
t.Fatalf("a freshly adopted shard is %s, expected Starting", got)
|
||||
}
|
||||
|
||||
// The engine answers, as node05's did. The shard must be able to reach
|
||||
// Ready from here; before this it had no path to it at all.
|
||||
health.ready.Store(true)
|
||||
if err := s2.WaitReady(t.Context(), adopted, "http://127.0.0.1:20201", "m", 2*time.Second); err != nil {
|
||||
t.Fatalf("an adopted, healthy engine never became ready: %v", err)
|
||||
}
|
||||
if got := adopted.Status().State; got != Ready {
|
||||
t.Errorf("state %s after a passing health check", got)
|
||||
}
|
||||
_ = s2.Stop(t.Context(), "a")
|
||||
}
|
||||
Reference in new issue
Block a user