diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d21e0f7..bb854f3 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -47,6 +47,13 @@ type Config struct { ReconnectBase time.Duration `koanf:"reconnect_base"` ReconnectMax time.Duration `koanf:"reconnect_max"` + // AdoptReadyTimeout bounds how long an adopted engine is given to answer a + // health check before the agent stops waiting on it. + // + // Shorter than a startup timeout on purpose: this engine is already + // running, so it is not loading a model, it is either serving or it is not. + AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"` + // TLS authenticates both ends of the control-plane stream. Leaving it unset // keeps the plaintext behaviour, which only belongs on a trusted network. TLS tlsconf.Config `koanf:"tls"` @@ -65,6 +72,7 @@ func DefaultConfig() Config { HeartbeatInterval: 5 * time.Second, ReconnectBase: time.Second, ReconnectMax: 30 * time.Second, + AdoptReadyTimeout: 60 * time.Second, Superv: supervisor.DefaultConfig(), // NVML gives NVLink topology, so prefer it when this build has it. Discovery: discovery.Options{PreferNVML: true}, @@ -94,6 +102,9 @@ func (c *Config) normalize() error { if c.ReconnectBase <= 0 { c.ReconnectBase = d.ReconnectBase } + if c.AdoptReadyTimeout <= 0 { + c.AdoptReadyTimeout = d.AdoptReadyTimeout + } if c.ReconnectMax < c.ReconnectBase { c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax) } @@ -262,11 +273,56 @@ func (a *Agent) recoverOrphans() { a.mu.Unlock() a.log.Info("adopted a running engine", "shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID) - _ = sh + go a.settleAdopted(sh, o) } } } +// settleAdopted brings an adopted engine to Ready, or lets it fail. +// +// Adoption stopped at "the process is alive", which left the shard in Starting +// with nothing that would ever move it. The control plane then waited out the +// profile's startup timeout and killed a perfectly healthy engine so it could +// start an identical one — so an agent restart, which exists precisely so that +// engines are not interrupted, cost every model on the node a full reload. +// +// Measured on node05: the engine answered /health the whole time, and the +// replica sat in Starting for three minutes until the timeout reaped it. +// +// Run in the background because recovery happens before the agent connects, and +// a slow health check must not delay registration — the control plane needs to +// hear about these shards promptly whatever state they are in. +func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { + ctx, cancel := context.WithTimeout(context.Background(), a.cfg.AdoptReadyTimeout) + defer cancel() + + eng, err := a.reg.Get(o.Spec.Profile.Engine) + if err != nil { + a.log.Warn("cannot judge an adopted engine; leaving it to the control plane", + "shard", o.ShardID, "err", err) + return + } + + // The same two paths a freshly started shard takes: a rank that serves HTTP + // is health-checked, a worker is judged by staying alive. + if eng.ServesHTTP(o.Spec) { + base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort) + err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout) + } else { + err = a.sup.WaitAlive(ctx, sh) + } + if err != nil { + // Not killed here. An adopted engine that will not answer is the + // control plane's decision to replace, and it has the timeout for it; + // killing it locally would race that. + a.log.Warn("an adopted engine did not become ready", + "shard", o.ShardID, "replica", o.ReplicaID, "err", err) + return + } + a.log.Info("adopted engine is serving", + "shard", o.ShardID, "replica", o.ReplicaID) +} + // runningShards describes what this agent is supervising, for registration. func (a *Agent) runningShards() []*llmgwv1.RunningShard { var out []*llmgwv1.RunningShard diff --git a/internal/agent/supervisor/supervisor_test.go b/internal/agent/supervisor/supervisor_test.go index 64c8153..8a50c54 100644 --- a/internal/agent/supervisor/supervisor_test.go +++ b/internal/agent/supervisor/supervisor_test.go @@ -739,3 +739,50 @@ func TestWaitAliveFailsOnEarlyExit(t *testing.T) { t.Errorf("waited %v for a process that had already exited", d) } } + +func TestAnAdoptedShardCanBecomeReady(t *testing.T) { + // Adoption stopped at "the process is alive", leaving the shard in Starting + // with nothing that would ever move it. The control plane then waited out + // the startup timeout and killed a perfectly healthy engine to start an + // identical one — so an agent restart, which exists precisely so engines + // are not interrupted, cost every model on the node a full reload. + // + // Seen on node05: the engine answered /health throughout and the replica + // sat in Starting for three minutes until the timeout reaped it. + s, _ := newSupervisor(t) + spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20201} + if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil { + t.Fatal(err) + } + + health := &scriptedHealth{} + s2, err := New(Config{ + WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second, + HealthInterval: 10 * time.Millisecond, + }, health, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + orphans, err := s2.FindOrphans() + if err != nil || len(orphans) != 1 { + t.Fatalf("orphan not found: %+v (%v)", orphans, err) + } + adopted, err := s2.Adopt(orphans[0]) + if err != nil { + t.Fatal(err) + } + if got := adopted.Status().State; got != Starting { + t.Fatalf("a freshly adopted shard is %s, expected Starting", got) + } + + // The engine answers, as node05's did. The shard must be able to reach + // Ready from here; before this it had no path to it at all. + health.ready.Store(true) + if err := s2.WaitReady(t.Context(), adopted, "http://127.0.0.1:20201", "m", 2*time.Second); err != nil { + t.Fatalf("an adopted, healthy engine never became ready: %v", err) + } + if got := adopted.Status().State; got != Ready { + t.Errorf("state %s after a passing health check", got) + } + _ = s2.Stop(t.Context(), "a") +}