diff --git a/CHANGELOG.md b/CHANGELOG.md index 851819ec..6bb471fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -212,6 +212,19 @@ APIs change between minors without a deprecation period. `firewall_networks`; `--no-firewall` skips it; `llmgw doctor` says when a node lacks it. + +### Fixed + +- **An engine that died after an agent restart went unnoticed.** The watcher + that reports a ready engine dying was tied to the stream it was started + on, so after a reconnect nothing reported it, and an engine adopted from + the previous agent had no watcher at all: its replica stayed "ready" and + was routed to. Every connection now watches every ready shard, started or + adopted. Found on the PoC: a killed engine on node06 stayed ready for + minutes. +- `server init` rewriting the store's address no longer leaves + `/etc/llmgw/secrets` unreadable to the service when it was made 0700 for + the service user, which kept the gateway from starting. ### Everything that can be changed on a running cluster, from the console `sudo llmgw` only watched the cluster and edited the configuration files; diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 832bd2fb..4d6008bc 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -518,6 +518,7 @@ func (a *Agent) session(ctx context.Context, server string) error { a.sendLoop(ctx, stream, out) }() defer wg.Wait() + go a.watchExits(ctx, out) for { msg, err := stream.Receive() @@ -1050,36 +1051,54 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< return } report(supervisor.Ready, "", nil) - - // Watch for the engine dying later. Nothing else would notice: the router - // would simply see failing requests. - go a.watch(ctx, sh, replicaID, shardID, out) + // Its dying later is reported by the session's watchExits. } -// watch reports a shard that exits after it was ready. -func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) { +// watchExits reports, for as long as this session lasts, every shard that +// was ready and has failed: the ones this agent started, and the ones it +// adopted from the agent before it. +// +// Per session rather than per shard. A watcher tied to the stream a shard +// was started on stopped with that stream, so after a reconnect nothing +// reported the engine dying; and an adopted engine had no watcher at all — +// on node06 one was killed and its replica stayed "ready" for the control +// plane, routed to and failing, indefinitely. +func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage) { t := time.NewTicker(time.Second) defer t.Stop() - + ready := map[string]bool{} for { + for _, st := range a.sup.Statuses() { + switch st.State { + case supervisor.Ready: + ready[st.ID] = true + case supervisor.Failed: + if !ready[st.ID] { + continue // never ready here: its start reported it + } + delete(ready, st.ID) + sh, err := a.sup.Get(st.ID) + if err != nil { + continue + } + msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{ + NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID, + State: agentsvc.StateToWire(st.State), Error: st.Err, + ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20), + }}} + select { + case out <- msg: + case <-ctx.Done(): + return + } + default: + delete(ready, st.ID) + } + } select { case <-ctx.Done(): return case <-t.C: - st := sh.Status() - if !st.State.Terminal() { - continue - } - msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{ - NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID, - State: agentsvc.StateToWire(st.State), Error: st.Err, - ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20), - }}} - select { - case out <- msg: - case <-ctx.Done(): - } - return } } } diff --git a/internal/agent/watchexits_test.go b/internal/agent/watchexits_test.go new file mode 100644 index 00000000..9343fe95 --- /dev/null +++ b/internal/agent/watchexits_test.go @@ -0,0 +1,56 @@ +package agent + +import ( + "context" + "log/slog" + "os/exec" + "testing" + "time" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +type alwaysReady struct{} + +func (alwaysReady) Ready(context.Context, string, string, string) error { return nil } + +func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) { + sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + // An engine a previous agent started: not this process's child. + cmd := exec.Command("sleep", "60") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + go func() { _ = cmd.Wait() }() + sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid, + Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true}) + if err != nil { + t.Fatal(err) + } + if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil { + t.Fatal(err) + } + + a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)} + out := make(chan *llmgwv1.AgentMessage, 4) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.watchExits(ctx, out) + time.Sleep(1500 * time.Millisecond) // seen ready + _ = cmd.Process.Kill() + + select { + case m := <-out: + s := m.GetShard() + if s.GetShardId() != "m-r0" || s.GetReplicaId() != "m" { + t.Errorf("reported %+v", s) + } + case <-time.After(10 * time.Second): + t.Fatal("the adopted engine died and nothing was reported") + } +}