Report a ready engine dying from every connection, adopted ones included

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 11:50:09 +02:00
1 parent bd155c18ac
commit efb4b06fc5
3 files changed
+109 -21

No files matched your search

+13
View File
@@ -212,6 +212,19 @@ APIs change between minors without a deprecation period.
`firewall_networks`; `--no-firewall` skips it; `llmgw doctor` says when a
node lacks it.
### Fixed
- **An engine that died after an agent restart went unnoticed.** The watcher
that reports a ready engine dying was tied to the stream it was started
on, so after a reconnect nothing reported it, and an engine adopted from
the previous agent had no watcher at all: its replica stayed "ready" and
was routed to. Every connection now watches every ready shard, started or
adopted. Found on the PoC: a killed engine on node06 stayed ready for
minutes.
- `server init` rewriting the store's address no longer leaves
`/etc/llmgw/secrets` unreadable to the service when it was made 0700 for
the service user, which kept the gateway from starting.
### Everything that can be changed on a running cluster, from the console
`sudo llmgw` only watched the cluster and edited the configuration files;
+40 -21
View File
@@ -518,6 +518,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
a.sendLoop(ctx, stream, out)
}()
defer wg.Wait()
go a.watchExits(ctx, out)
for {
msg, err := stream.Receive()
@@ -1050,36 +1051,54 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
return
}
report(supervisor.Ready, "", nil)
// Watch for the engine dying later. Nothing else would notice: the router
// would simply see failing requests.
go a.watch(ctx, sh, replicaID, shardID, out)
// Its dying later is reported by the session's watchExits.
}
// watch reports a shard that exits after it was ready.
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
// watchExits reports, for as long as this session lasts, every shard that
// was ready and has failed: the ones this agent started, and the ones it
// adopted from the agent before it.
//
// Per session rather than per shard. A watcher tied to the stream a shard
// was started on stopped with that stream, so after a reconnect nothing
// reported the engine dying; and an adopted engine had no watcher at all —
// on node06 one was killed and its replica stayed "ready" for the control
// plane, routed to and failing, indefinitely.
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
t := time.NewTicker(time.Second)
defer t.Stop()
ready := map[string]bool{}
for {
for _, st := range a.sup.Statuses() {
switch st.State {
case supervisor.Ready:
ready[st.ID] = true
case supervisor.Failed:
if !ready[st.ID] {
continue // never ready here: its start reported it
}
delete(ready, st.ID)
sh, err := a.sup.Get(st.ID)
if err != nil {
continue
}
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID,
State: agentsvc.StateToWire(st.State), Error: st.Err,
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
}}}
select {
case out <- msg:
case <-ctx.Done():
return
}
default:
delete(ready, st.ID)
}
}
select {
case <-ctx.Done():
return
case <-t.C:
st := sh.Status()
if !st.State.Terminal() {
continue
}
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(st.State), Error: st.Err,
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
}}}
select {
case out <- msg:
case <-ctx.Done():
}
return
}
}
}
+56
View File
@@ -0,0 +1,56 @@
package agent
import (
"context"
"log/slog"
"os/exec"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
type alwaysReady struct{}
func (alwaysReady) Ready(context.Context, string, string, string) error { return nil }
func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
// An engine a previous agent started: not this process's child.
cmd := exec.Command("sleep", "60")
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
go func() { _ = cmd.Wait() }()
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
if err != nil {
t.Fatal(err)
}
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
t.Fatal(err)
}
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
out := make(chan *llmgwv1.AgentMessage, 4)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
go a.watchExits(ctx, out)
time.Sleep(1500 * time.Millisecond) // seen ready
_ = cmd.Process.Kill()
select {
case m := <-out:
s := m.GetShard()
if s.GetShardId() != "m-r0" || s.GetReplicaId() != "m" {
t.Errorf("reported %+v", s)
}
case <-time.After(10 * time.Second):
t.Fatal("the adopted engine died and nothing was reported")
}
}