Report a ready engine dying from every connection, adopted ones included
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bd155c18ac
commit
efb4b06fc5
3 files changed
+109
-21
No files matched your search
@@ -212,6 +212,19 @@ APIs change between minors without a deprecation period.
|
||||
`firewall_networks`; `--no-firewall` skips it; `llmgw doctor` says when a
|
||||
node lacks it.
|
||||
|
||||
|
||||
### Fixed
|
||||
|
||||
- **An engine that died after an agent restart went unnoticed.** The watcher
|
||||
that reports a ready engine dying was tied to the stream it was started
|
||||
on, so after a reconnect nothing reported it, and an engine adopted from
|
||||
the previous agent had no watcher at all: its replica stayed "ready" and
|
||||
was routed to. Every connection now watches every ready shard, started or
|
||||
adopted. Found on the PoC: a killed engine on node06 stayed ready for
|
||||
minutes.
|
||||
- `server init` rewriting the store's address no longer leaves
|
||||
`/etc/llmgw/secrets` unreadable to the service when it was made 0700 for
|
||||
the service user, which kept the gateway from starting.
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
`sudo llmgw` only watched the cluster and edited the configuration files;
|
||||
|
||||
+40
-21
@@ -518,6 +518,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
a.sendLoop(ctx, stream, out)
|
||||
}()
|
||||
defer wg.Wait()
|
||||
go a.watchExits(ctx, out)
|
||||
|
||||
for {
|
||||
msg, err := stream.Receive()
|
||||
@@ -1050,36 +1051,54 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
return
|
||||
}
|
||||
report(supervisor.Ready, "", nil)
|
||||
|
||||
// Watch for the engine dying later. Nothing else would notice: the router
|
||||
// would simply see failing requests.
|
||||
go a.watch(ctx, sh, replicaID, shardID, out)
|
||||
// Its dying later is reported by the session's watchExits.
|
||||
}
|
||||
|
||||
// watch reports a shard that exits after it was ready.
|
||||
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
|
||||
// watchExits reports, for as long as this session lasts, every shard that
|
||||
// was ready and has failed: the ones this agent started, and the ones it
|
||||
// adopted from the agent before it.
|
||||
//
|
||||
// Per session rather than per shard. A watcher tied to the stream a shard
|
||||
// was started on stopped with that stream, so after a reconnect nothing
|
||||
// reported the engine dying; and an adopted engine had no watcher at all —
|
||||
// on node06 one was killed and its replica stayed "ready" for the control
|
||||
// plane, routed to and failing, indefinitely.
|
||||
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
ready := map[string]bool{}
|
||||
for {
|
||||
for _, st := range a.sup.Statuses() {
|
||||
switch st.State {
|
||||
case supervisor.Ready:
|
||||
ready[st.ID] = true
|
||||
case supervisor.Failed:
|
||||
if !ready[st.ID] {
|
||||
continue // never ready here: its start reported it
|
||||
}
|
||||
delete(ready, st.ID)
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID,
|
||||
State: agentsvc.StateToWire(st.State), Error: st.Err,
|
||||
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
default:
|
||||
delete(ready, st.ID)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
st := sh.Status()
|
||||
if !st.State.Terminal() {
|
||||
continue
|
||||
}
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||
State: agentsvc.StateToWire(st.State), Error: st.Err,
|
||||
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
type alwaysReady struct{}
|
||||
|
||||
func (alwaysReady) Ready(context.Context, string, string, string) error { return nil }
|
||||
|
||||
func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
||||
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An engine a previous agent started: not this process's child.
|
||||
cmd := exec.Command("sleep", "60")
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() { _ = cmd.Wait() }()
|
||||
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
|
||||
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
|
||||
out := make(chan *llmgwv1.AgentMessage, 4)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go a.watchExits(ctx, out)
|
||||
time.Sleep(1500 * time.Millisecond) // seen ready
|
||||
_ = cmd.Process.Kill()
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
s := m.GetShard()
|
||||
if s.GetShardId() != "m-r0" || s.GetReplicaId() != "m" {
|
||||
t.Errorf("reported %+v", s)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the adopted engine died and nothing was reported")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user