Ilker ManapandClaude Opus 5.5
2d14918b99
The registration carries each shard's state as it is then. An adopted engine is health-checked after that, and nothing reported the ready that followed, so when the agent registered first the control plane kept the replica starting while it served. After llmgw restart on the PoC two replicas stayed so, one the only replica of qwen2.5-coder-14b. The session's watcher now reports every shard the registration called not ready once it is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
105 lines
3.2 KiB
Go
105 lines
3.2 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os/exec"
|
|
"testing"
|
|
"time"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
|
)
|
|
|
|
type alwaysReady struct{}
|
|
|
|
func (alwaysReady) Ready(context.Context, string, string, string) error { return nil }
|
|
|
|
func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
|
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// An engine a previous agent started: not this process's child.
|
|
cmd := exec.Command("sleep", "60")
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go func() { _ = cmd.Wait() }()
|
|
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
|
|
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
|
|
out := make(chan *llmgwv1.AgentMessage, 4)
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
go a.watchExits(ctx, out, nil)
|
|
time.Sleep(1500 * time.Millisecond) // seen ready
|
|
_ = cmd.Process.Kill()
|
|
|
|
select {
|
|
case m := <-out:
|
|
s := m.GetShard()
|
|
if s.GetShardId() != "m-r0" || s.GetReplicaId() != "m" {
|
|
t.Errorf("reported %+v", s)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("the adopted engine died and nothing was reported")
|
|
}
|
|
}
|
|
|
|
// Registered while its health check still ran: the ready that follows is
|
|
// reported, or the control plane keeps the replica starting while it serves.
|
|
func TestAShardRegisteredStartingIsReportedReady(t *testing.T) {
|
|
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.Command("sleep", "60")
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
|
|
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
|
|
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
|
|
pending := unready(a.runningShards())
|
|
if !pending["m-r0"] {
|
|
t.Fatalf("registration reported it ready already: %v", pending)
|
|
}
|
|
out := make(chan *llmgwv1.AgentMessage, 4)
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
go a.watchExits(ctx, out, pending)
|
|
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
select {
|
|
case m := <-out:
|
|
s := m.GetShard()
|
|
if s.GetShardId() != "m-r0" || s.GetState() != agentsvc.StateToWire(supervisor.Ready) {
|
|
t.Errorf("reported %+v", s)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("it became ready and nothing was reported")
|
|
}
|
|
select {
|
|
case m := <-out:
|
|
t.Errorf("reported twice: %+v", m.GetShard())
|
|
case <-time.After(2500 * time.Millisecond):
|
|
}
|
|
}
|