An adopted engine's state is written without its command; engine-check finds the installed build
CI / test (push) Successful in 3m46s

SetCompute on an engine adopted from an earlier agent called writeState,
which read the command line from sh.cmd - nil for a process this agent did
not start - and the agent crashed at every start on the PoC's nodes until
fixed. An adopted shard now keeps the command line its state file had.

engine-check looked for llama-server one directory too shallow; the agent
installs it under <version-variant>/llama-<build>/. It now looks at both,
preferring a CUDA build on a machine with NVIDIA GPUs. Run on cuda01: a
token computed on CUDA0, 25 of 25 layers offloaded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 14:26:28 +02:00
1 parent b5d650af16
commit 2f595f5999
4 files changed
+85 -7

No files matched your search

+12 -2
View File
@@ -113,7 +113,10 @@ type Shard struct {
exitCode int
logs *ringBuffer
cmd *exec.Cmd
// cmdline is an adopted engine's command line, from its state file: it
// has no cmd of this agent's to read it from.
cmdline []string
cmd *exec.Cmd
// sidecars are the shard's supporting processes, torn down with it.
sidecars []*exec.Cmd
cancel context.CancelFunc
@@ -729,11 +732,17 @@ type shardState struct {
// writeState records everything needed to recognise and describe the process later.
func (s *Supervisor) writeState(sh *Shard) error {
// An adopted engine was started by an earlier agent: its command line
// is the one its state file kept.
cmdline := sh.cmdline
if sh.cmd != nil {
cmdline = sh.cmd.Args
}
b, err := json.Marshal(shardState{
ShardID: sh.ID,
ReplicaID: sh.Spec.ReplicaID,
PID: sh.pid,
Cmdline: sh.cmd.Args,
Cmdline: cmdline,
StartedAt: sh.started,
Spec: sh.Spec,
})
@@ -830,6 +839,7 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
started: o.StartedAt,
done: make(chan struct{}),
adopted: true,
cmdline: o.Cmdline,
}
s.shards[o.ShardID] = sh
s.mu.Unlock()
@@ -873,3 +873,35 @@ func TestWhatAShardComputesOnSurvivesTheAgent(t *testing.T) {
t.Errorf("not kept: %+v", c)
}
}
func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
t.Fatal(err)
}
// An agent restart: the engine is adopted by a new supervisor, which did
// not start it and has no command of its own for it.
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, _ := s2.FindOrphans()
if len(orphans) != 1 {
t.Fatalf("%d orphans", len(orphans))
}
if _, err := s2.Adopt(orphans[0]); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s2.Stop(context.Background(), "a") })
s2.SetCompute("a", engines.Compute{Backend: "CPU"})
// And a third agent still finds it, with its command line and compute.
s3, _ := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
again, _ := s3.FindOrphans()
if len(again) != 1 || len(again[0].Cmdline) == 0 || again[0].Spec.Compute == nil {
t.Fatalf("lost: %+v", again)
}
}