An adopted engine's state is written without its command; engine-check finds the installed build
CI / test (push) Successful in 3m46s
CI / test (push) Successful in 3m46s
SetCompute on an engine adopted from an earlier agent called writeState, which read the command line from sh.cmd - nil for a process this agent did not start - and the agent crashed at every start on the PoC's nodes until fixed. An adopted shard now keeps the command line its state file had. engine-check looked for llama-server one directory too shallow; the agent installs it under <version-variant>/llama-<build>/. It now looks at both, preferring a CUDA build on a machine with NVIDIA GPUs. Run on cuda01: a token computed on CUDA0, 25 of 25 layers offloaded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b5d650af16
commit
2f595f5999
4 files changed
+85
-7
No files matched your search
@@ -113,7 +113,10 @@ type Shard struct {
|
||||
exitCode int
|
||||
|
||||
logs *ringBuffer
|
||||
cmd *exec.Cmd
|
||||
// cmdline is an adopted engine's command line, from its state file: it
|
||||
// has no cmd of this agent's to read it from.
|
||||
cmdline []string
|
||||
cmd *exec.Cmd
|
||||
// sidecars are the shard's supporting processes, torn down with it.
|
||||
sidecars []*exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
@@ -729,11 +732,17 @@ type shardState struct {
|
||||
|
||||
// writeState records everything needed to recognise and describe the process later.
|
||||
func (s *Supervisor) writeState(sh *Shard) error {
|
||||
// An adopted engine was started by an earlier agent: its command line
|
||||
// is the one its state file kept.
|
||||
cmdline := sh.cmdline
|
||||
if sh.cmd != nil {
|
||||
cmdline = sh.cmd.Args
|
||||
}
|
||||
b, err := json.Marshal(shardState{
|
||||
ShardID: sh.ID,
|
||||
ReplicaID: sh.Spec.ReplicaID,
|
||||
PID: sh.pid,
|
||||
Cmdline: sh.cmd.Args,
|
||||
Cmdline: cmdline,
|
||||
StartedAt: sh.started,
|
||||
Spec: sh.Spec,
|
||||
})
|
||||
@@ -830,6 +839,7 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
||||
started: o.StartedAt,
|
||||
done: make(chan struct{}),
|
||||
adopted: true,
|
||||
cmdline: o.Cmdline,
|
||||
}
|
||||
s.shards[o.ShardID] = sh
|
||||
s.mu.Unlock()
|
||||
|
||||
@@ -873,3 +873,35 @@ func TestWhatAShardComputesOnSurvivesTheAgent(t *testing.T) {
|
||||
t.Errorf("not kept: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An agent restart: the engine is adopted by a new supervisor, which did
|
||||
// not start it and has no command of its own for it.
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, _ := s2.FindOrphans()
|
||||
if len(orphans) != 1 {
|
||||
t.Fatalf("%d orphans", len(orphans))
|
||||
}
|
||||
if _, err := s2.Adopt(orphans[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s2.Stop(context.Background(), "a") })
|
||||
s2.SetCompute("a", engines.Compute{Backend: "CPU"})
|
||||
|
||||
// And a third agent still finds it, with its command line and compute.
|
||||
s3, _ := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
again, _ := s3.FindOrphans()
|
||||
if len(again) != 1 || len(again[0].Cmdline) == 0 || again[0].Spec.Compute == nil {
|
||||
t.Fatalf("lost: %+v", again)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user