An adopted engine's state is written without its command; engine-check finds the installed build
CI / test (push) Successful in 3m46s
CI / test (push) Successful in 3m46s
SetCompute on an engine adopted from an earlier agent called writeState, which read the command line from sh.cmd - nil for a process this agent did not start - and the agent crashed at every start on the PoC's nodes until fixed. An adopted shard now keeps the command line its state file had. engine-check looked for llama-server one directory too shallow; the agent installs it under <version-variant>/llama-<build>/. It now looks at both, preferring a CUDA build on a machine with NVIDIA GPUs. Run on cuda01: a token computed on CUDA0, 25 of 25 layers offloaded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b5d650af16
commit
2f595f5999
4 files changed
+85
-7
No files matched your search
@@ -113,7 +113,10 @@ type Shard struct {
|
|||||||
exitCode int
|
exitCode int
|
||||||
|
|
||||||
logs *ringBuffer
|
logs *ringBuffer
|
||||||
cmd *exec.Cmd
|
// cmdline is an adopted engine's command line, from its state file: it
|
||||||
|
// has no cmd of this agent's to read it from.
|
||||||
|
cmdline []string
|
||||||
|
cmd *exec.Cmd
|
||||||
// sidecars are the shard's supporting processes, torn down with it.
|
// sidecars are the shard's supporting processes, torn down with it.
|
||||||
sidecars []*exec.Cmd
|
sidecars []*exec.Cmd
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
@@ -729,11 +732,17 @@ type shardState struct {
|
|||||||
|
|
||||||
// writeState records everything needed to recognise and describe the process later.
|
// writeState records everything needed to recognise and describe the process later.
|
||||||
func (s *Supervisor) writeState(sh *Shard) error {
|
func (s *Supervisor) writeState(sh *Shard) error {
|
||||||
|
// An adopted engine was started by an earlier agent: its command line
|
||||||
|
// is the one its state file kept.
|
||||||
|
cmdline := sh.cmdline
|
||||||
|
if sh.cmd != nil {
|
||||||
|
cmdline = sh.cmd.Args
|
||||||
|
}
|
||||||
b, err := json.Marshal(shardState{
|
b, err := json.Marshal(shardState{
|
||||||
ShardID: sh.ID,
|
ShardID: sh.ID,
|
||||||
ReplicaID: sh.Spec.ReplicaID,
|
ReplicaID: sh.Spec.ReplicaID,
|
||||||
PID: sh.pid,
|
PID: sh.pid,
|
||||||
Cmdline: sh.cmd.Args,
|
Cmdline: cmdline,
|
||||||
StartedAt: sh.started,
|
StartedAt: sh.started,
|
||||||
Spec: sh.Spec,
|
Spec: sh.Spec,
|
||||||
})
|
})
|
||||||
@@ -830,6 +839,7 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
|||||||
started: o.StartedAt,
|
started: o.StartedAt,
|
||||||
done: make(chan struct{}),
|
done: make(chan struct{}),
|
||||||
adopted: true,
|
adopted: true,
|
||||||
|
cmdline: o.Cmdline,
|
||||||
}
|
}
|
||||||
s.shards[o.ShardID] = sh
|
s.shards[o.ShardID] = sh
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
|
|||||||
@@ -873,3 +873,35 @@ func TestWhatAShardComputesOnSurvivesTheAgent(t *testing.T) {
|
|||||||
t.Errorf("not kept: %+v", c)
|
t.Errorf("not kept: %+v", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||||
|
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// An agent restart: the engine is adopted by a new supervisor, which did
|
||||||
|
// not start it and has no command of its own for it.
|
||||||
|
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||||
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
orphans, _ := s2.FindOrphans()
|
||||||
|
if len(orphans) != 1 {
|
||||||
|
t.Fatalf("%d orphans", len(orphans))
|
||||||
|
}
|
||||||
|
if _, err := s2.Adopt(orphans[0]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = s2.Stop(context.Background(), "a") })
|
||||||
|
s2.SetCompute("a", engines.Compute{Backend: "CPU"})
|
||||||
|
|
||||||
|
// And a third agent still finds it, with its command line and compute.
|
||||||
|
s3, _ := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||||
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||||
|
again, _ := s3.FindOrphans()
|
||||||
|
if len(again) != 1 || len(again[0].Cmdline) == 0 || again[0].Spec.Compute == nil {
|
||||||
|
t.Fatalf("lost: %+v", again)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -309,14 +309,30 @@ func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// installedLlama is the newest llama-server the agent installed here.
|
// llamaRoot is where the agent installs llama.cpp builds.
|
||||||
|
var llamaRoot = "/opt/llmgw/engines/llamacpp"
|
||||||
|
|
||||||
|
// installedLlama is the llama-server the agent installed here: under
|
||||||
|
// llamaRoot/<version-variant>/llama-<build>/, or one level up for older
|
||||||
|
// layouts; on a machine with NVIDIA GPUs a CUDA build first, then the newest.
|
||||||
func installedLlama() string {
|
func installedLlama() string {
|
||||||
m, _ := filepath.Glob("/opt/llmgw/engines/llamacpp/*/llama-server")
|
var found []string
|
||||||
if len(m) == 0 {
|
for _, pat := range []string{"*/llama-server", "*/*/llama-server"} {
|
||||||
|
m, _ := filepath.Glob(filepath.Join(llamaRoot, pat))
|
||||||
|
found = append(found, m...)
|
||||||
|
}
|
||||||
|
if len(found) == 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
sort.Strings(m)
|
sort.Strings(found)
|
||||||
return m[len(m)-1]
|
if _, err := exec.LookPath("nvidia-smi"); err == nil {
|
||||||
|
for i := len(found) - 1; i >= 0; i-- {
|
||||||
|
if strings.Contains(found[i], "cuda") {
|
||||||
|
return found[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found[len(found)-1]
|
||||||
}
|
}
|
||||||
|
|
||||||
// weightDirs are where a node keeps models it has fetched.
|
// weightDirs are where a node keeps models it has fetched.
|
||||||
|
|||||||
@@ -64,3 +64,23 @@ func TestEngineCheckComputesATokenOnTheGPU(t *testing.T) {
|
|||||||
t.Fatalf("a build that falls back to the CPU: %v %s", err, out.String())
|
t.Fatalf("a build that falls back to the CPU: %v %s", err, out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEngineCheckFindsTheBuildTheAgentInstalled(t *testing.T) {
|
||||||
|
old := llamaRoot
|
||||||
|
llamaRoot = t.TempDir()
|
||||||
|
t.Cleanup(func() { llamaRoot = old })
|
||||||
|
if got := installedLlama(); got != "" {
|
||||||
|
t.Fatalf("nothing installed, found %q", got)
|
||||||
|
}
|
||||||
|
for _, p := range []string{"b10991-cpu/llama-b10991/llama-server", "b10991-cuda-13.3/llama-b10991/llama-server"} {
|
||||||
|
full := filepath.Join(llamaRoot, p)
|
||||||
|
_ = os.MkdirAll(filepath.Dir(full), 0o755)
|
||||||
|
_ = os.WriteFile(full, []byte("x"), 0o755)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
_ = os.WriteFile(filepath.Join(dir, "nvidia-smi"), []byte("#!/bin/sh\n"), 0o755)
|
||||||
|
t.Setenv("PATH", dir)
|
||||||
|
if got := installedLlama(); !strings.Contains(got, "cuda-13.3/llama-b10991/llama-server") {
|
||||||
|
t.Errorf("with GPUs, not the CUDA build: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user