Found on a fresh three-machine install: what the screens said wrong
CI / test (push) Successful in 11m5s

- server init printed every engine's line again on each look while the
  node installed (some 280 lines); each engine's line now once, and again
  only when it changes, as node add does.
- A package install that is no engine's (Slurm's, MariaDB's) read
  ": installing installing mariadb-server, ..."; it is "system
  packages: installing: mariadb-server, ..., with apt-get".
- The slow-judge advice says "ctl model resources JUDGE --cores 4",
  which was refused for want of --memory: cores alone now keep the
  memory as it is.
- server add said both control planes for three.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 10:38:18 +01:00
1 parent 21828fefce
commit 557a5a1ef3
6 files changed
+88 -10

No files matched your search

+2 -1
View File
@@ -483,7 +483,8 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
return fmt.Errorf("%s is missing and no package manager was found to install it",
strings.Join(missing, ", "))
}
in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
// The state already says installing.
in.progress(st, "%s, with %s", strings.Join(missing, ", "), tool)
for _, c := range cmds {
if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
return err
+34
View File
@@ -0,0 +1,34 @@
package cli
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func TestAPackageInstallThatIsNoEnginesIsNamed(t *testing.T) {
if got := provisionLine("", "", "installing", "mariadb-server, munge, with apt-get"); got !=
"system packages: installing: mariadb-server, munge, with apt-get" {
t.Fatalf("got %q", got)
}
if got := provisionLine("llamacpp", "b10991", "ready", ""); got != "llamacpp b10991: ready" {
t.Fatalf("got %q", got)
}
}
func TestCoresAloneKeepTheMemory(t *testing.T) {
m := catalog.Model{Name: "judge", Profiles: []catalog.Profile{{Name: "cpu", MinMemoryGB: 2}, {Name: "cpu-big", MinMemoryGB: 2}}}
if got, err := keptMemory(m, ""); err != nil || got != 2 {
t.Fatalf("got %d, %v", got, err)
}
m.Profiles[1].MinMemoryGB = 4
if _, err := keptMemory(m, ""); err == nil {
t.Fatal("profiles that differ were given one memory")
}
if got, err := keptMemory(m, "cpu-big"); err != nil || got != 4 {
t.Fatalf("got %d, %v", got, err)
}
if _, err := keptMemory(m, "gpu"); err == nil {
t.Fatal("a profile the model has not was taken")
}
}
+34 -3
View File
@@ -338,7 +338,7 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command {
var memory, cores int
var profile string
c := &cobra.Command{
Use: "resources <model> --memory GB [--cores N]",
Use: "resources <model> [--memory GB] [--cores N]",
Short: "Set what a model reserves of a node: memory, and CPU cores",
Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
"weights, KV cache and engine together. Placement reserves it on the node,\n" +
@@ -351,8 +351,20 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command {
"--model M` moves them to the new figures.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if !cmd.Flags().Changed("memory") && !cmd.Flags().Changed("cores") {
return fmt.Errorf("give --memory, --cores or both")
}
if !cmd.Flags().Changed("memory") {
return fmt.Errorf("--memory is required")
// Cores alone: the memory stays what it is.
var cur catalog.Model
if err := client().get(cmd.Context(), "/api/v1/models/"+args[0], &cur); err != nil {
return err
}
kept, err := keptMemory(cur, profile)
if err != nil {
return err
}
memory = kept
}
body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
if err != nil {
@@ -369,12 +381,31 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command {
return nil
},
}
c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB; kept as it is when left out")
c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
return c
}
// keptMemory is the memory a model's profiles reserve now, for a change of
// cores alone: one profile's, or every profile's when they agree.
func keptMemory(m catalog.Model, profile string) (int, error) {
memory := -1
for _, p := range m.Profiles {
if profile != "" && p.Name != profile {
continue
}
if memory >= 0 && p.MinMemoryGB != memory {
return 0, fmt.Errorf("%s's profiles reserve different memory: give --memory, or --profile", m.Name)
}
memory = p.MinMemoryGB
}
if memory < 0 {
return 0, fmt.Errorf("%s has no profile %q", m.Name, profile)
}
return memory, nil
}
func newClassCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "class <model> critical|standard|opportunistic|default",
+11 -1
View File
@@ -495,7 +495,17 @@ func engineEvent(engine, version, state, detail string) onboard.Event {
level = onboard.Warn
}
return onboard.Event{Step: onboard.StepEngines, Level: level,
Detail: fmt.Sprintf("%s %s: %s%s", engine, version, state, prefixed(": ", detail))}
Detail: provisionLine(engine, version, state, detail)}
}
// provisionLine is one line of what a node is installing. A package
// install that is no engine's, Slurm's or MariaDB's, has no name or version.
func provisionLine(engine, version, state, detail string) string {
what := strings.TrimSpace(engine + " " + version)
if what == "" {
what = "system packages"
}
return fmt.Sprintf("%s: %s%s", what, state, prefixed(": ", detail))
}
func prefixed(p, s string) string {
+1 -1
View File
@@ -443,7 +443,7 @@ func (sa *serverAdd) storeAccess(ctx context.Context) error {
return err
}
}
sa.done("nodes will be told of both control planes within half a minute")
sa.done("nodes will be told of every control plane within half a minute")
return nil
}
+6 -4
View File
@@ -398,7 +398,9 @@ labels:
// checks its own llama.cpp, the CUDA build when it has a GPU, and a new
// machine is often busy with its own package updates first.
client := in.adminAPI()
last := ""
// Each engine's line once, and again only when it changes: the engines'
// states come back together on every look.
seen := map[string]string{}
err = waitFor(ctx, 20*time.Minute, "this node to be ready with an engine", func() (bool, error) {
var nodes []adminapi.NodeView
if err := client.get(ctx, "/api/v1/nodes", &nodes); err != nil {
@@ -412,9 +414,9 @@ labels:
return true, nil
}
for _, p := range n.Provision {
if line := fmt.Sprintf("%s %s: %s %s", p.Engine, p.Version, p.State, p.Detail); line != last {
fmt.Fprintf(in.out, " .. %s\n", strings.TrimSpace(line))
last = line
if line := provisionLine(p.Engine, p.Version, p.State, p.Detail); seen[p.Engine] != line {
fmt.Fprintf(in.out, " .. %s\n", line)
seen[p.Engine] = line
}
// Only llama.cpp is waited for: vLLM installs for minutes after,
// and one this node cannot run is not a reason to stop here.