From 557a5a1ef3e23771e387801147d7d072d0b1fe09 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 12 Nov 2026 10:38:17 +0100 Subject: [PATCH] Found on a fresh three-machine install: what the screens said wrong - server init printed every engine's line again on each look while the node installed (some 280 lines); each engine's line now once, and again only when it changes, as node add does. - A package install that is no engine's (Slurm's, MariaDB's) read ": installing installing mariadb-server, ..."; it is "system packages: installing: mariadb-server, ..., with apt-get". - The slow-judge advice says "ctl model resources JUDGE --cores 4", which was refused for want of --memory: cores alone now keep the memory as it is. - server add said both control planes for three. Co-Authored-By: Claude Opus 5.5 --- internal/agent/provision/install.go | 3 ++- internal/cli/install_lines_test.go | 34 ++++++++++++++++++++++++++ internal/cli/model.go | 37 ++++++++++++++++++++++++++--- internal/cli/node.go | 12 +++++++++- internal/cli/serveradd.go | 2 +- internal/cli/serverinit_cluster.go | 10 ++++---- 6 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 internal/cli/install_lines_test.go diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 27311ba..12a24e0 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -483,7 +483,8 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, return fmt.Errorf("%s is missing and no package manager was found to install it", strings.Join(missing, ", ")) } - in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool) + // The state already says installing. + in.progress(st, "%s, with %s", strings.Join(missing, ", "), tool) for _, c := range cmds { if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil { return err diff --git a/internal/cli/install_lines_test.go b/internal/cli/install_lines_test.go new file mode 100644 index 0000000..73b9a08 --- /dev/null +++ b/internal/cli/install_lines_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +func TestAPackageInstallThatIsNoEnginesIsNamed(t *testing.T) { + if got := provisionLine("", "", "installing", "mariadb-server, munge, with apt-get"); got != + "system packages: installing: mariadb-server, munge, with apt-get" { + t.Fatalf("got %q", got) + } + if got := provisionLine("llamacpp", "b10991", "ready", ""); got != "llamacpp b10991: ready" { + t.Fatalf("got %q", got) + } +} + +func TestCoresAloneKeepTheMemory(t *testing.T) { + m := catalog.Model{Name: "judge", Profiles: []catalog.Profile{{Name: "cpu", MinMemoryGB: 2}, {Name: "cpu-big", MinMemoryGB: 2}}} + if got, err := keptMemory(m, ""); err != nil || got != 2 { + t.Fatalf("got %d, %v", got, err) + } + m.Profiles[1].MinMemoryGB = 4 + if _, err := keptMemory(m, ""); err == nil { + t.Fatal("profiles that differ were given one memory") + } + if got, err := keptMemory(m, "cpu-big"); err != nil || got != 4 { + t.Fatalf("got %d, %v", got, err) + } + if _, err := keptMemory(m, "gpu"); err == nil { + t.Fatal("a profile the model has not was taken") + } +} diff --git a/internal/cli/model.go b/internal/cli/model.go index 900cf25..9a7240f 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -338,7 +338,7 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command { var memory, cores int var profile string c := &cobra.Command{ - Use: "resources --memory GB [--cores N]", + Use: "resources [--memory GB] [--cores N]", Short: "Set what a model reserves of a node: memory, and CPU cores", Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" + "weights, KV cache and engine together. Placement reserves it on the node,\n" + @@ -351,8 +351,20 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command { "--model M` moves them to the new figures.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("memory") && !cmd.Flags().Changed("cores") { + return fmt.Errorf("give --memory, --cores or both") + } if !cmd.Flags().Changed("memory") { - return fmt.Errorf("--memory is required") + // Cores alone: the memory stays what it is. + var cur catalog.Model + if err := client().get(cmd.Context(), "/api/v1/models/"+args[0], &cur); err != nil { + return err + } + kept, err := keptMemory(cur, profile) + if err != nil { + return err + } + memory = kept } body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores}) if err != nil { @@ -369,12 +381,31 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command { return nil }, } - c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB") + c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB; kept as it is when left out") c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none") c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out") return c } +// keptMemory is the memory a model's profiles reserve now, for a change of +// cores alone: one profile's, or every profile's when they agree. +func keptMemory(m catalog.Model, profile string) (int, error) { + memory := -1 + for _, p := range m.Profiles { + if profile != "" && p.Name != profile { + continue + } + if memory >= 0 && p.MinMemoryGB != memory { + return 0, fmt.Errorf("%s's profiles reserve different memory: give --memory, or --profile", m.Name) + } + memory = p.MinMemoryGB + } + if memory < 0 { + return 0, fmt.Errorf("%s has no profile %q", m.Name, profile) + } + return memory, nil +} + func newClassCmd(client func() *ctlClient) *cobra.Command { return &cobra.Command{ Use: "class critical|standard|opportunistic|default", diff --git a/internal/cli/node.go b/internal/cli/node.go index 6da7de5..681107b 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -495,7 +495,17 @@ func engineEvent(engine, version, state, detail string) onboard.Event { level = onboard.Warn } return onboard.Event{Step: onboard.StepEngines, Level: level, - Detail: fmt.Sprintf("%s %s: %s%s", engine, version, state, prefixed(": ", detail))} + Detail: provisionLine(engine, version, state, detail)} +} + +// provisionLine is one line of what a node is installing. A package +// install that is no engine's, Slurm's or MariaDB's, has no name or version. +func provisionLine(engine, version, state, detail string) string { + what := strings.TrimSpace(engine + " " + version) + if what == "" { + what = "system packages" + } + return fmt.Sprintf("%s: %s%s", what, state, prefixed(": ", detail)) } func prefixed(p, s string) string { diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 9b27bef..c07d5e8 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -443,7 +443,7 @@ func (sa *serverAdd) storeAccess(ctx context.Context) error { return err } } - sa.done("nodes will be told of both control planes within half a minute") + sa.done("nodes will be told of every control plane within half a minute") return nil } diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 856d2a0..29fb783 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -398,7 +398,9 @@ labels: // checks its own llama.cpp, the CUDA build when it has a GPU, and a new // machine is often busy with its own package updates first. client := in.adminAPI() - last := "" + // Each engine's line once, and again only when it changes: the engines' + // states come back together on every look. + seen := map[string]string{} err = waitFor(ctx, 20*time.Minute, "this node to be ready with an engine", func() (bool, error) { var nodes []adminapi.NodeView if err := client.get(ctx, "/api/v1/nodes", &nodes); err != nil { @@ -412,9 +414,9 @@ labels: return true, nil } for _, p := range n.Provision { - if line := fmt.Sprintf("%s %s: %s %s", p.Engine, p.Version, p.State, p.Detail); line != last { - fmt.Fprintf(in.out, " .. %s\n", strings.TrimSpace(line)) - last = line + if line := provisionLine(p.Engine, p.Version, p.State, p.Detail); seen[p.Engine] != line { + fmt.Fprintf(in.out, " .. %s\n", line) + seen[p.Engine] = line } // Only llama.cpp is waited for: vLLM installs for minutes after, // and one this node cannot run is not a reason to stop here.