diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bbf6ab..134fe04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,13 @@ APIs change between minors without a deprecation period. ### Fixed +- **A machine too small for the models it would serve is said so first.** + The control plane that is also a node, serving the starter model and two + judges, needs about 10.5 GB. On 8 GB the install finished every step and + then had no room for the starter. Now the setup's checks fail its memory + with what it would serve, and `server init` refuses before it installs + anything, naming `--no-model`, `--no-guard-model` and `--no-node`. + - **Turkish pages are Turkish to the end.** - **Errors:** those from the store, catalog and mirror that a page shows ("not found", "not empty", "already in the catalog", the settings changed meanwhile, and others) are now in the page's language. diff --git a/docs/quick-install.md b/docs/quick-install.md index 0a1b754..49ef7e6 100644 --- a/docs/quick-install.md +++ b/docs/quick-install.md @@ -8,7 +8,7 @@ explains every choice. | | | |---|---| -| **A control plane** | One machine, or three for high availability (the store replicated, either one able to take over). Linux, x86-64 or ARM64: Debian 12, Ubuntu 22.04+ or RHEL 10. No GPU needed; 2 GB RAM, 20 GB disk each — 8 GB RAM when it is also a node running the starter model and the two judges. | +| **A control plane** | One machine, or three for high availability (the store replicated, either one able to take over). Linux, x86-64 or ARM64: Debian 12, Ubuntu 22.04+ or RHEL 10. No GPU needed; 2 GB RAM, 20 GB disk each — 12 GB RAM when it is also a node running the starter model and the two judges, which setup checks before it installs anything; four cores or more, or the judges on machines of their own, keep a judgment within its wait. | | **GPU nodes** (optional to start) | The same Linux, with the NVIDIA driver installed. vLLM needs driver 525 or newer and a Turing GPU or newer; a V100 gets an older vLLM. | | **root** | on the control plane. On the other two of a three-machine setup, `sudo` (its password is asked for if it needs one); on nodes, `sudo` without a password. | | **SSH** | from the control plane to the other machines. The setup screens take a key, the ssh agent or a password; `node add` uses ssh keys. A machine nothing may log into joins with a code instead (`llmgw join`). | diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index d6b0a14..a2ce9cc 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -266,6 +266,9 @@ func (in *serverInit) run(ctx context.Context) error { if err := in.checkBundle(); err != nil { return fmt.Errorf("offline bundle: %w", err) } + if err := in.checkMemory(); err != nil { + return err + } steps := []struct { name string fn func(context.Context) error diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index 654833b..cf72682 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "net" + "os" "path" "slices" "strconv" @@ -11,6 +12,8 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" + "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" ) // What a first install lets a person choose beyond which parts it has: which @@ -154,6 +157,62 @@ func (s starterSpec) minMemoryGB() int { return max(2, int(math.Ceil(s.SizeGB+1.5))) } +// servingMemoryMB is what this machine needs, beyond what the control plane +// does, for the models it will serve itself: the starter's and the judges' +// reservations, and a gigabyte more because the scheduler counts a machine's +// memory in whole gigabytes, rounded down — 7.75 GB is 7. Zero when it serves +// none of them; what names those it does. +// +// Asked before anything is installed: the 8 GB machine the guide was first +// recorded on finished every step and then had no room for the starter, the +// one thing a person asks first. +func (in *serverInit) servingMemoryMB(judgesElsewhere bool) (mb int, what string) { + if in.noNode { + return 0, "" + } + gb := 0 + var parts []string + if !in.noModel && (!in.noMirror || in.mirrorURL != "") { + gb += in.starterModelSpec().minMemoryGB() + parts = append(parts, "the starter model") + } + if !in.noGuardModel && !judgesElsewhere { + n := in.judgeFor().Replicas + gb += n * judgeMemoryGB + parts = append(parts, fmt.Sprintf("%d judges", n)) + } + if gb == 0 { + return 0, "" + } + return (gb + 1) * 1024, strings.Join(parts, " and ") +} + +// checkMemory refuses, before anything is installed, a machine too small for +// the models it would serve as well as the control plane. +func (in *serverInit) checkMemory() error { + if in.unitPath == "" { + return nil // a development run, served by nothing + } + meminfo, err := os.ReadFile("/proc/meminfo") + if err != nil { + return nil + } + total, ok := pgha.MemTotal(string(meminfo)) + if !ok { + return nil + } + extra, what := in.servingMemoryMB(len(in.judgeMachines) > 0) + if extra == 0 { + return nil + } + have, need := int(total>>20), preflight.Needs[preflight.RoleControl].MemoryMB+extra + if have >= need { + return nil + } + return fmt.Errorf("memory: this machine has %d MB; the control plane and %s it would serve need %d. "+ + "Give it more, or leave them off it: --no-model, --no-guard-model, or --no-node to serve models only on nodes added later", have, what, need) +} + // defaultContext is the context a client may use of the starter. const defaultContext = 4096 diff --git a/internal/cli/serverinit_memory_test.go b/internal/cli/serverinit_memory_test.go new file mode 100644 index 0000000..5436df4 --- /dev/null +++ b/internal/cli/serverinit_memory_test.go @@ -0,0 +1,27 @@ +package cli + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" +) + +func TestTheMemoryAMachineNeedsCountsWhatItServes(t *testing.T) { + in := &serverInit{} + mb, what := in.servingMemoryMB(false) + need := preflight.Needs[preflight.RoleControl].MemoryMB + mb + if what == "" || need <= 7941 || need > 11961 { + t.Errorf("node, starter and judges here: need %d MB (%q); an 8 GB machine must fail and a 12 GB one pass", need, what) + } + // Judges on machines of their own take their share off this one. + if mb2, _ := in.servingMemoryMB(true); mb2 >= mb { + t.Errorf("with judge machines %d MB, without %d", mb2, mb) + } + // A control plane that serves nothing needs only its own. + if mb, what := (&serverInit{noNode: true}).servingMemoryMB(false); mb != 0 || what != "" { + t.Errorf("no node: %d %q", mb, what) + } + if mb, _ := (&serverInit{noModel: true, noGuardModel: true}).servingMemoryMB(false); mb != 0 { + t.Errorf("nothing served: %d", mb) + } +} diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index cbf85f5..510b4a4 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -512,9 +512,11 @@ func (m *setupModel) startCheck() tea.Cmd { } } m.conns = nil + serves, what := m.in.servingMemoryMB(m.judges > 0) machines := []preflight.Machine{{ Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{}, - Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness}, + Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness}, + ServesMB: serves, Serves: what, }} for i, d := range m.db[:m.shown()] { roles, judge := m.dbRoles(), i >= 2 diff --git a/internal/preflight/preflight.go b/internal/preflight/preflight.go index 7646491..4002944 100644 --- a/internal/preflight/preflight.go +++ b/internal/preflight/preflight.go @@ -53,6 +53,11 @@ type Machine struct { Remote onboard.Remote // Local is this machine, where sudo is not needed (setup runs as root). Local bool + // ServesMB is memory it needs beyond its roles' for the models it will + // serve itself, and Serves names them. Short of it is a failure, not a + // warning: the install would finish and then have nowhere to run them. + ServesMB int + Serves string } // Level is how bad a finding is. @@ -269,7 +274,17 @@ func Evaluate(m Machine, facts map[string]string, at time.Time) []Finding { } } room("cpus", cpus, need.CPUs, "CPUs") - room("memory", mem, need.MemoryMB, "MB") + if m.ServesMB > 0 { + want := need.MemoryMB + m.ServesMB + d := fmt.Sprintf("%d MB (needs %d with %s)", mem, want, m.Serves) + if mem >= want { + f("memory", OK, d, "") + } else { + f("memory", Fail, d, "give it more memory, or turn off "+m.Serves+" or This machine as a node") + } + } else { + room("memory", mem, need.MemoryMB, "MB") + } room("disk", disk, need.DiskGB, "GB free") // Time. Leases and etcd's elections go wrong on clocks that disagree. diff --git a/internal/preflight/preflight_test.go b/internal/preflight/preflight_test.go index e95bc22..784c406 100644 --- a/internal/preflight/preflight_test.go +++ b/internal/preflight/preflight_test.go @@ -107,3 +107,16 @@ func TestAgainstRealMachines(t *testing.T) { } } } + +// The guide's first recording: 8 GB, a control plane serving the starter and +// both judges, every step done and then no room for the starter. Now said at +// the checks, as a failure, before anything is installed. +func TestAMachineTooSmallForTheModelsItServesFailsTheCheck(t *testing.T) { + m := Machine{Name: "llm-1", Roles: []Role{RoleControl, RoleWitness}, ServesMB: 7 * 1024, Serves: "the starter model and 2 judges"} + if l, d := levelOf(Evaluate(m, facts(map[string]string{"mem_mb": "7941"}), time.Now()), "memory"); l != Fail || !strings.Contains(d, "2 judges") { + t.Errorf("8 GB: %v %q", l, d) + } + if l, d := levelOf(Evaluate(m, facts(map[string]string{"mem_mb": "11961"}), time.Now()), "memory"); l != OK { + t.Errorf("12 GB: %v %q", l, d) + } +} diff --git a/internal/web/locales/en/install.json b/internal/web/locales/en/install.json index ba91672..dc45c33 100644 --- a/internal/web/locales/en/install.json +++ b/internal/web/locales/en/install.json @@ -37,7 +37,7 @@ "install.req.names.text": "A hostname of its own on each machine. It becomes the machine's name in the cluster.", "install.req.gpus": "GPU nodes", "install.req.gpus.text": "Added later, with the NVIDIA driver installed. llmgw installs llama.cpp, vLLM, Python and CUDA itself.", - "install.req.memory_note": "Memory, when the first machine also serves models. In this recording it runs the starter model and both guardrail judges. With 8 GB there was no room left for the starter model; with 12 GB everything fits.", + "install.req.memory_note": "Memory, when the first machine also serves models. In this recording it runs the starter model and both guardrail judges, and needs 12 GB; the setup checks this before it installs anything, and says what to turn off on a smaller machine.", "install.binary.title": "Get llmgw", "install.binary.intro": "One static file, the same on every machine. Only the first needs it: setup copies it to the others.", "install.binary.text": "The latest release, its checksum checked. sha256sum must say OK; if it does not, the download is damaged.", diff --git a/internal/web/locales/tr/install.json b/internal/web/locales/tr/install.json index 22cff1d..14342ea 100644 --- a/internal/web/locales/tr/install.json +++ b/internal/web/locales/tr/install.json @@ -37,7 +37,7 @@ "install.req.names.text": "Her makinenin kendine ait bir adı (hostname). Kümede makinenin adı bu olur.", "install.req.gpus": "GPU düğümleri", "install.req.gpus.text": "Sonradan, NVIDIA sürücüsü kurulu olarak eklenir. llama.cpp, vLLM, Python ve CUDA'yı llmgw kendisi kurar.", - "install.req.memory_note": "İlk makine model de çalıştırıyorsa bellek. Bu kayıtta ilk makine başlangıç modelini ve iki koruma hakemini birlikte çalıştırıyor. 8 GB ile başlangıç modeline yer kalmadı; 12 GB ile hepsi sığıyor.", + "install.req.memory_note": "İlk makine model de çalıştıracaksa bellek. Bu kayıtta ilk makine başlangıç modelini ve iki koruma hakemini birlikte çalıştırıyor ve 12 GB istiyor; kurulum bunu bir şey kurmadan önce denetler ve daha küçük bir makinede neyin kapatılacağını söyler.", "install.binary.title": "llmgw'yi edinin", "install.binary.intro": "Tek bir statik dosya, her makinede aynı. Yalnızca ilk makinede gerekir: kurulum onu diğerlerine kendisi kopyalar.", "install.binary.text": "Son sürüm, sağlama toplamı denetlenerek. sha256sum OK demelidir; demiyorsa indirme bozuktur.",