diff --git a/docs/install.md b/docs/install.md index 9050a69b..234275b9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1277,10 +1277,28 @@ made by ComfyUI: it fills a ComfyUI workflow with the request, queues it, waits, and sends the pictures back as `b64_json`. **On the cluster's own cards.** Use engine `comfyui`. The model is placed on a -GPU node like any other, started when asked for, and stopped when unused. The -node must offer the engine. For now ComfyUI is installed by hand: a directory -with `main.py` and its environment in `.venv`, readable by the user engines run -as (`llmgw-engine`). Name it in that node's `agent.yaml`: +GPU node like any other, started when asked for, and stopped when unused. + +The node must offer the engine. Ask for it in the cluster's settings, and every +node with an NVIDIA GPU installs it: + +``` +llmgw ctl config set engines.comfyui 0.39.2 +``` + +The agent fetches ComfyUI's source from GitHub and checks it against the +digest llmgw holds for that version. GitHub publishes none for a tag's archive, +so a version llmgw does not know is refused. The agent then builds a Python +environment beside the source, under `/opt/llmgw/engines/comfyui/`. Its +dependencies are resolved as PyPI stood the day after the release. torch comes +from PyPI (CUDA 13), or from PyTorch's CUDA 12.9 build where the driver is older +than 580. The install is checked on the node's GPU, and takes about 10 GB. +`engine_overrides` can keep it off nodes that should not have it. + +Alternatively, install ComfyUI by hand. You need a directory with `main.py` and +its environment in `.venv`, readable by the user engines run as +(`llmgw-engine`). Name it in that node's `agent.yaml`; a node with one named +there is not given the installed one: ```yaml engines: diff --git a/internal/agent/engines/comfyui.go b/internal/agent/engines/comfyui.go index c0b04b75..e1597833 100644 --- a/internal/agent/engines/comfyui.go +++ b/internal/agent/engines/comfyui.go @@ -42,24 +42,28 @@ func (*ComfyUI) MetricsPath() string { return "" } // ServesHTTP is true: a ComfyUI replica is one process on one node. func (*ComfyUI) ServesHTTP(ShardSpec) bool { return true } -// UseInstalled points the engine at a ComfyUI directory the agent installed. -func (c *ComfyUI) UseInstalled(dir string) { c.installed.Store(&dir) } +// UseInstalled points the engine at the environment the agent installed: +// its interpreter, with ComfyUI's code beside its bin, in comfyui/. +func (c *ComfyUI) UseInstalled(python string) { c.installed.Store(&python) } // Configured reports whether the configuration names a ComfyUI. func (c *ComfyUI) Configured() bool { return c.cfg.Dir != "" } func (c *ComfyUI) dir() string { if p := c.installed.Load(); p != nil { - return *p + return filepath.Join(filepath.Dir(filepath.Dir(*p)), "comfyui") } return c.cfg.Dir } func (c *ComfyUI) python() string { - if c.cfg.Python != "" && c.installed.Load() == nil { + if p := c.installed.Load(); p != nil { + return *p + } + if c.cfg.Python != "" { return c.cfg.Python } - return filepath.Join(c.dir(), ".venv", "bin", "python") + return filepath.Join(c.cfg.Dir, ".venv", "bin", "python") } var comfyVersion = regexp.MustCompile(`__version__\s*=\s*"([^"]+)"`) diff --git a/internal/agent/engines/comfyui_test.go b/internal/agent/engines/comfyui_test.go index eb541436..ea92818b 100644 --- a/internal/agent/engines/comfyui_test.go +++ b/internal/agent/engines/comfyui_test.go @@ -64,3 +64,17 @@ func TestComfyUIIsOfferedByTheVersionItsBuildWrote(t *testing.T) { t.Error("a profile may install ComfyUI nodes") } } + +func TestAnInstalledComfyUIIsFoundBesideItsInterpreter(t *testing.T) { + c := &ComfyUI{cfg: ComfyUIConfig{Dir: "/opt/by-hand"}} + c.UseInstalled("/opt/llmgw/engines/comfyui/0.39.2-cu13/bin/python") + cmds, err := c.Commands(ShardSpec{ReplicaID: "r", HTTPPort: 1, Profile: Profile{ModelPath: "/w", ServedModelName: "m"}}) + if err != nil { + t.Fatal(err) + } + args := strings.Join(cmds[0].Args, " ") + if !strings.Contains(args, "--python /opt/llmgw/engines/comfyui/0.39.2-cu13/bin/python") || + !strings.Contains(args, "--comfyui /opt/llmgw/engines/comfyui/0.39.2-cu13/comfyui") { + t.Errorf("the installed ComfyUI is not what runs: %s", args) + } +} diff --git a/internal/agent/provision/comfyui_test.go b/internal/agent/provision/comfyui_test.go new file mode 100644 index 00000000..4dab2434 --- /dev/null +++ b/internal/agent/provision/comfyui_test.go @@ -0,0 +1,132 @@ +package provision + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// comfyuiSource is a ComfyUI release archive as GitHub makes one, and the +// recipe pointed at it. +func comfyuiSource(t *testing.T, in *Installer, rel *release, digest string) { + t.Helper() + archive := tarGz(t, map[string]string{ + "ComfyUI-0.39.2/main.py": "print('comfy')\n", + "ComfyUI-0.39.2/comfyui_version.py": "__version__ = \"0.39.2\"\n", + "ComfyUI-0.39.2/requirements.txt": "comfyui-frontend-package==1.53.10\ntorch\ntorchvision\n" + + "numpy>=1.25.0\n\n#non essential dependencies:\nkornia>=0.7.1\n", + }, nil) + rel.assets["v0.39.2.tar.gz"] = archive + sum := sha256.Sum256(archive) + if digest == "" { + digest = hex.EncodeToString(sum[:]) + } + old := comfyuiReleases + comfyuiReleases = map[string]struct { + sha256 string + published string + }{"0.39.2": {digest, "2026-10-07T21:58:07Z"}} + t.Cleanup(func() { comfyuiReleases = old }) +} + +var turingNode = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64", + NVIDIADriver: "580.82.07", MinComputeCapability: 7.5} + +func TestComfyUIIsInstalledFromItsSourceCheckedAgainstTheRecipe(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + in, rel := pythonInstaller(t, turingNode, sys) + comfyuiSource(t, in, rel, "") + + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}) + if got[0].State != Ready { + t.Fatalf("not ready: %+v", got[0]) + } + dir := filepath.Join(in.Root, "comfyui", "0.39.2-cu13") + if got[0].Binary != filepath.Join(dir, "bin", "python") { + t.Errorf("binary %s, want the environment's interpreter", got[0].Binary) + } + if _, err := os.Stat(filepath.Join(dir, "comfyui", "main.py")); err != nil { + t.Errorf("ComfyUI's code is not beside the interpreter: %v", err) + } + all := strings.Join(sys.commands, "\n") + for _, want := range []string{ + // Not PyPI's date, which ComfyUI has none of: the recipe's, a day on. + "--exclude-newer 2026-10-08T21:58:07Z", + "comfyui-frontend-package==1.53.10", "numpy>=1.25.0", "kornia>=0.7.1", "torch", + "--require-hashes", + } { + if !strings.Contains(all, want) { + t.Errorf("no %q in:\n%s", want, all) + } + } + if strings.Contains(all, "/pypi/") { + t.Error("PyPI was asked when ComfyUI was published") + } +} + +func TestComfyUIOnAnOlderDriverTakesPyTorchsCUDA12Torch(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + host := turingNode + host.NVIDIADriver = "550.163.01" + in, rel := pythonInstaller(t, host, sys) + comfyuiSource(t, in, rel, "") + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}) + if got[0].State != Ready || got[0].Variant != "cu129" { + t.Fatalf("got %+v", got[0]) + } + var project string + for _, c := range sys.commands { + if strings.HasPrefix(c, "pyproject: ") { + project = c + } + } + // torch from PyTorch's index alone, named once, as the build names it. + if !strings.Contains(project, `torch = { index = "extra" }`) || strings.Count(project, `"torch"`) != 1 { + t.Errorf("torch is not from PyTorch's index alone:\n%s", project) + } +} + +func TestComfyUIWhoseSourceIsNotTheOneTheRecipeNamesIsNotInstalled(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + in, rel := pythonInstaller(t, turingNode, sys) + comfyuiSource(t, in, rel, strings.Repeat("ab", 32)) + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "does not match") { + t.Fatalf("got %s %q", got[0].State, got[0].Detail) + } + for _, c := range sys.commands { + if strings.Contains(c, "pip install") { + t.Errorf("installed from a source that did not match: %s", c) + } + } +} + +func TestComfyUIIsInstalledOnlyWhereItCanRun(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + in, _ := pythonInstaller(t, ubuntu, sys) + if got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}); got[0].State != Skipped { + t.Errorf("a node with no GPU: %+v", got[0]) + } + in, _ = pythonInstaller(t, turingNode, sys) + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.1.0"}}) + if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "knows the source digest of ComfyUI 0.39.2") { + t.Errorf("a version without a digest: %s %q", got[0].State, got[0].Detail) + } +} + +func TestARequirementsFileIsReadForPackagesAndNothingElse(t *testing.T) { + got, err := requirementsIn("torch\ntorchvision>=0.20\nPillow\n# a comment\naiohttp>=3.11 # inline\n", pytorchPackages) + if err != nil || strings.Join(got, ",") != "Pillow,aiohttp>=3.11" { + t.Errorf("%v %v", got, err) + } + for _, bad := range []string{"-r other.txt", "--index-url https://evil", "x @ https://evil/x.whl", "-e ."} { + if _, err := requirementsIn(bad+"\n", nil); err == nil { + t.Errorf("%q was accepted", bad) + } + } +} diff --git a/internal/agent/provision/python.go b/internal/agent/provision/python.go index 7eb848d9..0477f9d0 100644 --- a/internal/agent/provision/python.go +++ b/internal/agent/provision/python.go @@ -66,12 +66,17 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil return fmt.Errorf("installing uv, which installs %s: %s", w.Engine, uv.Detail) } - in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version) - published, err := in.published(ctx, r.PyPI, w.Version) - if err != nil { - return err + var asOf string + if !b.AsOf.IsZero() { + asOf = b.AsOf.UTC().Format(time.RFC3339) + } else { + in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version) + published, err := in.published(ctx, r.PyPI, w.Version) + if err != nil { + return err + } + asOf = published.Add(24 * time.Hour).UTC().Format(time.RFC3339) } - asOf := published.Add(24 * time.Hour).UTC().Format(time.RFC3339) cache := filepath.Join(in.root(), ".uv-cache") env := []string{ @@ -96,7 +101,14 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil } python := filepath.Join(tmp, "bin", "python") - reqs := b.Requirements + reqs := slices.Clone(b.Requirements) + if b.Source != nil { + more, err := in.unpackSource(ctx, w, b, tmp, st) + if err != nil { + return err + } + reqs = append(reqs, more...) + } if b.Wheel != nil { // The engine from its GitHub release, pinned to the digest GitHub // publishes for the file: the same trust as an archive, and uv @@ -320,3 +332,68 @@ func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Buil } return nil } + +// unpackSource fetches an engine's source archive, checks it against the +// digest the recipe holds, puts its code in the environment, and returns the +// requirements its requirements file names — less the packages a second +// index gives, which the build names itself. +func (in *Installer) unpackSource(ctx context.Context, w Want, b Build, env string, st *Status) ([]string, error) { + src := b.Source + url := src.URL + if strings.HasPrefix(url, "/") { + url = strings.TrimSuffix(in.downloadBase(), "/") + url + } + scratch := env + ".source" + _ = os.RemoveAll(scratch) + if err := os.MkdirAll(scratch, 0o755); err != nil { + return nil, err + } + defer os.RemoveAll(scratch) //nolint:errcheck // scratch + archive := filepath.Join(scratch, "source.tar.gz") + name := w.Engine + " " + w.Version + "'s source" + if err := in.download(ctx, url, name, archive, src.SHA256, st); err != nil { + return nil, err + } + in.progress(st, "unpacking %s", name) + if err := untarGz(archive, scratch); err != nil { + return nil, fmt.Errorf("unpacking %s: %w", name, err) + } + dst := filepath.Join(env, src.Dir) + if err := os.Rename(filepath.Join(scratch, src.Top), dst); err != nil { + return nil, fmt.Errorf("%s has no %s at its top: %w", name, src.Top, err) + } + if src.Requirements == "" { + return nil, nil + } + f, err := os.ReadFile(filepath.Join(dst, src.Requirements)) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + return requirementsIn(string(f), b.IndexPackages) +} + +// requirementsIn reads a requirements file's lines. Options (-r, -e, --index-url +// and the like) are refused: they would fetch from somewhere other than PyPI, +// or install what the lock never saw. +func requirementsIn(text string, skip []string) ([]string, error) { + var out []string + for _, line := range strings.Split(text, "\n") { + line, _, _ = strings.Cut(line, "#") + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "-") || strings.Contains(line, "://") || strings.Contains(line, " @ ") { + return nil, fmt.Errorf("the requirements name %q, which is not a package from PyPI", line) + } + name := strings.ToLower(line) + if i := strings.IndexAny(name, "<>=!~;[ "); i >= 0 { + name = name[:i] + } + if slices.Contains(skip, name) { + continue + } + out = append(out, line) + } + return out, nil +} diff --git a/internal/agent/provision/python_test.go b/internal/agent/provision/python_test.go index 9de3edf9..032b0ec8 100644 --- a/internal/agent/provision/python_test.go +++ b/internal/agent/provision/python_test.go @@ -53,6 +53,7 @@ func (f *fakePython) run(ctx context.Context, env []string, name string, args .. return []byte("error: Hash mismatch for torch\n"), errExit } venv := filepath.Dir(filepath.Dir(args[slices.Index(args, "--python")+1])) + _ = os.WriteFile(filepath.Join(venv, "bin", "python"), []byte("#!/bin/sh\n"), 0o755) return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755) case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version": return []byte("0.30.0\n"), nil diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index a816e708..c0c50924 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -13,6 +13,7 @@ import ( "fmt" "slices" "strings" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) @@ -68,12 +69,31 @@ type Build struct { // Note says why this build, when it is not the plain one: shown beside // it wherever the installation is. Note string + // Source, for a Python engine that is not on PyPI, is its own code: an + // archive pinned by its digest, unpacked into the environment, whose + // requirements file is installed with Requirements. AsOf is then the + // date its dependencies are resolved as of — the day after its version + // was published, which PyPI cannot be asked for. + Source *Source + AsOf time.Time // FreeGB is the disk space the installation needs before it starts: a // Python engine is several gigabytes, and one that fills the disk halfway // leaves the node worse off than one that never began. FreeGB int } +// Source is an engine's code as an archive. +type Source struct { + // URL is fetched as is; SHA256 is what it must hash to, written into + // the recipe because a project's source archives come with no published + // digest of their own. + URL, SHA256 string + // Top is the archive's one top directory, put in the environment as Dir. + Top, Dir string + // Requirements is the requirements file, in Dir. + Requirements string +} + // Wheel is a package published as a GitHub release asset: the asset whose // name starts with Prefix and ends with Suffix. type Wheel struct { @@ -104,6 +124,7 @@ var Recipes = map[string]Recipe{ "llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates}, "vllm": {PyPI: "vllm", Candidates: vllmCandidates}, "sglang": {PyPI: "sglang", Candidates: sglangCandidates}, + "comfyui": {Candidates: comfyuiCandidates}, "uv": {Repo: "astral-sh/uv", Candidates: uvCandidates}, } @@ -556,3 +577,78 @@ func orUnknown(s string) string { } return s } + +// --- ComfyUI --- + +// comfyuiReleases are the ComfyUI versions this agent installs: each source +// archive's digest, taken when it was added here and confirmed from a second +// machine, and when the version was published. GitHub publishes no digest +// for a tag's archive, so a version not listed is refused rather than +// installed unchecked. +var comfyuiReleases = map[string]struct { + sha256 string + published string +}{ + "0.39.2": {"1d655e22cf546700fc1d6d2536e27701695c5832a51f63bb16c41866ac09be0b", "2026-10-07T21:58:07Z"}, +} + +// ComfyUIRepo is where ComfyUI's source is published. +const ComfyUIRepo = "Comfy-Org/ComfyUI" + +// comfyuiDir is where ComfyUI's code is put in its environment: main.py is +// /comfyui/main.py, beside bin/python. +const comfyuiDir = "comfyui" + +// comfyuiCandidates are the ComfyUI builds for a machine: its source, in a +// Python environment with the torch for the node's driver — PyPI's, built +// for CUDA 13, or PyTorch's CUDA 12.9 build for a driver CUDA 13 does not +// run on. +func comfyuiCandidates(version, variant string, h discovery.Host) ([]Build, error) { + if _, err := gpuArch("ComfyUI", h); err != nil { + return nil, err + } + rel, ok := comfyuiReleases[version] + if !ok { + known := make([]string, 0, len(comfyuiReleases)) + for v := range comfyuiReleases { + known = append(known, v) + } + slices.Sort(known) + return nil, fmt.Errorf("this agent knows the source digest of ComfyUI %s, not of %s; "+ + "install that one by hand and name it in agent.yaml (engines.comfyui.dir)", + strings.Join(known, ", "), version) + } + published, err := time.Parse(time.RFC3339, rel.published) + if err != nil { + return nil, err + } + build := func(variant string) Build { + return Build{ + Variant: variant, + Python: pythonForEngines, + Source: &Source{ + URL: "/" + ComfyUIRepo + "/archive/refs/tags/v" + version + ".tar.gz", + SHA256: rel.sha256, Top: "ComfyUI-" + version, Dir: comfyuiDir, + Requirements: "requirements.txt", + }, + AsOf: published.Add(24 * time.Hour), + // ComfyUI is started as a script, so what the agent is given is + // the environment's interpreter; the proof is importing what it + // imports first, from where its code is. + Binary: "python", + VersionArgs: []string{"-c", `import os, sys, torch, aiohttp, safetensors +d = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(sys.executable))), "` + comfyuiDir + `") +assert os.path.isfile(os.path.join(d, "main.py")), d + " has no main.py" +print("comfyui", "` + version + `", "torch", torch.__version__)`}, + Packages: compilerPackages, + FreeGB: 15, + GPUCheck: true, + } + } + cu129 := build("cu129") + cu129.ExtraIndex, cu129.IndexPackages = pytorchCU129, pytorchPackages + return pick("ComfyUI "+version, variant, []candidate{ + {build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, + {cu129, gpuNeeds{minDriverForCUDA12, 7.0}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + }, h) +}