Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
754ab256a2 | ||
|
|
6765d430d8 | ||
|
|
09ebe77513 | ||
|
|
0ffd633bf6 | ||
|
|
32d4d9c3af |
No files matched your search
@@ -9944,6 +9944,12 @@
|
|||||||
"created_by": {
|
"created_by": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"project": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"publish": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"artifacts": {
|
"artifacts": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -9989,18 +9995,26 @@
|
|||||||
},
|
},
|
||||||
"base_repo": {
|
"base_repo": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "BaseRepo and BaseRevision are the weights trained on: the model's own on its registry, in safetensors (Hugging Face), not a GGUF."
|
"description": "BaseRepo and BaseRevision are the weights trained on: the model's own on its registry, in safetensors (Hugging Face), not a GGUF. Empty finds them from where the model's came: the repository itself, or the one its card says it was made from."
|
||||||
},
|
},
|
||||||
"base_revision": {
|
"base_revision": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"node": {
|
"node": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Node trains it; GPUMemoryMiB is what it needs of a card, which can be what an engine there leaves free."
|
"description": "Node trains it; empty is the first with a card that has enough free. GPUMemoryMiB is what it needs of a card, which can be what an engine there leaves free; 0 is worked out from the weights' size."
|
||||||
},
|
},
|
||||||
"gpu_memory_mib": {
|
"gpu_memory_mib": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"project": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Project is the project the job is started in: its members see and manage it, and only one of its jobs trains at a time."
|
||||||
|
},
|
||||||
|
"publish": {
|
||||||
|
"type": "boolean",
|
||||||
|
"description": "Publish puts what the job makes on the mirror and makes it the model's adapter by itself when the job is done, restarting a llama.cpp model's replicas to load it."
|
||||||
|
},
|
||||||
"owner": {
|
"owner": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Owner is who the adapter will belong to: org:, team:, project: (by path or ID), user: or key:. Only callers there see it. When the pairs come from one knowledge base, it defaults to that collection's owner; it must be able to read every collection they come from."
|
"description": "Owner is who the adapter will belong to: org:, team:, project: (by path or ID), user: or key:. Only callers there see it. When the pairs come from one knowledge base, it defaults to that collection's owner; it must be able to read every collection they come from."
|
||||||
|
|||||||
+2
-2
@@ -787,7 +787,7 @@ Nothing to do: the agent installs them once the node joins, under
|
|||||||
|
|
||||||
| Engine | On | How |
|
| Engine | On | How |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
|
| llama.cpp | every node | the release build for its processor and GPU — CUDA 13 (13.3, 13.4 or 13.5, whichever the build was published for), CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
|
||||||
| vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
|
| vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
|
||||||
| SGLang | only when you ask for it | the same way as vLLM |
|
| SGLang | only when you ask for it | the same way as vLLM |
|
||||||
| TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) |
|
| TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) |
|
||||||
@@ -3312,7 +3312,7 @@ That carries:
|
|||||||
`--version` for another), with `checksums.txt` and its signature, checked
|
`--version` for another), with `checksums.txt` and its signature, checked
|
||||||
against the project's signing key as `ctl upgrade` checks it.
|
against the project's signing key as `ctl upgrade` checks it.
|
||||||
- **llama.cpp** at the version this llmgw installs, every build for each
|
- **llama.cpp** at the version this llmgw installs, every build for each
|
||||||
architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked
|
architecture — CPU, CUDA 13 (cuda-13.3 up to b11070, cuda-13.4 to b12169, cuda-13.5 after), CUDA 12.8 (amd64), Vulkan — each checked
|
||||||
against the checksum GitHub publishes, the same check a node does online.
|
against the checksum GitHub publishes, the same check a node does online.
|
||||||
`--variant cpu --variant cuda-13.3` carries only those, for a site whose
|
`--variant cpu --variant cuda-13.3` carries only those, for a site whose
|
||||||
hardware you know; `--engine llamacpp=b10991` names the version.
|
hardware you know; `--engine llamacpp=b10991` names the version.
|
||||||
|
|||||||
@@ -529,6 +529,7 @@ func (a *Agent) scan(ctx context.Context) error {
|
|||||||
inv, err := discovery.Discover(ctx, opts, a.log)
|
inv, err := discovery.Discover(ctx, opts, a.log)
|
||||||
host := discovery.DiscoverHost()
|
host := discovery.DiscoverHost()
|
||||||
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
|
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
|
||||||
|
host.DriverCUDA = discovery.DriverCUDA(inv.GPUs)
|
||||||
bw := a.memBandwidth()
|
bw := a.memBandwidth()
|
||||||
host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core
|
host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|||||||
@@ -65,6 +65,11 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
|
|||||||
return cards, err
|
return cards, err
|
||||||
}
|
}
|
||||||
readPCIe(ctx, binary, cards)
|
readPCIe(ctx, binary, cards)
|
||||||
|
if cuda := driverCUDAFromSMI(ctx, binary); cuda != "" {
|
||||||
|
for i := range cards {
|
||||||
|
cards[i].CUDAVersion = cuda
|
||||||
|
}
|
||||||
|
}
|
||||||
// The query lists whole cards, MIG mode or not. A card in MIG mode is
|
// The query lists whole cards, MIG mode or not. A card in MIG mode is
|
||||||
// replaced by its instances, which only -L names and only the full
|
// replaced by its instances, which only -L names and only the full
|
||||||
// report sizes; asked only when some card could be in MIG mode, since
|
// report sizes; asked only when some card could be in MIG mode, since
|
||||||
@@ -173,3 +178,40 @@ func assessHealth(g *GPU) {
|
|||||||
g.Health.Problems = append(g.Health.Problems, p)
|
g.Health.Problems = append(g.Health.Problems, p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// driverCUDAFromSMI is the newest CUDA the driver runs, from `nvidia-smi
|
||||||
|
// --version`: "CUDA version : 12.4" on older drivers, "CUDA UMD version :
|
||||||
|
// 13.4" on newer ones, where the old line says "Deprecated". Empty when
|
||||||
|
// neither says; the query fields have no such column.
|
||||||
|
func driverCUDAFromSMI(ctx context.Context, binary string) string {
|
||||||
|
out, err := exec.CommandContext(ctx, binary, "--version").Output()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return parseSMIDriverCUDA(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSMIDriverCUDA(out string) string {
|
||||||
|
found := ""
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
key, val, ok := strings.Cut(line, ":")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, val = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(val)
|
||||||
|
if !strings.HasPrefix(key, "cuda") || !strings.HasSuffix(key, "version") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
major, minor, ok := strings.Cut(val, ".")
|
||||||
|
if _, err := strconv.Atoi(major); err != nil || !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := strconv.Atoi(minor); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.Contains(key, "umd") || found == "" {
|
||||||
|
found = val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
@@ -177,3 +177,18 @@ func TestUtilisationIsReadWhenPresent(t *testing.T) {
|
|||||||
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) {
|
||||||
|
for out, want := range map[string]string{
|
||||||
|
// cuda01, driver 615.
|
||||||
|
"NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" +
|
||||||
|
"CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4",
|
||||||
|
// An older driver.
|
||||||
|
"NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4",
|
||||||
|
"nothing useful": "",
|
||||||
|
} {
|
||||||
|
if got := parseSMIDriverCUDA(out); got != want {
|
||||||
|
t.Errorf("%q: %q, want %q", out, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,6 +36,11 @@ type Host struct {
|
|||||||
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
|
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
|
||||||
// is not known. Not read here but from the GPU inventory, by the agent.
|
// is not known. Not read here but from the GPU inventory, by the agent.
|
||||||
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
|
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
|
||||||
|
// DriverCUDA is the newest CUDA the loaded driver runs ("13.4"), empty
|
||||||
|
// when not known. Not read here but from the GPU inventory, by the agent.
|
||||||
|
// A build carrying a GPU's code as PTX needs a driver at least this new
|
||||||
|
// to compile it.
|
||||||
|
DriverCUDA string `json:"driver_cuda,omitempty"`
|
||||||
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
|
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
|
||||||
// measured: what decides a model's speed on the CPU. Not read here but
|
// measured: what decides a model's speed on the CPU. Not read here but
|
||||||
// measured by the agent when it starts (MeasureMemoryBandwidth).
|
// measured by the agent when it starts (MeasureMemoryBandwidth).
|
||||||
@@ -209,6 +214,17 @@ func nvidiaDriverFrom(path string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DriverCUDA is the newest CUDA the GPUs' driver runs, as NVML reports it,
|
||||||
|
// empty when none says.
|
||||||
|
func DriverCUDA(gpus []GPU) string {
|
||||||
|
for _, g := range gpus {
|
||||||
|
if v := strings.TrimSpace(g.CUDAVersion); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// MinComputeCapability is the oldest GPU's compute capability, 0 when none
|
// MinComputeCapability is the oldest GPU's compute capability, 0 when none
|
||||||
// is known: an engine built for newer GPUs is refused by the oldest.
|
// is known: an engine built for newer GPUs is refused by the oldest.
|
||||||
func MinComputeCapability(gpus []GPU) float64 {
|
func MinComputeCapability(gpus []GPU) float64 {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
in.progress(st, "unpacking %s", asset)
|
in.progress(st, "unpacking %s", asset)
|
||||||
if err := untarGz(archive, tmp); err != nil {
|
into := tmp
|
||||||
|
if slices.Contains(b.Beside, asset) {
|
||||||
|
into = filepath.Join(tmp, ".beside", asset)
|
||||||
|
if err := os.MkdirAll(into, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := untarGz(archive, into); err != nil {
|
||||||
return fmt.Errorf("unpacking %s: %w", asset, err)
|
return fmt.Errorf("unpacking %s: %w", asset, err)
|
||||||
}
|
}
|
||||||
_ = os.Remove(archive)
|
_ = os.Remove(archive)
|
||||||
got[asset] = want
|
got[asset] = want
|
||||||
}
|
}
|
||||||
|
if len(b.Beside) > 0 {
|
||||||
|
if err := moveBesideBinary(tmp, b.Binary); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
in.progress(st, "checking that %s runs", b.Binary)
|
in.progress(st, "checking that %s runs", b.Binary)
|
||||||
bin, _, err := in.verify(ctx, tmp, b)
|
bin, _, err := in.verify(ctx, tmp, b)
|
||||||
@@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// moveBesideBinary moves every file the Beside assets unpacked, under
|
||||||
|
// dir/.beside, into the directory the binary is in, by its own name: the
|
||||||
|
// directories the archive put them in are its packaging, not where they are
|
||||||
|
// loaded from. A file the build already has is kept.
|
||||||
|
func moveBesideBinary(dir, binary string) error {
|
||||||
|
staged := filepath.Join(dir, ".beside")
|
||||||
|
defer os.RemoveAll(staged) //nolint:errcheck // scratch
|
||||||
|
bin, err := findFile(dir, binary)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
into := filepath.Dir(bin)
|
||||||
|
return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
to := filepath.Join(into, d.Name())
|
||||||
|
if _, err := os.Lstat(to); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return os.Rename(p, to)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// verify finds the binary and runs it.
|
// verify finds the binary and runs it.
|
||||||
//
|
//
|
||||||
// Two checks, because they fail differently. ldd names a missing library
|
// Two checks, because they fail differently. ldd names a missing library
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -288,6 +289,21 @@ func TestTheBuildFollowsTheHardware(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Found on cuda01 (Turing, driver 615, which runs CUDA 13.4): b12606's
|
||||||
|
// CUDA 13.5 build listed the GPU, then failed its first kernel with "the
|
||||||
|
// provided PTX was compiled with an unsupported toolchain".
|
||||||
|
turing := discovery.Host{Arch: "amd64", NVIDIADriver: "615.71.09", MinComputeCapability: 7.5}
|
||||||
|
for _, c := range []struct {
|
||||||
|
driverCUDA string
|
||||||
|
want string
|
||||||
|
}{{"13.4", "cuda-12.8"}, {"13.5", "cuda-13.5"}, {"14.0", "cuda-13.5"}, {"", "cuda-13.5"}} {
|
||||||
|
turing.DriverCUDA = c.driverCUDA
|
||||||
|
got, err := llamaCppCandidates("b12606", "", turing)
|
||||||
|
if err != nil || got[0].Variant != c.want {
|
||||||
|
t.Errorf("driver for CUDA %q: %+v, %v; want %s first", c.driverCUDA, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
slow := ubuntu
|
slow := ubuntu
|
||||||
slow.CPUFlags = nil
|
slow.CPUFlags = nil
|
||||||
st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
|
st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
|
||||||
@@ -429,7 +445,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
|
|||||||
rel := newRelease(map[string][]byte{
|
rel := newRelease(map[string][]byte{
|
||||||
"llama-b1-" + cuda: tarGz(t, map[string]string{
|
"llama-b1-" + cuda: tarGz(t, map[string]string{
|
||||||
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
|
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
|
||||||
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
|
// As the release packs it: under a directory of its own.
|
||||||
|
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil),
|
||||||
})
|
})
|
||||||
gpu := ubuntu
|
gpu := ubuntu
|
||||||
gpu.NVIDIADriver = "570.86.15"
|
gpu.NVIDIADriver = "570.86.15"
|
||||||
@@ -446,7 +463,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
|
|||||||
|
|
||||||
// The same build seeing its GPU is installed.
|
// The same build seeing its GPU is installed.
|
||||||
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
||||||
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
|
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
||||||
|
if got[0].State != Ready {
|
||||||
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
|
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
|
||||||
}
|
}
|
||||||
|
// The runtime is beside the backend, where its $ORIGIN runpath finds it,
|
||||||
|
// not in the directory its archive gave it.
|
||||||
|
if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil {
|
||||||
|
t.Errorf("the CUDA runtime is not beside llama-server: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil {
|
||||||
|
t.Error("the staging directory was left behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) {
|
||||||
|
for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3",
|
||||||
|
"b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} {
|
||||||
|
if got := llamaCUDA13(v); got != want {
|
||||||
|
t.Errorf("%s: CUDA %s, want %s", v, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bs, err := Builds("llamacpp", "b12606", "amd64")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") {
|
||||||
|
t.Errorf("b12606's CUDA 13 build: %+v", bs[1])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -12,6 +12,7 @@ package provision
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -32,6 +33,11 @@ type Build struct {
|
|||||||
// Binary is the executable's file name, found anywhere in the unpacked
|
// Binary is the executable's file name, found anywhere in the unpacked
|
||||||
// tree.
|
// tree.
|
||||||
Binary string
|
Binary string
|
||||||
|
// Beside are assets whose files belong in the directory Binary is in,
|
||||||
|
// wherever the archive puts them: llama.cpp's CUDA runtime, which its
|
||||||
|
// archive holds under a directory of its own, while the backend looks for
|
||||||
|
// it only beside itself ($ORIGIN).
|
||||||
|
Beside []string
|
||||||
// CPUFlags are extensions the build is slow without. Missing ones are
|
// CPUFlags are extensions the build is slow without. Missing ones are
|
||||||
// reported, not refused: slow is still running.
|
// reported, not refused: slow is still running.
|
||||||
CPUFlags []string
|
CPUFlags []string
|
||||||
@@ -251,6 +257,11 @@ const (
|
|||||||
type gpuNeeds struct {
|
type gpuNeeds struct {
|
||||||
driver int
|
driver int
|
||||||
cc float64
|
cc float64
|
||||||
|
// cuda, when set, is the CUDA the driver must run: a build that carries
|
||||||
|
// a GPU's code as PTX has the driver compile it, and a driver older than
|
||||||
|
// the toolkit refuses ("the provided PTX was compiled with an unsupported
|
||||||
|
// toolchain") at the first kernel — after the build has listed the GPU.
|
||||||
|
cuda string
|
||||||
}
|
}
|
||||||
|
|
||||||
// fits says why a machine cannot run a build, or nil.
|
// fits says why a machine cannot run a build, or nil.
|
||||||
@@ -258,12 +269,26 @@ func (n gpuNeeds) fits(what string, h discovery.Host) error {
|
|||||||
if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
|
if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
|
||||||
return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
|
return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
|
||||||
}
|
}
|
||||||
|
if n.cuda != "" && h.DriverCUDA != "" && cudaBefore(h.DriverCUDA, n.cuda) {
|
||||||
|
return fmt.Errorf("%s needs a driver for CUDA %s or newer; this node's driver %s runs CUDA %s",
|
||||||
|
what, n.cuda, h.NVIDIADriver, h.DriverCUDA)
|
||||||
|
}
|
||||||
if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
|
if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
|
||||||
return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
|
return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cudaBefore reports whether CUDA version a ("13.4") is older than b.
|
||||||
|
func cudaBefore(a, b string) bool {
|
||||||
|
am, an, _ := strings.Cut(a, ".")
|
||||||
|
bm, bn, _ := strings.Cut(b, ".")
|
||||||
|
if x, y := leadingInt(am), leadingInt(bm); x != y {
|
||||||
|
return x < y
|
||||||
|
}
|
||||||
|
return leadingInt(an) < leadingInt(bn)
|
||||||
|
}
|
||||||
|
|
||||||
// candidate is a build and what it needs.
|
// candidate is a build and what it needs.
|
||||||
type candidate struct {
|
type candidate struct {
|
||||||
b Build
|
b Build
|
||||||
@@ -400,7 +425,7 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
|
|||||||
variant = "cu12"
|
variant = "cu12"
|
||||||
}
|
}
|
||||||
return pick("vLLM "+v, variant, []candidate{
|
return pick("vLLM "+v, variant, []candidate{
|
||||||
{pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v},
|
{pypi("cu12"), gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v},
|
||||||
}, h)
|
}, h)
|
||||||
}
|
}
|
||||||
cu129 := pypi("cu129")
|
cu129 := pypi("cu129")
|
||||||
@@ -414,8 +439,8 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
|
|||||||
cu129.ExtraIndex = pytorchCU129
|
cu129.ExtraIndex = pytorchCU129
|
||||||
cu129.IndexPackages = pytorchPackages
|
cu129.IndexPackages = pytorchPackages
|
||||||
return pick("vLLM "+v, variant, []candidate{
|
return pick("vLLM "+v, variant, []candidate{
|
||||||
{pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"},
|
{pypi("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: minCC}, "vLLM " + v + " built for CUDA 13"},
|
||||||
{cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"},
|
{cu129, gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v + " built for CUDA 12.9"},
|
||||||
}, h)
|
}, h)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -454,9 +479,9 @@ func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error
|
|||||||
FreeGB: 25,
|
FreeGB: 25,
|
||||||
GPUCheck: true,
|
GPUCheck: true,
|
||||||
}
|
}
|
||||||
needs := gpuNeeds{minDriverForCUDA13, 7.5}
|
needs := gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}
|
||||||
if versionBefore(version, sglangCUDA13Since) {
|
if versionBefore(version, sglangCUDA13Since) {
|
||||||
b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0}
|
b.Variant, needs = "cu12", gpuNeeds{driver: minDriverForCUDA12, cc: 7.0}
|
||||||
}
|
}
|
||||||
return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
|
return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
|
||||||
}
|
}
|
||||||
@@ -509,9 +534,25 @@ func leadingInt(s string) int {
|
|||||||
// compile.
|
// compile.
|
||||||
var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
|
var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
|
||||||
|
|
||||||
|
// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the
|
||||||
|
// archive's name carries it, and it moves on with the project's CI. Taken
|
||||||
|
// from the releases themselves (the first build of each).
|
||||||
|
func llamaCUDA13(version string) string {
|
||||||
|
n, err := strconv.Atoi(strings.TrimPrefix(version, "b"))
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return "13.3"
|
||||||
|
case n >= 12170:
|
||||||
|
return "13.5"
|
||||||
|
case n >= 11071:
|
||||||
|
return "13.4"
|
||||||
|
}
|
||||||
|
return "13.3"
|
||||||
|
}
|
||||||
|
|
||||||
// llamaCppCandidates are the llama.cpp builds for a machine.
|
// llamaCppCandidates are the llama.cpp builds for a machine.
|
||||||
//
|
//
|
||||||
// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580,
|
// On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580,
|
||||||
// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
|
// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
|
||||||
// the GPUs it ships as PTX, 525 for those it ships as machine code), and
|
// the GPUs it ships as PTX, 525 for those it ships as machine code), and
|
||||||
// Vulkan, which runs on any card whose driver has Vulkan — the generations
|
// Vulkan, which runs on any card whose driver has Vulkan — the generations
|
||||||
@@ -560,8 +601,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
|
|||||||
cuda := func(cv string) Build {
|
cuda := func(cv string) Build {
|
||||||
name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
|
name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
|
||||||
// The release carries the CUDA runtime beside the build, so the node
|
// The release carries the CUDA runtime beside the build, so the node
|
||||||
// needs a driver and nothing else — no toolkit.
|
// needs a driver and nothing else — no toolkit. Its archive unpacks
|
||||||
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name))
|
// into a directory of its own, where the backend never looks; Beside
|
||||||
|
// moves it next to the backend.
|
||||||
|
runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name)
|
||||||
|
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime)
|
||||||
|
b.Beside = []string{runtime}
|
||||||
b.GPUBackend = "libggml-cuda.so"
|
b.GPUBackend = "libggml-cuda.so"
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
@@ -573,9 +618,10 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
|
|||||||
if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
|
if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
|
||||||
cuda12Driver = minDriverForCUDA12
|
cuda12Driver = minDriverForCUDA12
|
||||||
}
|
}
|
||||||
cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}}
|
cv := llamaCUDA13(version)
|
||||||
|
cands := []candidate{{cuda(cv), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5, cuda: cv}, "llama.cpp built for CUDA " + cv}}
|
||||||
if arch == "x64" {
|
if arch == "x64" {
|
||||||
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
|
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{driver: cuda12Driver, cc: 5.0}, "llama.cpp built for CUDA 12.8"})
|
||||||
}
|
}
|
||||||
cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
|
cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
|
||||||
return pick("llama.cpp", variant, cands, h)
|
return pick("llama.cpp", variant, cands, h)
|
||||||
@@ -677,9 +723,9 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`},
|
|||||||
cu126 := build("cu126")
|
cu126 := build("cu126")
|
||||||
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
|
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
|
||||||
return pick("ComfyUI "+version, variant, []candidate{
|
return pick("ComfyUI "+version, variant, []candidate{
|
||||||
{build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
{build("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
||||||
{cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
{cu129, gpuNeeds{driver: minDriverForCUDA12, cc: 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||||
{cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
{cu126, gpuNeeds{driver: minDriverForCUDA12, cc: 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
||||||
}, h)
|
}, h)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -753,10 +799,10 @@ print("strata version", "` + version + `", "engine", "` + rel.engine + `")`},
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return pick("Strata "+version, variant, []candidate{
|
return pick("Strata "+version, variant, []candidate{
|
||||||
{build("cuda13"), gpuNeeds{minDriverForCUDA13, 7.5}, "Strata " + version + " built for CUDA 13"},
|
{build("cuda13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "Strata " + version + " built for CUDA 13"},
|
||||||
// 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its
|
// 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its
|
||||||
// PTX was compiled by 12.9, so the driver must be 575.
|
// PTX was compiled by 12.9, so the driver must be 575.
|
||||||
{build("cuda12"), gpuNeeds{575, 6.0}, "Strata " + version + " built for CUDA 12.9"},
|
{build("cuda12"), gpuNeeds{driver: 575, cc: 6.0}, "Strata " + version + " built for CUDA 12.9"},
|
||||||
}, h)
|
}, h)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -809,6 +855,6 @@ func freetokenCandidates(version, variant string, h discovery.Host) ([]Build, er
|
|||||||
GPUCheck: true,
|
GPUCheck: true,
|
||||||
}
|
}
|
||||||
return pick("FreeToken "+version, variant, []candidate{
|
return pick("FreeToken "+version, variant, []candidate{
|
||||||
{b, gpuNeeds{minDriverForCUDA13, freetokenMinCompute}, "FreeToken " + version},
|
{b, gpuNeeds{driver: minDriverForCUDA13, cc: freetokenMinCompute}, "FreeToken " + version},
|
||||||
}, h)
|
}, h)
|
||||||
}
|
}
|
||||||
@@ -131,7 +131,7 @@ func newBundleCreateCmd() *cobra.Command {
|
|||||||
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
|
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
|
||||||
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
|
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
|
||||||
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
|
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
|
||||||
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)")
|
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3 (cuda-13.4 from llama.cpp b11071, cuda-13.5 from b12170), cuda-12.8, vulkan; repeatable (default: every build)")
|
||||||
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
|
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
|
||||||
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
|
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
|
||||||
f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face")
|
f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face")
|
||||||
|
|||||||
@@ -679,7 +679,7 @@ func newServerCmd() *cobra.Command {
|
|||||||
if adminAPI != nil && catalogSvc != nil {
|
if adminAPI != nil && catalogSvc != nil {
|
||||||
serve := modelserve.New(modelserve.Deps{
|
serve := modelserve.New(modelserve.Deps{
|
||||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
LlamaVersion: cfg.WantedEngines()["llamacpp"], LlamaOverrides: cfg.OverrideVersions("llamacpp"), Log: log,
|
||||||
// Updates: the catalog in force, the service that
|
// Updates: the catalog in force, the service that
|
||||||
// changes it, and what replaces replicas.
|
// changes it, and what replaces replicas.
|
||||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||||
|
|||||||
@@ -1164,6 +1164,18 @@ func (s Server) EnginesFor(node string, classes []string, labels map[string]stri
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OverrideVersions are the versions of an engine that engine_overrides give
|
||||||
|
// some nodes, in the order they are set.
|
||||||
|
func (s Server) OverrideVersions(engine string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, o := range s.EngineOverrides {
|
||||||
|
if v := o.Engines[engine]; v != "" && !slices.Contains(out, v) {
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// WantedEngines is DefaultEngines with the configuration laid over it.
|
// WantedEngines is DefaultEngines with the configuration laid over it.
|
||||||
func (s Server) WantedEngines() map[string]string {
|
func (s Server) WantedEngines() map[string]string {
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -54,3 +55,17 @@ func TestOIDCSettingsAreCheckedBeforeAProviderIsAsked(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOverrideVersionsAreTheEnginesNodesAreGivenInstead(t *testing.T) {
|
||||||
|
s := Server{EngineOverrides: []EngineOverride{
|
||||||
|
{Nodes: []string{"cuda01"}, Engines: map[string]string{"strata": "0.2.7", "llamacpp": "b12606"}},
|
||||||
|
{GPUClass: "v100-32g", Engines: map[string]string{"llamacpp": ""}},
|
||||||
|
{Labels: map[string]string{"site": "b"}, Engines: map[string]string{"llamacpp": "b12606"}},
|
||||||
|
}}
|
||||||
|
if got := s.OverrideVersions("llamacpp"); !slices.Equal(got, []string{"b12606"}) {
|
||||||
|
t.Errorf("llamacpp: %v", got)
|
||||||
|
}
|
||||||
|
if got := s.OverrideVersions("vllm"); got != nil {
|
||||||
|
t.Errorf("vllm: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -68,8 +68,10 @@ type Deps struct {
|
|||||||
Catalog Catalog
|
Catalog Catalog
|
||||||
Fleet Fleet
|
Fleet Fleet
|
||||||
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
||||||
// which architectures load.
|
// which architectures load. LlamaOverrides are the builds some nodes are
|
||||||
|
// given instead (engine_overrides): a file one of them loads is servable.
|
||||||
LlamaVersion string
|
LlamaVersion string
|
||||||
|
LlamaOverrides []string
|
||||||
HTTP *http.Client
|
HTTP *http.Client
|
||||||
Log *slog.Logger
|
Log *slog.Logger
|
||||||
// Poll is how often the job looks at the mirror and the fleet.
|
// Poll is how often the job looks at the mirror and the fleet.
|
||||||
@@ -240,6 +242,18 @@ type Server struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// New builds a server.
|
// New builds a server.
|
||||||
|
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
|
||||||
|
// first.
|
||||||
|
func (s *Server) llamaVersions() []string {
|
||||||
|
out := []string{s.deps.LlamaVersion}
|
||||||
|
for _, v := range s.deps.LlamaOverrides {
|
||||||
|
if !slices.Contains(out, v) {
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func New(d Deps) *Server {
|
func New(d Deps) *Server {
|
||||||
if d.Poll <= 0 {
|
if d.Poll <= 0 {
|
||||||
d.Poll = 2 * time.Second
|
d.Poll = 2 * time.Second
|
||||||
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
|
|||||||
s.step(j, "check", Running, "reading its header from the mirror")
|
s.step(j, "check", Running, "reading its header from the mirror")
|
||||||
var header gguf.Header
|
var header gguf.Header
|
||||||
if u := headerURL(source, set); u != "" {
|
if u := headerURL(source, set); u != "" {
|
||||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||||
if !v.Servable {
|
if !v.Servable {
|
||||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
header = v.Header
|
header = v.Header
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
|||||||
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
|
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
|
||||||
base = b.Base()
|
base = b.Base()
|
||||||
}
|
}
|
||||||
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
|
v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
|
||||||
if v.Checked && !v.Servable {
|
if v.Checked && !v.Servable {
|
||||||
p.Servable, p.Reason = false, v.Reason
|
p.Servable, p.Reason = false, v.Reason
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
|
|||||||
s.step(j, "check", Running, "reading its header from the mirror")
|
s.step(j, "check", Running, "reading its header from the mirror")
|
||||||
detail := "no header to read"
|
detail := "no header to read"
|
||||||
if u := headerURL(source, set); u != "" {
|
if u := headerURL(source, set); u != "" {
|
||||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||||
if !v.Servable {
|
if !v.Servable {
|
||||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
|
||||||
return prepared{}, false
|
return prepared{}, false
|
||||||
}
|
}
|
||||||
detail = orUnknown(v.Header.Architecture)
|
detail = orUnknown(v.Header.Architecture)
|
||||||
|
|||||||
@@ -222,6 +222,7 @@ func Collect(ctx context.Context, o Options) Facts {
|
|||||||
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
|
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
|
||||||
}
|
}
|
||||||
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
|
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
|
||||||
|
f.Host.DriverCUDA = discovery.DriverCUDA(f.GPUs)
|
||||||
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
|
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
|
||||||
if f.EnginesDir == "" {
|
if f.EnginesDir == "" {
|
||||||
f.EnginesDir = provision.DefaultRoot
|
f.EnginesDir = provision.DefaultRoot
|
||||||
|
|||||||
@@ -404,6 +404,32 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
|||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CheckAny is Check against several llama.cpp builds, the header read once:
|
||||||
|
// servable when any of them loads it. A cluster whose GPU nodes are given a
|
||||||
|
// newer build than the rest (engine_overrides) can serve an architecture
|
||||||
|
// only that build knows. The first version is the cluster's own; its verdict
|
||||||
|
// is the one given when none loads it.
|
||||||
|
func CheckAny(ctx context.Context, url string, versions []string) Verdict {
|
||||||
|
if len(versions) == 0 {
|
||||||
|
versions = []string{""}
|
||||||
|
}
|
||||||
|
v := Check(ctx, url, versions[0])
|
||||||
|
if v.Servable || v.Header.Architecture == "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
for _, other := range versions[1:] {
|
||||||
|
if o := LlamaCpp(other, v.Header.Architecture); o.Servable {
|
||||||
|
o.Header = v.Header
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(versions) > 1 {
|
||||||
|
v.Reason = "its architecture is \"" + v.Header.Architecture + "\", which no llama.cpp the nodes install (" +
|
||||||
|
strings.Join(versions, ", ") + ") can load"
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
// CardUsable is the share of a card's memory a model may take: the rest is
|
// CardUsable is the share of a card's memory a model may take: the rest is
|
||||||
// CUDA's own and the engine's compute buffers.
|
// CUDA's own and the engine's compute buffers.
|
||||||
const CardUsable = 0.9
|
const CardUsable = 0.9
|
||||||
|
|||||||
@@ -221,3 +221,25 @@ func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) {
|
|||||||
t.Error("a dense model reads all its weights")
|
t.Error("a dense model reads all its weights")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnArchitectureOnlyAnOverriddenBuildKnowsIsServable(t *testing.T) {
|
||||||
|
// Found on the PoC: Inkling 2 Mini (inkling2_loop) refused by `ctl model
|
||||||
|
// serve` for the cluster's b10991, though the GPU node had been given
|
||||||
|
// b12606 through engine_overrides.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(header("inkling2_loop")))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
url := srv.URL + "/m.gguf"
|
||||||
|
if v := CheckAny(context.Background(), url, []string{"b10991"}); v.Servable {
|
||||||
|
t.Errorf("b10991 alone: %+v", v)
|
||||||
|
}
|
||||||
|
if v := CheckAny(context.Background(), url, []string{"b10991", "b12606"}); !v.Servable ||
|
||||||
|
v.Header.Architecture != "inkling2_loop" {
|
||||||
|
t.Errorf("with b12606 on some nodes: %+v", v)
|
||||||
|
}
|
||||||
|
if v := CheckAny(context.Background(), url, []string{"b10991", "b10991"}); v.Servable ||
|
||||||
|
!strings.Contains(v.Reason, "no llama.cpp the nodes install (b10991, b10991)") {
|
||||||
|
t.Errorf("refused by every build: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user