Plan a repository's files against the fleet, and serve the one that suits it the way it fits

The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:03:48 +02:00
1 parent 630d2788c0
commit 54db7aecb7
15 files changed
+1224 -158

No files matched your search

+44 -5
View File
@@ -14,6 +14,8 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -283,7 +285,41 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil {
t.Fatal(err)
}
m := newConsoleModel("127.0.0.1:9090", cfgPath)
// The control plane's plan of the repository's files.
plan := modelserve.Plan{Repo: "bartowski/Qwen2.5-3B-Instruct-GGUF", Revision: "abc", Format: "gguf", Context: 16384,
Servable: true, Artifacts: []modelserve.PlannedArtifact{
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q4_K_M.gguf", Format: "gguf", Bytes: 2 << 30}},
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q8_0.gguf", Format: "gguf", Bytes: 3 << 30}},
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-F16.gguf", Format: "gguf", Bytes: 6 << 30}},
}}
gpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1,
NeedGB: 5, HaveGB: 21.6, Hosts: 1, FreeNow: true}
cpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 5, HaveGB: 27, Hosts: 2, FreeNow: true}
for _, o := range []struct {
art string
o modelplan.Option
}{{"Qwen2.5-3B-Instruct-Q8_0.gguf", gpu}, {"Qwen2.5-3B-Instruct-Q4_K_M.gguf", gpu},
{"Qwen2.5-3B-Instruct-Q8_0.gguf", cpu}, {"Qwen2.5-3B-Instruct-F16.gguf", cpu}} {
o.o.Artifact = o.art
plan.Options = append(plan.Options, o.o)
}
for i := range plan.Artifacts {
for j := range plan.Options {
if plan.Options[j].Artifact == plan.Artifacts[i].Name {
plan.Artifacts[i].Best = &plan.Options[j]
break
}
}
}
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo {
_ = json.NewEncoder(w).Encode(plan)
return
}
_, _ = w.Write([]byte("[]"))
}))
defer admin.Close()
m := newConsoleModel(admin.URL, cfgPath)
m.loaded, m.menu, m.inPanel = true, cServe, true
var tm tea.Model = m
@@ -303,19 +339,22 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
tm = run(tm, cmd)
d := tm.(consoleModel).menuD
if d == nil || len(d.items) != 2 {
if d == nil || len(d.items) != 3 || !strings.Contains(d.items[0].label, "★") ||
!strings.Contains(d.items[0].label, "Q8_0") || !strings.Contains(d.items[0].label, "1 × 6000-24g") ||
!strings.Contains(d.items[2].label, "CPU") {
t.Fatalf("files offered: %+v", d)
}
tm = press(tm, "down", "enter")
tm = press(tm, "enter")
f := tm.(consoleModel).formD
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" {
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" || len(f.fields[4].options) != 2 {
t.Fatalf("form: %+v", f)
}
f.focus = len(f.fields)
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
act := tm.(consoleModel).action
got := strings.Join(act.targets, " ")
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") {
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") ||
!strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") {
t.Fatalf("would run %q %q", act.kind, got)
}
}