A picture or 3D model is added by its name: llmgw ctl model add triposr
flux-schnell and triposr come with llmgw as presets: their workflow, custom nodes, packages and licence as they were found and tried on the PoC, and a profile that says the card it needs rather than a class, so the entry places on any site's cards. TripoSR's workflow is built in, with the LLMGWPrepareForMesh node that keeps a picture's background out of the mesh, which the copy in the document had left out. `ctl model add PRESET` adds one, and has the nodes install ComfyUI when the cluster's settings name none; `ctl model presets` lists them; the console's add-model dialog offers them beside a definition file. A ComfyUI build is now also checked to keep its order on a Turing card, so a node's installed cu13 is not replaced by the cu126 build. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
52898fbb80
commit
40b8948d7d
12 files changed
+537
-36
No files matched your search
+52
-22
@@ -32,7 +32,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -46,34 +46,42 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
var path string
|
||||
var skipCheck bool
|
||||
c := &cobra.Command{
|
||||
Use: "add -f <file.yaml>",
|
||||
Short: "Add a model from a definition file",
|
||||
Long: "The definition is the same YAML a model takes in server.yaml, so an\n" +
|
||||
Use: "add PRESET | -f <file.yaml>",
|
||||
Short: "Add a model that comes with llmgw, or one from a definition file",
|
||||
Long: "A preset is a model that comes with llmgw, its workflow, nodes and licence\n" +
|
||||
"already found and tried: " + strings.Join(catalog.PresetNames(), ", ") + " (llmgw ctl model presets).\n" +
|
||||
"Adding a ComfyUI one also has the nodes install ComfyUI, when the cluster\n" +
|
||||
"has not been told otherwise.\n\n" +
|
||||
"A definition is the same YAML a model takes in server.yaml, so an\n" +
|
||||
"entry can be moved between the two by copying it.\n\n" +
|
||||
"The catalog is rebuilt and validated before anything is written, so a\n" +
|
||||
"definition that collides with an existing name, or names a fallback that\n" +
|
||||
"does not exist, is refused rather than stored and then ignored on every\n" +
|
||||
"start. Once accepted it takes effect on the reconciler's next pass — no\n" +
|
||||
"restart — and survives one.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if path == "" {
|
||||
return fmt.Errorf("--file is required: the model definition, as YAML")
|
||||
}
|
||||
// Read by koanf, the same parser and the same tags that read
|
||||
// server.yaml — so an entry can be moved between the two by copying
|
||||
// it, and a field that works in one cannot silently mean nothing in
|
||||
// the other.
|
||||
k := koanf.New(".")
|
||||
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
Example: " llmgw ctl model add triposr\n" +
|
||||
" llmgw ctl model add -f qwen.yaml",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var m catalog.Model
|
||||
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if m.Name == "" {
|
||||
return fmt.Errorf("%s: the definition has no name", path)
|
||||
switch {
|
||||
case len(args) == 1 && path != "":
|
||||
return fmt.Errorf("a preset or a file, not both")
|
||||
case len(args) == 1:
|
||||
p, ok := catalog.PresetNamed(args[0])
|
||||
if !ok {
|
||||
return fmt.Errorf("no preset %q: there are %s; any other model is added from a file (-f)",
|
||||
args[0], strings.Join(catalog.PresetNames(), ", "))
|
||||
}
|
||||
m = p.Model
|
||||
case path == "":
|
||||
return fmt.Errorf("a preset (%s) or --file, the model definition as YAML",
|
||||
strings.Join(catalog.PresetNames(), ", "))
|
||||
default:
|
||||
var err error
|
||||
if m, err = readModelFile(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Before the catalog is touched, because a source that does not
|
||||
@@ -107,6 +115,9 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
fmt.Fprintf(cmd.OutOrStdout(),
|
||||
"%s added. It will be placed on the next reconcile pass; watch it with:\n"+
|
||||
" llmgw ctl models\n", m.Name)
|
||||
if m.ServesComfyUI() && !m.IsExternal() {
|
||||
return ensureComfyUI(cmd, client, m.Name)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -116,6 +127,25 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
return c
|
||||
}
|
||||
|
||||
// readModelFile reads a model's definition. By koanf, the same parser and
|
||||
// the same tags that read server.yaml — so an entry can be moved between the
|
||||
// two by copying it, and a field that works in one cannot silently mean
|
||||
// nothing in the other.
|
||||
func readModelFile(path string) (catalog.Model, error) {
|
||||
var m catalog.Model
|
||||
k := koanf.New(".")
|
||||
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
||||
return m, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
||||
return m, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if m.Name == "" {
|
||||
return m, fmt.Errorf("%s: the definition has no name", path)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "remove <name>",
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newModelPresetsCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "presets",
|
||||
Short: "The models that come with llmgw, added by name",
|
||||
Long: "Each is a catalog entry whose workflow, custom nodes, packages and licence\n" +
|
||||
"were found and tried, and whose profile says the card it needs rather than\n" +
|
||||
"a class, so it places on any site's cards. Add one with\n" +
|
||||
"`llmgw ctl model add NAME`.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
w := tabTo(cmd.OutOrStdout())
|
||||
fmt.Fprintln(w, "NAME\tMAKES\tCARD\tLICENCE\tWHAT IT IS FOR")
|
||||
for _, p := range catalog.Presets() {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", p.Model.Name, p.Makes, presetCard(p.Model),
|
||||
presetLicence(p.Model), p.Model.Description)
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func presetCard(m catalog.Model) string {
|
||||
if len(m.Profiles) == 0 || m.Profiles[0].GPU == nil {
|
||||
return "—"
|
||||
}
|
||||
return m.Profiles[0].GPU.String()
|
||||
}
|
||||
|
||||
func presetLicence(m catalog.Model) string {
|
||||
if m.License == nil {
|
||||
return "—"
|
||||
}
|
||||
l := m.License.Name
|
||||
if c := m.License.Commercial; c != nil && !*c {
|
||||
l += ", not commercial"
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// ensureComfyUI has the nodes install ComfyUI for a model that needs it,
|
||||
// unless the cluster names a version already, or none on purpose.
|
||||
func ensureComfyUI(cmd *cobra.Command, client func() *ctlClient, model string) error {
|
||||
out := cmd.OutOrStdout()
|
||||
path := []string{"engines", catalog.EngineComfyUI}
|
||||
var cur adminapi.SettingsView
|
||||
if err := client().get(cmd.Context(), "/api/v1/settings", &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
doc, err := settings.FromBytes("settings", []byte(cur.Body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
version := provision.ComfyUIVersion()
|
||||
if n := doc.Node(path); n != nil {
|
||||
if n.Value == "" {
|
||||
fmt.Fprintf(out, "engines.comfyui is set to none, so the nodes install no ComfyUI for it: set a version "+
|
||||
"(llmgw ctl config set engines.comfyui %s) or name one installed by hand in agent.yaml\n", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "Nodes with an NVIDIA GPU install ComfyUI %s for it, about 10 GB each:\n", version)
|
||||
if err := doc.SetYAML(path, version); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := doc.Bytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return putSettings(cmd, client, cur, b, "ComfyUI for "+model)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
// presetAdmin is an admin API holding settings, which records the models
|
||||
// added and the settings put.
|
||||
type presetAdmin struct {
|
||||
mu sync.Mutex
|
||||
settings string
|
||||
added []catalog.Model
|
||||
put []string
|
||||
}
|
||||
|
||||
func (a *presetAdmin) client(t *testing.T) func() *ctlClient {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models":
|
||||
var m catalog.Model
|
||||
_ = json.Unmarshal(body, &m)
|
||||
a.added = append(a.added, m)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"name": m.Name})
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 1, Body: a.settings})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
|
||||
var req adminapi.SettingsRequest
|
||||
_ = json.Unmarshal(body, &req)
|
||||
a.put = append(a.put, req.Body)
|
||||
a.settings = req.Body
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 2, Body: req.Body})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return func() *ctlClient { return newCtlClient(strings.TrimPrefix(srv.URL, "http://"), "") }
|
||||
}
|
||||
|
||||
func runModelAdd(t *testing.T, client func() *ctlClient, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := newModelAddCmd(client)
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs(append(args, "--no-source-check"))
|
||||
err := cmd.ExecuteContext(t.Context())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestAPresetIsAddedByNameAndHasTheNodesInstallComfyUI(t *testing.T) {
|
||||
a := &presetAdmin{settings: "router:\n retries: 2\n"}
|
||||
out, err := runModelAdd(t, a.client(t), "triposr")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if len(a.added) != 1 || a.added[0].Name != "triposr" || a.added[0].ComfyUI.Workflow != "triposr" {
|
||||
t.Fatalf("added %+v", a.added)
|
||||
}
|
||||
if len(a.put) != 1 || !strings.Contains(a.put[0], "comfyui: 0.39.2") || !strings.Contains(a.put[0], "retries: 2") {
|
||||
t.Fatalf("settings put %q", a.put)
|
||||
}
|
||||
if !strings.Contains(out, "install ComfyUI 0.39.2") {
|
||||
t.Errorf("not said:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAClusterThatNamesItsComfyUIKeepsIt(t *testing.T) {
|
||||
for _, settings := range []string{"engines:\n comfyui: 0.38.0\n", "engines:\n comfyui: \"\"\n"} {
|
||||
a := &presetAdmin{settings: settings}
|
||||
out, err := runModelAdd(t, a.client(t), "flux-schnell")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if len(a.put) != 0 {
|
||||
t.Errorf("%q: settings changed to %q", settings, a.put)
|
||||
}
|
||||
if none := strings.Contains(settings, `""`); none != strings.Contains(out, "set to none") {
|
||||
t.Errorf("%q: %s", settings, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownPresetNamesTheOnesThereAre(t *testing.T) {
|
||||
a := &presetAdmin{}
|
||||
_, err := runModelAdd(t, a.client(t), "sdxl")
|
||||
if err == nil || !strings.Contains(err.Error(), "flux-schnell, triposr") || len(a.added) != 0 {
|
||||
t.Errorf("%v, added %v", err, a.added)
|
||||
}
|
||||
if _, err := runModelAdd(t, a.client(t), "triposr", "-f", "x.yaml"); err == nil {
|
||||
t.Error("a preset and a file were both taken")
|
||||
}
|
||||
}
|
||||
@@ -1237,7 +1237,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
strings.Join(md.Profiles, ","), upd, warn})
|
||||
}
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave"
|
||||
keys = "↑↓ row · Enter actions · A add a model · ← back · Q leave"
|
||||
}
|
||||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys
|
||||
case cReplicas:
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
@@ -253,19 +254,37 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
// addModelForm adds a model from a definition file on this machine: the
|
||||
// same YAML a model takes in server.yaml.
|
||||
// fromFile is the add-model choice that reads a definition file rather
|
||||
// than naming a preset.
|
||||
const fromFile = "from a file"
|
||||
|
||||
// addModelForm adds a model that comes with llmgw, or one from a definition
|
||||
// file on this machine: the same YAML a model takes in server.yaml.
|
||||
func (m *consoleModel) addModelForm() {
|
||||
server := "--server=" + m.client.base
|
||||
var note strings.Builder
|
||||
note.WriteString("A model that comes with llmgw, tried and with its licence:\n")
|
||||
for _, p := range catalog.Presets() {
|
||||
fmt.Fprintf(¬e, " %s: %s, %s, %s\n", p.Model.Name, p.Makes, presetCard(p.Model), presetLicence(p.Model))
|
||||
}
|
||||
note.WriteString("A ComfyUI one has the nodes install ComfyUI if the cluster has none.\n" +
|
||||
"Or a definition file: the YAML a model takes in server.yaml's models.")
|
||||
m.openForm(&formDialog{title: "Add a model",
|
||||
note: "From a definition file on this machine: the YAML a model takes in\nserver.yaml's models. Serve a model, under Do, needs none.",
|
||||
note: note.String(),
|
||||
fields: []formField{
|
||||
textF("file", "", "like /root/qwen.yaml"),
|
||||
choiceF("model", append(catalog.PresetNames(), fromFile), catalog.PresetNames()[0]),
|
||||
textF("file", "", "for "+fromFile+": like /root/qwen.yaml"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if _, err := os.Stat(v[0]); err != nil {
|
||||
if v[0] != fromFile {
|
||||
return m.runCLI("Add "+v[0], "ctl", server, "model", "add", v[0]), nil
|
||||
}
|
||||
if v[1] == "" {
|
||||
return nil, fmt.Errorf("a file, or a model that comes with llmgw")
|
||||
}
|
||||
if _, err := os.Stat(v[1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.runCLI("Add a model", "ctl", server, "model", "add", "-f", v[0]), nil
|
||||
return m.runCLI("Add a model", "ctl", server, "model", "add", "-f", v[1]), nil
|
||||
}})
|
||||
}
|
||||
Reference in new issue
Block a user