diff --git a/CHANGELOG.md b/CHANGELOG.md index 6481394..546b495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,9 @@ SGLang profiles, whose adapters are loaded differently. adapters, offers the held adapter files of its architecture to attach under a name and scale, and removes them. A file that is not an adapter, or is for another architecture, is refused before it reaches the catalog. +- **`llmgw ctl model adapters MODEL [--add NAME=URL[@SCALE]] [--remove NAME]`** + lists and changes them from the command line, through + `POST /api/v1/models/{name}/adapters`, with the same checks. ### Models served outside the cluster diff --git a/docs/install.md b/docs/install.md index 0989b0a..1ef91d6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -694,6 +694,12 @@ follow key allowlists on their own: a key allowed `coder-sql` reaches that adapter and not the model under it, while a key allowed the model reaches its adapters too. Adding or removing an adapter is a change to the model; running replicas load the new set when restarted (`llmgw ctl restart --model M`). +Besides the mirror page's model card, adapters are attached from the command +line, which reads the file first and refuses one that is not an adapter for +this model's architecture: + + llmgw ctl model adapters coder --add coder-sql=http://mirror:8081/sql-lora/v1/sql.gguf@0.8 + llmgw ctl model adapters coder --remove coder-sql The adapter file must be a GGUF LoRA made for this model — llama.cpp's `convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with diff --git a/internal/cli/model.go b/internal/cli/model.go index 3a7654b..f60b854 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -5,6 +5,9 @@ import ( "fmt" "os" "os/user" + "slices" + "strconv" + "strings" "time" "github.com/knadh/koanf/parsers/yaml" @@ -31,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), - newTargetCmd(client, asJSON), + newTargetCmd(client, asJSON), newAdaptersCmd(client), newModelServeCmd(client)) return c } @@ -264,3 +267,69 @@ func newClassCmd(client func() *ctlClient) *cobra.Command { }, } } + +func newAdaptersCmd(client func() *ctlClient) *cobra.Command { + var add, remove []string + c := &cobra.Command{ + Use: "adapters [--add NAME=URL[@SCALE]] [--remove NAME]", + Short: "List, add or remove a model's LoRA adapters", + Long: "An adapter is asked for by its NAME as though it were a model, and is served\n" + + "by this model's replicas with it applied. URL is the adapter file, usually in\n" + + "the mirror; it must be a GGUF LoRA adapter for this model's architecture,\n" + + "which is checked before it is attached. Replicas load a changed set when\n" + + "restarted: llmgw ctl restart --model MODEL.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + var models []adminapi.ModelView + if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil { + return err + } + var current []catalog.Adapter + found := false + for _, m := range models { + if m.Name == name { + current, found = m.Adapters, true + } + } + if !found { + return fmt.Errorf("no model %q", name) + } + out := cmd.OutOrStdout() + if len(add) == 0 && len(remove) == 0 { + if len(current) == 0 { + fmt.Fprintf(out, "%s has no adapters\n", name) + } + for _, a := range current { + fmt.Fprintf(out, "%s\tscale %g\t%s\n", a.Name, a.EffectiveScale(), a.Source) + } + return nil + } + next := slices.DeleteFunc(slices.Clone(current), func(a catalog.Adapter) bool { return slices.Contains(remove, a.Name) }) + for _, spec := range add { + n, rest, ok := strings.Cut(spec, "=") + if !ok || n == "" || rest == "" { + return fmt.Errorf("--add %q is not NAME=URL[@SCALE]", spec) + } + ad := catalog.Adapter{Name: n, Source: rest} + if i := strings.LastIndex(rest, "@"); i > 0 { + if s, err := strconv.ParseFloat(rest[i+1:], 64); err == nil { + ad.Source, ad.Scale = rest[:i], s + } + } + next = append(next, ad) + } + body, _ := json.Marshal(adminapi.AdaptersRequest{Adapters: next}) + var res adminapi.AdaptersRequest + if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/adapters", body, &res); err != nil { + return err + } + fmt.Fprintf(out, "%s serves %d adapter(s); restart its replicas to load them: llmgw ctl restart --model %s\n", + name, len(res.Adapters), name) + return nil + }, + } + c.Flags().StringArrayVar(&add, "add", nil, "NAME=URL[@SCALE]; repeatable") + c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable") + return c +} diff --git a/internal/control/adminapi/adapters_test.go b/internal/control/adminapi/adapters_test.go new file mode 100644 index 0000000..00dfb1d --- /dev/null +++ b/internal/control/adminapi/adapters_test.go @@ -0,0 +1,71 @@ +package adminapi + +import ( + "bytes" + "encoding/binary" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +// ggufStart is the start of a GGUF file with these string keys. +func ggufStart(kv ...string) []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(len(kv) / 2)) + for i := 0; i < len(kv); i += 2 { + str(kv[i]) + w(uint32(8)) // string + str(kv[i+1]) + } + b.Write(make([]byte, 4096)) + return b.Bytes() +} + +func TestOnlyAnAdapterForTheModelsArchitectureIsAttached(t *testing.T) { + files := map[string][]byte{ + "/base.gguf": ggufStart("general.architecture", "qwen2"), + "/model.gguf": ggufStart("general.architecture", "qwen2", "general.type", "model"), + "/llama.gguf": ggufStart("general.architecture", "llama", "general.type", "adapter"), + "/lora.gguf": ggufStart("general.architecture", "qwen2", "general.type", "adapter"), + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(files[r.URL.Path]) + })) + defer srv.Close() + + models := []catalog.Model{{ + Name: "coder", Source: srv.URL + "/base.gguf", + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}, + }} + for _, tc := range []struct { + file, want string + code int + }{ + {"/model.gguf", "not a LoRA adapter", http.StatusBadRequest}, + {"/llama.gguf", "for llama models", http.StatusBadRequest}, + {"/lora.gguf", "", http.StatusOK}, + } { + a, _ := setup(t, models) + fm := &fakeModels{} + a.SetModels(fm) + mux := http.NewServeMux() + a.Register(mux) + req := httptest.NewRequest(http.MethodPost, "/api/v1/models/coder/adapters", + strings.NewReader(`{"adapters":[{"name":"fix","source":"`+srv.URL+tc.file+`"}]}`)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if rec.Code != tc.code || !strings.Contains(rec.Body.String(), tc.want) { + t.Errorf("%s: %d %s; want %d mentioning %q", tc.file, rec.Code, rec.Body.String(), tc.code, tc.want) + } + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 56d2bf0..d47b51f 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -20,6 +20,7 @@ import ( "unicode" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" @@ -62,6 +63,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources) a.registerQuotas(mux) + mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters) mux.HandleFunc("GET /api/v1/models/{name}/target", a.target) mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) { if a.models == nil { @@ -318,9 +320,11 @@ type ModelView struct { Owner string `json:"owner,omitempty"` // External is where a model served outside the cluster goes, provider // host and its name for the model; empty for one served here. - External string `json:"external,omitempty"` - Preempted int `json:"preempted,omitempty"` - HeldFor string `json:"held_for,omitempty"` + External string `json:"external,omitempty"` + // Adapters are the LoRA adapters it serves. + Adapters []catalog.Adapter `json:"adapters,omitempty"` + Preempted int `json:"preempted,omitempty"` + HeldFor string `json:"held_for,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -574,7 +578,7 @@ func (a *API) Models() []ModelView { Desired: h.Desired, Failures: h.Failures, Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable, Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor, - External: externalLabel(m), + External: externalLabel(m), Adapters: m.Adapters, } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) @@ -657,6 +661,7 @@ type Models interface { Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) + SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) } // ResourcesRequest sets what a model's profiles reserve of a node. @@ -1461,3 +1466,58 @@ func externalLabel(m catalog.Model) string { } return host + " · " + m.External.Model } + +// AdaptersRequest replaces a model's LoRA adapters with these. +type AdaptersRequest struct { + Adapters []catalog.Adapter `json:"adapters"` +} + +// setAdapters replaces a model's adapters, after reading each new file to +// check it is an adapter for the model's architecture: one that is not is a +// replica that fails to start, found minutes later on a node. +func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) { + if a.models == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req AdaptersRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + name := r.PathValue("name") + m, ok := a.ctl.Catalog().Resolve(name) + if !ok { + writeError(w, http.StatusBadRequest, "no model "+name) + return + } + base, baseErr := gguf.KindFromURL(r.Context(), nil, m.Source) + have := map[string]bool{} + for _, ad := range m.Adapters { + have[ad.Source] = true + } + for _, ad := range req.Adapters { + if have[ad.Source] { + continue // already checked when it was attached + } + h, err := gguf.KindFromURL(r.Context(), nil, ad.Source) + switch { + case err != nil: + writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %v", ad.Name, err)) + return + case !h.IsAdapter(): + writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %s is a model, not a LoRA adapter", ad.Name, ad.Source)) + return + case baseErr == nil && h.Architecture != base.Architecture: + writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s is for %s models and %s is %s", + ad.Name, h.Architecture, m.Name, base.Architecture)) + return + } + } + out, err := a.models.SetAdapters(r.Context(), m.Name, req.Adapters, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters}) +} diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 3d465d8..1fde64a 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -258,6 +258,10 @@ type fakeModels struct { err error } +func (f *fakeModels) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) { + return catalog.Model{Name: name, Adapters: as}, f.err +} + func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) { return catalog.Model{Name: name, Owner: owner}, f.err }