Attach LoRA adapters from the command line

llmgw ctl model adapters lists a model's adapters and adds or removes
them through POST /api/v1/models/{name}/adapters, which reads each new
file first and refuses one that is not an adapter for the model's
architecture, as the mirror page's card does.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 07:30:13 +02:00
1 parent 6b1fa65f59
commit 323d172538
6 files changed
+218 -5

No files matched your search

+3
View File
@@ -67,6 +67,9 @@ SGLang profiles, whose adapters are loaded differently.
adapters, offers the held adapter files of its architecture to attach under a
name and scale, and removes them. A file that is not an adapter, or is for
another architecture, is refused before it reaches the catalog.
- **`llmgw ctl model adapters MODEL [--add NAME=URL[@SCALE]] [--remove NAME]`**
lists and changes them from the command line, through
`POST /api/v1/models/{name}/adapters`, with the same checks.
### Models served outside the cluster
+6
View File
@@ -694,6 +694,12 @@ follow key allowlists on their own: a key allowed `coder-sql` reaches that
adapter and not the model under it, while a key allowed the model reaches its
adapters too. Adding or removing an adapter is a change to the model; running
replicas load the new set when restarted (`llmgw ctl restart --model M`).
Besides the mirror page's model card, adapters are attached from the command
line, which reads the file first and refuses one that is not an adapter for
this model's architecture:
llmgw ctl model adapters coder --add coder-sql=http://mirror:8081/sql-lora/v1/sql.gguf@0.8
llmgw ctl model adapters coder --remove coder-sql
The adapter file must be a GGUF LoRA made for this model — llama.cpp's
`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
+70 -1
View File
@@ -5,6 +5,9 @@ import (
"fmt"
"os"
"os/user"
"slices"
"strconv"
"strings"
"time"
"github.com/knadh/koanf/parsers/yaml"
@@ -31,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client))
return c
}
@@ -264,3 +267,69 @@ func newClassCmd(client func() *ctlClient) *cobra.Command {
},
}
}
func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
var add, remove []string
c := &cobra.Command{
Use: "adapters <model> [--add NAME=URL[@SCALE]] [--remove NAME]",
Short: "List, add or remove a model's LoRA adapters",
Long: "An adapter is asked for by its NAME as though it were a model, and is served\n" +
"by this model's replicas with it applied. URL is the adapter file, usually in\n" +
"the mirror; it must be a GGUF LoRA adapter for this model's architecture,\n" +
"which is checked before it is attached. Replicas load a changed set when\n" +
"restarted: llmgw ctl restart --model MODEL.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
name := args[0]
var models []adminapi.ModelView
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
return err
}
var current []catalog.Adapter
found := false
for _, m := range models {
if m.Name == name {
current, found = m.Adapters, true
}
}
if !found {
return fmt.Errorf("no model %q", name)
}
out := cmd.OutOrStdout()
if len(add) == 0 && len(remove) == 0 {
if len(current) == 0 {
fmt.Fprintf(out, "%s has no adapters\n", name)
}
for _, a := range current {
fmt.Fprintf(out, "%s\tscale %g\t%s\n", a.Name, a.EffectiveScale(), a.Source)
}
return nil
}
next := slices.DeleteFunc(slices.Clone(current), func(a catalog.Adapter) bool { return slices.Contains(remove, a.Name) })
for _, spec := range add {
n, rest, ok := strings.Cut(spec, "=")
if !ok || n == "" || rest == "" {
return fmt.Errorf("--add %q is not NAME=URL[@SCALE]", spec)
}
ad := catalog.Adapter{Name: n, Source: rest}
if i := strings.LastIndex(rest, "@"); i > 0 {
if s, err := strconv.ParseFloat(rest[i+1:], 64); err == nil {
ad.Source, ad.Scale = rest[:i], s
}
}
next = append(next, ad)
}
body, _ := json.Marshal(adminapi.AdaptersRequest{Adapters: next})
var res adminapi.AdaptersRequest
if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/adapters", body, &res); err != nil {
return err
}
fmt.Fprintf(out, "%s serves %d adapter(s); restart its replicas to load them: llmgw ctl restart --model %s\n",
name, len(res.Adapters), name)
return nil
},
}
c.Flags().StringArrayVar(&add, "add", nil, "NAME=URL[@SCALE]; repeatable")
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
return c
}
@@ -0,0 +1,71 @@
package adminapi
import (
"bytes"
"encoding/binary"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// ggufStart is the start of a GGUF file with these string keys.
func ggufStart(kv ...string) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(len(kv) / 2))
for i := 0; i < len(kv); i += 2 {
str(kv[i])
w(uint32(8)) // string
str(kv[i+1])
}
b.Write(make([]byte, 4096))
return b.Bytes()
}
func TestOnlyAnAdapterForTheModelsArchitectureIsAttached(t *testing.T) {
files := map[string][]byte{
"/base.gguf": ggufStart("general.architecture", "qwen2"),
"/model.gguf": ggufStart("general.architecture", "qwen2", "general.type", "model"),
"/llama.gguf": ggufStart("general.architecture", "llama", "general.type", "adapter"),
"/lora.gguf": ggufStart("general.architecture", "qwen2", "general.type", "adapter"),
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(files[r.URL.Path])
}))
defer srv.Close()
models := []catalog.Model{{
Name: "coder", Source: srv.URL + "/base.gguf",
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}},
}}
for _, tc := range []struct {
file, want string
code int
}{
{"/model.gguf", "not a LoRA adapter", http.StatusBadRequest},
{"/llama.gguf", "for llama models", http.StatusBadRequest},
{"/lora.gguf", "", http.StatusOK},
} {
a, _ := setup(t, models)
fm := &fakeModels{}
a.SetModels(fm)
mux := http.NewServeMux()
a.Register(mux)
req := httptest.NewRequest(http.MethodPost, "/api/v1/models/coder/adapters",
strings.NewReader(`{"adapters":[{"name":"fix","source":"`+srv.URL+tc.file+`"}]}`))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
if rec.Code != tc.code || !strings.Contains(rec.Body.String(), tc.want) {
t.Errorf("%s: %d %s; want %d mentioning %q", tc.file, rec.Code, rec.Body.String(), tc.code, tc.want)
}
}
}
+64 -4
View File
@@ -20,6 +20,7 @@ import (
"unicode"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
@@ -62,6 +63,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
a.registerQuotas(mux)
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
@@ -318,9 +320,11 @@ type ModelView struct {
Owner string `json:"owner,omitempty"`
// External is where a model served outside the cluster goes, provider
// host and its name for the model; empty for one served here.
External string `json:"external,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
External string `json:"external,omitempty"`
// Adapters are the LoRA adapters it serves.
Adapters []catalog.Adapter `json:"adapters,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
}
// ReplicaView is a replica as `ctl replicas` shows it.
@@ -574,7 +578,7 @@ func (a *API) Models() []ModelView {
Desired: h.Desired, Failures: h.Failures,
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m),
External: externalLabel(m), Adapters: m.Adapters,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -657,6 +661,7 @@ type Models interface {
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
}
// ResourcesRequest sets what a model's profiles reserve of a node.
@@ -1461,3 +1466,58 @@ func externalLabel(m catalog.Model) string {
}
return host + " · " + m.External.Model
}
// AdaptersRequest replaces a model's LoRA adapters with these.
type AdaptersRequest struct {
Adapters []catalog.Adapter `json:"adapters"`
}
// setAdapters replaces a model's adapters, after reading each new file to
// check it is an adapter for the model's architecture: one that is not is a
// replica that fails to start, found minutes later on a node.
func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req AdaptersRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
name := r.PathValue("name")
m, ok := a.ctl.Catalog().Resolve(name)
if !ok {
writeError(w, http.StatusBadRequest, "no model "+name)
return
}
base, baseErr := gguf.KindFromURL(r.Context(), nil, m.Source)
have := map[string]bool{}
for _, ad := range m.Adapters {
have[ad.Source] = true
}
for _, ad := range req.Adapters {
if have[ad.Source] {
continue // already checked when it was attached
}
h, err := gguf.KindFromURL(r.Context(), nil, ad.Source)
switch {
case err != nil:
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %v", ad.Name, err))
return
case !h.IsAdapter():
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %s is a model, not a LoRA adapter", ad.Name, ad.Source))
return
case baseErr == nil && h.Architecture != base.Architecture:
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s is for %s models and %s is %s",
ad.Name, h.Architecture, m.Name, base.Architecture))
return
}
}
out, err := a.models.SetAdapters(r.Context(), m.Name, req.Adapters, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
}
+4
View File
@@ -258,6 +258,10 @@ type fakeModels struct {
err error
}
func (f *fakeModels) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
return catalog.Model{Name: name, Adapters: as}, f.err
}
func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
return catalog.Model{Name: name, Owner: owner}, f.err
}