Attach LoRA adapters from the command line
llmgw ctl model adapters lists a model's adapters and adds or removes
them through POST /api/v1/models/{name}/adapters, which reads each new
file first and refuses one that is not an adapter for the model's
architecture, as the mirror page's card does.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6b1fa65f59
commit
323d172538
6 files changed
+218
-5
No files matched your search
@@ -0,0 +1,71 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
// ggufStart is the start of a GGUF file with these string keys.
|
||||
func ggufStart(kv ...string) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(len(kv) / 2))
|
||||
for i := 0; i < len(kv); i += 2 {
|
||||
str(kv[i])
|
||||
w(uint32(8)) // string
|
||||
str(kv[i+1])
|
||||
}
|
||||
b.Write(make([]byte, 4096))
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestOnlyAnAdapterForTheModelsArchitectureIsAttached(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
"/base.gguf": ggufStart("general.architecture", "qwen2"),
|
||||
"/model.gguf": ggufStart("general.architecture", "qwen2", "general.type", "model"),
|
||||
"/llama.gguf": ggufStart("general.architecture", "llama", "general.type", "adapter"),
|
||||
"/lora.gguf": ggufStart("general.architecture", "qwen2", "general.type", "adapter"),
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(files[r.URL.Path])
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
models := []catalog.Model{{
|
||||
Name: "coder", Source: srv.URL + "/base.gguf",
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}},
|
||||
}}
|
||||
for _, tc := range []struct {
|
||||
file, want string
|
||||
code int
|
||||
}{
|
||||
{"/model.gguf", "not a LoRA adapter", http.StatusBadRequest},
|
||||
{"/llama.gguf", "for llama models", http.StatusBadRequest},
|
||||
{"/lora.gguf", "", http.StatusOK},
|
||||
} {
|
||||
a, _ := setup(t, models)
|
||||
fm := &fakeModels{}
|
||||
a.SetModels(fm)
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/models/coder/adapters",
|
||||
strings.NewReader(`{"adapters":[{"name":"fix","source":"`+srv.URL+tc.file+`"}]}`))
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.code || !strings.Contains(rec.Body.String(), tc.want) {
|
||||
t.Errorf("%s: %d %s; want %d mentioning %q", tc.file, rec.Code, rec.Body.String(), tc.code, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"unicode"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
@@ -62,6 +63,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
@@ -318,9 +320,11 @@ type ModelView struct {
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// External is where a model served outside the cluster goes, provider
|
||||
// host and its name for the model; empty for one served here.
|
||||
External string `json:"external,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
External string `json:"external,omitempty"`
|
||||
// Adapters are the LoRA adapters it serves.
|
||||
Adapters []catalog.Adapter `json:"adapters,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
@@ -574,7 +578,7 @@ func (a *API) Models() []ModelView {
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m),
|
||||
External: externalLabel(m), Adapters: m.Adapters,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -657,6 +661,7 @@ type Models interface {
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
@@ -1461,3 +1466,58 @@ func externalLabel(m catalog.Model) string {
|
||||
}
|
||||
return host + " · " + m.External.Model
|
||||
}
|
||||
|
||||
// AdaptersRequest replaces a model's LoRA adapters with these.
|
||||
type AdaptersRequest struct {
|
||||
Adapters []catalog.Adapter `json:"adapters"`
|
||||
}
|
||||
|
||||
// setAdapters replaces a model's adapters, after reading each new file to
|
||||
// check it is an adapter for the model's architecture: one that is not is a
|
||||
// replica that fails to start, found minutes later on a node.
|
||||
func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req AdaptersRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
name := r.PathValue("name")
|
||||
m, ok := a.ctl.Catalog().Resolve(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "no model "+name)
|
||||
return
|
||||
}
|
||||
base, baseErr := gguf.KindFromURL(r.Context(), nil, m.Source)
|
||||
have := map[string]bool{}
|
||||
for _, ad := range m.Adapters {
|
||||
have[ad.Source] = true
|
||||
}
|
||||
for _, ad := range req.Adapters {
|
||||
if have[ad.Source] {
|
||||
continue // already checked when it was attached
|
||||
}
|
||||
h, err := gguf.KindFromURL(r.Context(), nil, ad.Source)
|
||||
switch {
|
||||
case err != nil:
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %v", ad.Name, err))
|
||||
return
|
||||
case !h.IsAdapter():
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %s is a model, not a LoRA adapter", ad.Name, ad.Source))
|
||||
return
|
||||
case baseErr == nil && h.Architecture != base.Architecture:
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s is for %s models and %s is %s",
|
||||
ad.Name, h.Architecture, m.Name, base.Architecture))
|
||||
return
|
||||
}
|
||||
}
|
||||
out, err := a.models.SetAdapters(r.Context(), m.Name, req.Adapters, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
|
||||
}
|
||||
@@ -258,6 +258,10 @@ type fakeModels struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeModels) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Adapters: as}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Owner: owner}, f.err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user