Attach LoRA adapters from the command line
llmgw ctl model adapters lists a model's adapters and adds or removes
them through POST /api/v1/models/{name}/adapters, which reads each new
file first and refuses one that is not an adapter for the model's
architecture, as the mirror page's card does.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6b1fa65f59
commit
323d172538
6 files changed
+218
-5
No files matched your search
@@ -20,6 +20,7 @@ import (
|
||||
"unicode"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
@@ -62,6 +63,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
@@ -318,9 +320,11 @@ type ModelView struct {
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// External is where a model served outside the cluster goes, provider
|
||||
// host and its name for the model; empty for one served here.
|
||||
External string `json:"external,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
External string `json:"external,omitempty"`
|
||||
// Adapters are the LoRA adapters it serves.
|
||||
Adapters []catalog.Adapter `json:"adapters,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
@@ -574,7 +578,7 @@ func (a *API) Models() []ModelView {
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m),
|
||||
External: externalLabel(m), Adapters: m.Adapters,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -657,6 +661,7 @@ type Models interface {
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
@@ -1461,3 +1466,58 @@ func externalLabel(m catalog.Model) string {
|
||||
}
|
||||
return host + " · " + m.External.Model
|
||||
}
|
||||
|
||||
// AdaptersRequest replaces a model's LoRA adapters with these.
|
||||
type AdaptersRequest struct {
|
||||
Adapters []catalog.Adapter `json:"adapters"`
|
||||
}
|
||||
|
||||
// setAdapters replaces a model's adapters, after reading each new file to
|
||||
// check it is an adapter for the model's architecture: one that is not is a
|
||||
// replica that fails to start, found minutes later on a node.
|
||||
func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req AdaptersRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
name := r.PathValue("name")
|
||||
m, ok := a.ctl.Catalog().Resolve(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "no model "+name)
|
||||
return
|
||||
}
|
||||
base, baseErr := gguf.KindFromURL(r.Context(), nil, m.Source)
|
||||
have := map[string]bool{}
|
||||
for _, ad := range m.Adapters {
|
||||
have[ad.Source] = true
|
||||
}
|
||||
for _, ad := range req.Adapters {
|
||||
if have[ad.Source] {
|
||||
continue // already checked when it was attached
|
||||
}
|
||||
h, err := gguf.KindFromURL(r.Context(), nil, ad.Source)
|
||||
switch {
|
||||
case err != nil:
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %v", ad.Name, err))
|
||||
return
|
||||
case !h.IsAdapter():
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %s is a model, not a LoRA adapter", ad.Name, ad.Source))
|
||||
return
|
||||
case baseErr == nil && h.Architecture != base.Architecture:
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s is for %s models and %s is %s",
|
||||
ad.Name, h.Architecture, m.Name, base.Architecture))
|
||||
return
|
||||
}
|
||||
}
|
||||
out, err := a.models.SetAdapters(r.Context(), m.Name, req.Adapters, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
|
||||
}
|
||||
Reference in new issue
Block a user