Attach LoRA adapters from the command line
llmgw ctl model adapters lists a model's adapters and adds or removes
them through POST /api/v1/models/{name}/adapters, which reads each new
file first and refuses one that is not an adapter for the model's
architecture, as the mirror page's card does.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6b1fa65f59
commit
323d172538
6 files changed
+218
-5
No files matched your search
+70
-1
@@ -5,6 +5,9 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/knadh/koanf/parsers/yaml"
|
||||
@@ -31,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
@@ -264,3 +267,69 @@ func newClassCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
|
||||
var add, remove []string
|
||||
c := &cobra.Command{
|
||||
Use: "adapters <model> [--add NAME=URL[@SCALE]] [--remove NAME]",
|
||||
Short: "List, add or remove a model's LoRA adapters",
|
||||
Long: "An adapter is asked for by its NAME as though it were a model, and is served\n" +
|
||||
"by this model's replicas with it applied. URL is the adapter file, usually in\n" +
|
||||
"the mirror; it must be a GGUF LoRA adapter for this model's architecture,\n" +
|
||||
"which is checked before it is attached. Replicas load a changed set when\n" +
|
||||
"restarted: llmgw ctl restart --model MODEL.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
name := args[0]
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||
return err
|
||||
}
|
||||
var current []catalog.Adapter
|
||||
found := false
|
||||
for _, m := range models {
|
||||
if m.Name == name {
|
||||
current, found = m.Adapters, true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no model %q", name)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(add) == 0 && len(remove) == 0 {
|
||||
if len(current) == 0 {
|
||||
fmt.Fprintf(out, "%s has no adapters\n", name)
|
||||
}
|
||||
for _, a := range current {
|
||||
fmt.Fprintf(out, "%s\tscale %g\t%s\n", a.Name, a.EffectiveScale(), a.Source)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
next := slices.DeleteFunc(slices.Clone(current), func(a catalog.Adapter) bool { return slices.Contains(remove, a.Name) })
|
||||
for _, spec := range add {
|
||||
n, rest, ok := strings.Cut(spec, "=")
|
||||
if !ok || n == "" || rest == "" {
|
||||
return fmt.Errorf("--add %q is not NAME=URL[@SCALE]", spec)
|
||||
}
|
||||
ad := catalog.Adapter{Name: n, Source: rest}
|
||||
if i := strings.LastIndex(rest, "@"); i > 0 {
|
||||
if s, err := strconv.ParseFloat(rest[i+1:], 64); err == nil {
|
||||
ad.Source, ad.Scale = rest[:i], s
|
||||
}
|
||||
}
|
||||
next = append(next, ad)
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.AdaptersRequest{Adapters: next})
|
||||
var res adminapi.AdaptersRequest
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/adapters", body, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "%s serves %d adapter(s); restart its replicas to load them: llmgw ctl restart --model %s\n",
|
||||
name, len(res.Adapters), name)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringArrayVar(&add, "add", nil, "NAME=URL[@SCALE]; repeatable")
|
||||
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
|
||||
return c
|
||||
}
|
||||
Reference in new issue
Block a user