Trained adapters published, private to their owners like the knowledge bases they come from

An adapter now has an owner (org, team, project, user or key): only callers
there see it, and to anyone else it is a model that does not exist, absent
from /v1/models too. A training job's owner defaults to the one collection
its pairs come from, must be able to read every collection they come from,
and pairs from a knowledge base are never made public. ctl train publish
keeps the GGUF or PEFT files on the mirror, which accepts uploads now, and
sets the owned adapter after the usual architecture check.

The knowledge bases' cache_salt follows the collections searched rather
than the organization, so two teams of one organization no longer share
cached passages.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 21:30:50 +02:00
1 parent 964b35ae65
commit 85025dfb03
29 files changed
+1049 -84

No files matched your search

+7
View File
@@ -518,6 +518,13 @@ func newServerCmd() *cobra.Command {
return fmt.Errorf("the model mirror: %w", err)
}
rt.SetMirror(mc)
// Adapters trained on the nodes are kept there too; sending one
// takes longer than the mirror's usual answers.
if adminAPI != nil {
if up, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 10*time.Minute); err == nil {
adminAPI.SetTrainMirror(up)
}
}
// Browsing the registry is what turns "paste a resolve URL"
// into "find the model". It needs a way out to the internet
// from this machine, which the mirror's own fetching does not.
+51 -6
View File
@@ -28,7 +28,8 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
"what they say. Facts belong to the knowledge base.",
}
var model, base, rev, node, data string
var model, base, rev, node, data, owner string
var public bool
var gpuMem int
var epochs, rank int
var lr float64
@@ -45,7 +46,7 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return err
}
req := adminapi.TrainRequest{Name: args[0], Model: model, BaseRepo: base, BaseRevision: rev, Node: node,
GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}}
GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public}
if cmd.Flags().Changed("epochs") {
req.Params["epochs"] = epochs
}
@@ -64,7 +65,12 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if err := client().patient(2*time.Minute).do(cmd.Context(), "POST", "/api/v1/train/jobs", json.RawMessage(body), &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs\n", v.ID, v.Name, v.Model, v.Node, v.Pairs)
who := "every caller of " + v.Model
if v.Owner != "" {
who = v.Owner + " only"
}
fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs; it will be seen by %s\n",
v.ID, v.Name, v.Model, v.Node, v.Pairs, who)
fmt.Fprintf(cmd.OutOrStdout(), "follow it with: llmgw ctl train show %s --follow\n", v.ID)
return nil
},
@@ -80,6 +86,9 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
f.IntVar(&rank, "rank", 16, "the adapter's rank")
f.Float64Var(&lr, "lr", 2e-4, "the learning rate")
f.BoolVar(&noGGUF, "no-gguf", false, "skip the llama.cpp GGUF, even for a model llama.cpp serves")
f.StringVar(&owner, "owner", "", "who the adapter belongs to — org:, team:, project: (path or ID), user: or key: — and only they see it "+
"(default: the knowledge base's owner, when the pairs come from one)")
f.BoolVar(&public, "public", false, "every caller of the model sees it; refused for pairs from a knowledge base")
for _, req := range []string{"model", "base-repo", "node", "data"} {
_ = start.MarkFlagRequired(req)
}
@@ -102,9 +111,9 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return nil
}
w := tab()
fmt.Fprintln(w, "JOB\tADAPTER\tMODEL\tNODE\tSTATE\tPROGRESS\tLOSS\tPEAK\tCREATED")
fmt.Fprintln(w, "JOB\tADAPTER\tMODEL\tOWNER\tNODE\tSTATE\tPROGRESS\tLOSS\tPEAK\tCREATED")
for _, j := range jobs {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j.ID, j.Name, j.Model, j.Node, j.State,
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j.ID, j.Name, j.Model, or(j.Owner, "everyone"), j.Node, j.State,
trainProgress(j), trainLoss(j), trainPeak(j), j.CreatedAt.Local().Format("01-02 15:04"))
}
return w.Flush()
@@ -129,7 +138,8 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
line := fmt.Sprintf("%s %s %s loss %s peak %s %s", j.State, trainProgress(j), j.Node, trainLoss(j), trainPeak(j), j.Message)
if !follow {
fmt.Fprintf(out, "%s: adapter %s for %s, on %s (base %s), %d pairs\n", j.ID, j.Name, j.Model, j.Node, j.BaseRepo, j.Pairs)
fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs\n", j.ID, j.Name, j.Model,
or(j.Owner, "everyone"), j.Node, j.BaseRepo, j.Pairs)
fmt.Fprintln(out, line)
if len(j.Artifacts) > 0 {
fmt.Fprintf(out, "made: %s (llmgw ctl train download %s --kind KIND)\n", strings.Join(j.Artifacts, ", "), j.ID)
@@ -182,6 +192,41 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
},
})
var restart bool
publish := &cobra.Command{
Use: "publish JOB",
Short: "Make what a finished job made its model's adapter, kept on the mirror",
Long: "The GGUF for a model llama.cpp serves, the PEFT files for one vLLM serves, are kept\n" +
"on the mirror and added to the model's adapters; a request then asks for the\n" +
"adapter by its name, as a model. llama.cpp loads adapters when a replica starts:\n" +
"--restart replaces the model's replicas, one at a time, to load it.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var v adminapi.TrainPublishView
if err := client().patient(15*time.Minute).do(cmd.Context(), "POST", "/api/v1/train/jobs/"+args[0]+"/publish", nil, &v); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s is %s's adapter, from %s\n", v.Adapter, v.Model, v.Source)
if v.Owner != "" {
fmt.Fprintf(out, "only %s sees it; to anyone else it is a model that does not exist\n", v.Owner)
}
if restart && strings.Contains(v.Note, "restart") {
body, _ := json.Marshal(adminapi.RestartRequest{Models: []string{v.Model}})
var res adminapi.RestartResult
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
return err
}
fmt.Fprintf(out, "%d replica(s) of %s restarting to load it; then ask for model %s\n", len(res.Queued), v.Model, v.Adapter)
return nil
}
fmt.Fprintln(out, v.Note)
return nil
},
}
publish.Flags().BoolVar(&restart, "restart", false, "restart the model's replicas to load it (llama.cpp)")
c.AddCommand(publish)
var kind, outFile string
download := &cobra.Command{
Use: "download JOB",