A model can be stopped and kept, or removed with its files from the mirror and the nodes

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -60,6 +60,9 @@ type API struct {
 	controlPlanes func() ControlPlanes
 	// serve runs registry-to-replica jobs; nil without a mirror and a store.
 	serve *modelserve.Server
+	// purgeMirror is the mirror a model removed with its files is removed
+	// from; nil without one.
+	purgeMirror PurgeMirror
 	// history is the Prometheus the console's charts read from.
 	history History
 	// audit reads the audit log.
@@ -123,6 +126,8 @@ func (a *API) Register(mux Mux) {
 	mux.HandleFunc("POST /api/v1/models", a.addModel)
 	mux.HandleFunc("GET /api/v1/models/{name}", a.getModel)
 	mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
+	mux.HandleFunc("POST /api/v1/models/{name}/stop", a.stopModel)
+	mux.HandleFunc("POST /api/v1/models/{name}/start", a.startModel)
 	mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
 	mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
 	a.registerQuotas(mux)
@@ -451,6 +456,8 @@ type ModelView struct {
 	// answer; 0 when the catalog does not say.
 	ContextLength int `json:"context_length,omitempty"`
 	MaxOutput     int `json:"max_output,omitempty"`
+	// Stopped is a model an administrator took out of service, kept.
+	Stopped bool `json:"stopped,omitempty"`
 	// Canary is a share of its requests sent to a candidate; CanaryOf is
 	// set on the candidate, naming the model it is a canary of.
 	Canary   *catalog.Canary `json:"canary,omitempty"`
@@ -843,7 +850,7 @@ func (a *API) Models() []ModelView {
 		h := health[m.Name]
 		v := ModelView{
 			Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
-			ContextLength: m.ContextLength, MaxOutput: m.MaxOutput,
+			ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Stopped: m.Stopped != nil,
 			Min: m.Scaling.Min, Max: m.Scaling.Max,
 			Ready: ready[m.Name], Total: total[m.Name],
 			Fallbacks: m.Fallbacks,
@@ -945,6 +952,8 @@ type Models interface {
 	Add(ctx context.Context, m catalog.Model, by string) error
 	Remove(ctx context.Context, name string) error
 	Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
+	Stop(ctx context.Context, name, by string) (catalog.Model, error)
+	Start(ctx context.Context, name, by string) (catalog.Model, error)
 	Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
 	SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
 	SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
@@ -1194,6 +1203,8 @@ type ScaleResponse struct {
 	Max         int           `json:"max"`
 	IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
 	OnDemand    bool          `json:"on_demand"`
+	// Stopped is a model taken out of service, kept, until it is started.
+	Stopped bool `json:"stopped,omitempty"`
 }
 
 func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
@@ -1227,6 +1238,32 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+// stopModel takes a model out of service, kept.
+func (a *API) stopModel(w http.ResponseWriter, r *http.Request) {
+	a.stopOrStart(w, r, a.models.Stop)
+}
+
+// startModel runs a stopped model again.
+func (a *API) startModel(w http.ResponseWriter, r *http.Request) {
+	a.stopOrStart(w, r, a.models.Start)
+}
+
+func (a *API) stopOrStart(w http.ResponseWriter, r *http.Request, f func(context.Context, string, string) (catalog.Model, error)) {
+	if a.models == nil {
+		writeError(w, http.StatusNotImplemented, "this control plane has no store")
+		return
+	}
+	m, err := f(r.Context(), r.PathValue("name"), actorOf(r))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, ScaleResponse{
+		Model: m.Name, Min: m.Scaling.Min, Max: m.Scaling.Max,
+		IdleTimeout: m.Scaling.IdleTimeout, OnDemand: m.Scaling.OnDemand(), Stopped: m.Stopped != nil,
+	})
+}
+
 // setResources sets a model's reserved memory and cores. Placed replicas keep
 // what they were started with until they are restarted.
 func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
@@ -1253,6 +1290,10 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	name := r.PathValue("name")
+	if r.URL.Query().Get("purge") == "true" {
+		a.purgeModel(w, r, name)
+		return
+	}
 	if err := a.models.Remove(r.Context(), name); err != nil {
 		writeError(w, http.StatusBadRequest, err.Error())
 		return

internal/control/adminapi/api_test.go

@@ -749,3 +749,11 @@ func TestANodesCardInTheWrongSlotIsAWarning(t *testing.T) {
 		t.Errorf("warnings %q", warned)
 	}
 }
+
+func (f *fakeModels) Stop(_ context.Context, name, _ string) (catalog.Model, error) {
+	return catalog.Model{Name: name, Stopped: &catalog.Scaling{Min: 1, Max: 1}}, f.err
+}
+
+func (f *fakeModels) Start(_ context.Context, name, _ string) (catalog.Model, error) {
+	return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
+}

internal/control/adminapi/purge.go

@@ -0,0 +1,230 @@
+package adminapi
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
+)
+
+// A model removed with its files: the catalog entry, every version of it the
+// mirror holds (and so, carried by its removal, a peer mirror's), and the
+// copies the nodes fetched, once its replicas have stopped. What another
+// model still uses stays.
+
+// PurgeMirror is what a purge asks of the mirror.
+type PurgeMirror interface {
+	List(ctx context.Context) (mirror.ModelsResponse, error)
+	Remove(ctx context.Context, model, version string) ([]string, error)
+}
+
+// SetPurgeMirror gives removing a model with its files the mirror.
+func (a *API) SetPurgeMirror(m PurgeMirror) { a.purgeMirror = m }
+
+// purgeWait is how long a purge waits for the model's replicas to stop
+// before deleting the nodes' copies; after it, they are left for llmgw ctl
+// weights --collect.
+var purgeWait = 60 * time.Second
+
+// PurgeView is what a model removed with its files took with it.
+type PurgeView struct {
+	Removed string `json:"removed"`
+	// Mirror are the versions removed from the mirror, model/version.
+	Mirror []string `json:"mirror,omitempty"`
+	// Kept are versions another model still uses, left on the mirror.
+	Kept  []string        `json:"kept,omitempty"`
+	Nodes []CollectResult `json:"nodes,omitempty"`
+	Note  string          `json:"note,omitempty"`
+}
+
+// purgeModel removes a model from the catalog and its files from the mirror
+// and the nodes.
+func (a *API) purgeModel(w http.ResponseWriter, r *http.Request, name string) {
+	m, ok := a.ctl.Catalog().Resolve(name)
+	if !ok {
+		writeError(w, http.StatusNotFound, "no model "+name+" in the catalog")
+		return
+	}
+	name = m.Name
+	others := []catalog.Model{}
+	for _, o := range a.ctl.Catalog().Models() {
+		if o.Name != name {
+			others = append(others, o)
+		}
+	}
+	if err := a.models.Remove(r.Context(), name); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	a.presetDeclined(r.Context(), name, actorOf(r))
+	out := PurgeView{Removed: name}
+
+	sources := purgeSources(m)
+	if a.purgeMirror == nil {
+		out.Note = "this control plane has no mirror; its files there, if any, are left"
+	} else {
+		out.Mirror, out.Kept, out.Note = purgeFromMirror(r.Context(), a.purgeMirror, sources, others)
+	}
+
+	// The nodes' copies, once nothing runs on them.
+	deadline := time.Now().Add(purgeWait)
+	for a.runs(name) && time.Now().Before(deadline) {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-time.After(time.Second):
+		}
+	}
+	if a.runs(name) {
+		out.Note = strings.TrimSpace(out.Note + " Its replicas had not stopped yet: its copies on the nodes are " +
+			"left, and llmgw ctl weights --collect deletes them.")
+		writeJSON(w, http.StatusOK, out)
+		return
+	}
+	out.Nodes = a.purgeFromNodes(r.Context(), sources, others)
+	writeJSON(w, http.StatusOK, out)
+}
+
+// runs says whether any replica of a model is still on a node.
+func (a *API) runs(model string) bool {
+	for _, rep := range a.ctl.Reconciler().Replicas() {
+		if rep.Model == model {
+			return true
+		}
+	}
+	return false
+}
+
+// purgeSources are the files a model fetched: its weights, its adapters,
+// and the weights its last update replaced.
+func purgeSources(m catalog.Model) []string {
+	out := m.WeightSources()
+	for _, ad := range m.Adapters {
+		out = append(out, ad.Source)
+	}
+	if m.Origin != nil && m.Origin.Previous != nil && m.Origin.Previous.Source != "" {
+		out = append(out, m.Origin.Previous.Source)
+	}
+	return out
+}
+
+// inUse is every file the other models fetch.
+func inUse(others []catalog.Model) []string {
+	var out []string
+	for _, o := range others {
+		out = append(out, purgeSources(o)...)
+	}
+	return out
+}
+
+// purgeFromMirror removes from the mirror every version of the model
+// directories the sources are in, but those another model uses.
+func purgeFromMirror(ctx context.Context, mc PurgeMirror, sources []string, others []catalog.Model) (removed, kept []string, note string) {
+	held, err := mc.List(ctx)
+	if err != nil {
+		return nil, nil, "the mirror could not be read (" + err.Error() + "): its files there are left"
+	}
+	dirs := map[string]bool{}
+	for _, s := range sources {
+		if model, _, ok := mirrorPlace(s); ok {
+			dirs[model] = true
+		}
+	}
+	used := map[string]bool{}
+	for _, s := range inUse(others) {
+		if model, version, ok := mirrorPlace(s); ok {
+			used[model+"/"+version] = true
+		}
+	}
+	seen := map[string]bool{}
+	for _, st := range held.Models {
+		key := st.Model + "/" + st.Version
+		if !dirs[st.Model] || seen[key] {
+			continue
+		}
+		seen[key] = true
+		if used[key] {
+			kept = append(kept, key)
+			continue
+		}
+		if _, err := mc.Remove(ctx, st.Model, st.Version); err != nil {
+			note = strings.TrimSpace(note + " " + key + " could not be removed from the mirror: " + err.Error() + ".")
+			continue
+		}
+		removed = append(removed, key)
+	}
+	slices.Sort(removed)
+	slices.Sort(kept)
+	return removed, kept, note
+}
+
+// mirrorPlace is the model directory and version a mirror URL names:
+// BASE/MODEL/VERSION/FILE.
+func mirrorPlace(source string) (model, version string, ok bool) {
+	u, err := url.Parse(source)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
+		return "", "", false
+	}
+	// A mirror serves from its root: a registry's address is longer.
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) != 3 {
+		return "", "", false
+	}
+	model, version = parts[0], parts[1]
+	if mirror.ValidName(model) != nil || mirror.ValidName(version) != nil {
+		return "", "", false
+	}
+	return model, version, true
+}
+
+// purgeFromNodes deletes the nodes' copies of the sources: the files llmgw
+// fetched for them under any mirror's address, that no other model wants.
+func (a *API) purgeFromNodes(ctx context.Context, sources []string, others []catalog.Model) []CollectResult {
+	rm := a.ctl.WeightRemover()
+	if rm == nil {
+		return nil
+	}
+	names := map[string]bool{}
+	for _, s := range sources {
+		names[filepath.Base(s)] = true
+	}
+	wanted := inUse(others)
+	var out []CollectResult
+	for _, n := range a.ctl.Inventory().Nodes() {
+		size := map[string]int64{}
+		have := make([]string, 0, len(n.Weights))
+		for _, w := range n.Weights {
+			have = append(have, w.Path)
+			size[w.Path] = w.SizeBytes
+		}
+		var paths []string
+		for _, p := range weights.Orphans(have, wanted) {
+			// The cache name is a hash of the address, then the file's own
+			// name: the address may be either mirror's, the name is the same.
+			if _, file, ok := strings.Cut(filepath.Base(p), "-"); ok && names[file] {
+				paths = append(paths, p)
+			}
+		}
+		if len(paths) == 0 {
+			continue
+		}
+		res := CollectResult{Node: n.ID, Files: len(paths)}
+		for _, p := range paths {
+			res.Bytes += size[p]
+		}
+		if err := rm.RemoveWeights(ctx, n.ID, paths); err != nil {
+			res.Error, res.Files, res.Bytes = err.Error(), 0, 0
+			slog.Warn("a purged model's copies could not be deleted on a node", "node", n.ID, "err", err)
+		}
+		out = append(out, res)
+	}
+	return out
+}

internal/control/adminapi/purge_test.go

@@ -0,0 +1,68 @@
+package adminapi
+
+import (
+	"context"
+	"slices"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
+)
+
+type purgeMirrorFake struct {
+	held    []mirror.Status
+	removed []string
+}
+
+func (f *purgeMirrorFake) List(context.Context) (mirror.ModelsResponse, error) {
+	return mirror.ModelsResponse{Base: "http://m:8081", Models: f.held}, nil
+}
+
+func (f *purgeMirrorFake) Remove(_ context.Context, model, version string) ([]string, error) {
+	f.removed = append(f.removed, model+"/"+version)
+	return nil, nil
+}
+
+func TestAPurgeRemovesEveryVersionOfTheModelButWhatAnotherUses(t *testing.T) {
+	f := &purgeMirrorFake{held: []mirror.Status{
+		{Model: "qwen", Version: "v2", File: "a.gguf"},
+		{Model: "qwen", Version: "v2", File: "b.gguf"},
+		{Model: "qwen", Version: "v1", File: "a.gguf"},
+		{Model: "qwen", Version: "v3", File: "a.gguf"},
+		{Model: "other", Version: "v1", File: "o.gguf"},
+		{Model: "lora", Version: "t1", File: "l.gguf"},
+	}}
+	m := catalog.Model{Name: "qwen", Source: "http://m:8081/qwen/v2/a.gguf",
+		Adapters: []catalog.Adapter{{Name: "tone", Source: "http://m:8081/lora/t1/l.gguf"}}}
+	// The canary runs v3: it stays.
+	others := []catalog.Model{{Name: "qwen-canary", Source: "http://other-mirror:8081/qwen/v3/a.gguf"},
+		{Name: "other", Source: "http://m:8081/other/v1/o.gguf"}}
+	removed, kept, note := purgeFromMirror(t.Context(), f, purgeSources(m), others)
+	if note != "" {
+		t.Fatal(note)
+	}
+	if !slices.Equal(removed, []string{"lora/t1", "qwen/v1", "qwen/v2"}) || !slices.Equal(kept, []string{"qwen/v3"}) {
+		t.Errorf("removed %v, kept %v", removed, kept)
+	}
+	if slices.Contains(f.removed, "other/v1") || len(f.removed) != 3 {
+		t.Errorf("asked to remove %v", f.removed)
+	}
+}
+
+func TestOnlyAMirrorsAddressNamesAPlaceOnIt(t *testing.T) {
+	for src, want := range map[string]string{
+		"http://10.0.0.146:8081/qwen/abc/q4.gguf":         "qwen/abc",
+		"https://huggingface.co/o/r/resolve/main/q4.gguf": "",
+		"/srv/models/q4.gguf":                             "",
+		"Qwen/Qwen2.5-7B":                                 "",
+	} {
+		model, version, ok := mirrorPlace(src)
+		got := ""
+		if ok {
+			got = model + "/" + version
+		}
+		if got != want {
+			t.Errorf("%s: %q, want %q", src, got, want)
+		}
+	}
+}

internal/control/adminapi/routes.go

@@ -155,8 +155,16 @@ var Routes = []Route{
 		Description: "With the defaults the catalog filled in. By its name only, not an alias.",
 		Response:    catalog.Model{}},
 	{Method: "DELETE", Path: "/api/v1/models/{name}", Summary: "Remove a model added at runtime",
-		Description: "A model declared in the cluster's settings is refused: it is changed there.",
-		Response:    Removed{}},
+		Description: "A model declared in the cluster's settings is refused: it is changed there. With purge, its " +
+			"files go too: every version of it the mirror holds (a peer mirror follows), and the nodes' copies once its " +
+			"replicas have stopped, waiting up to a minute for them; what another model uses stays. The answer is then a PurgeView.",
+		Query:    []Param{{Name: "purge", Type: "boolean", Description: "Delete its files from the mirror and the nodes as well."}},
+		Response: Removed{}},
+	{Method: "POST", Path: "/api/v1/models/{name}/stop", Summary: "Take a model out of service and keep it",
+		Description: "None of its replicas run, and requests are told an administrator stopped it; what it was scaled to is kept for start.",
+		Response:    ScaleResponse{}},
+	{Method: "POST", Path: "/api/v1/models/{name}/start", Summary: "Run a stopped model again, scaled as it was",
+		Response: ScaleResponse{}},
 	{Method: "POST", Path: "/api/v1/models/{name}/scale", Summary: "Set how many replicas a model runs",
 		Request: ScaleRequest{}, Response: ScaleResponse{}},
 	{Method: "POST", Path: "/api/v1/models/{name}/resources", Summary: "Set what a model's profiles reserve of a node",