Deleting unused model files says what went, and sets go whole

A sweep counted every file the nodes were asked to delete as freed as soon
as it asked, and a node refused a set's directory as not empty, so tens of
gigabytes were reported freed and nothing was. The agent removes a set's
directory whole (a link only as a link), and the control plane waits for
each node to list its files again, saying what it no longer lists and what
it kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 10:29:16 +01:00
1 parent 3bb52742a4
commit cba4897987
8 files changed
+293 -46

No files matched your search

+19 -15
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/spf13/cobra"
@@ -87,13 +88,10 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
if err != nil {
return err
}
var results []struct {
Node string `json:"node"`
Files int `json:"files"`
Bytes int64 `json:"bytes"`
Error string `json:"error,omitempty"`
}
if err := c.post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil {
var results []adminapi.CollectResult
// The control plane waits for the nodes to list their files without the
// deleted ones: half a minute at most.
if err := c.patient(time.Minute).post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil {
return err
}
if asJSON {
@@ -103,7 +101,7 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
fmt.Fprintln(cmd.OutOrStdout(), "Nothing to reclaim.")
return nil
}
var freed int64
var freed, left int64
for _, r := range results {
if r.Error != "" {
// Reported rather than fatal: the other nodes' space is still worth
@@ -111,14 +109,20 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", r.Node, r.Error)
continue
}
fmt.Fprintf(cmd.OutOrStdout(), "%s: asked to remove %d files, %s\n",
r.Node, r.Files, fileSize(r.Bytes))
freed += r.Bytes
line := fmt.Sprintf("%s: removed %d files, %s", r.Node, r.Files, fileSize(r.Bytes))
if r.Left > 0 {
line += fmt.Sprintf("; %d still there, %s", r.Left, fileSize(r.LeftBytes))
}
fmt.Fprintln(cmd.OutOrStdout(), line)
freed, left = freed+r.Bytes, left+r.LeftBytes
}
fmt.Fprintf(cmd.OutOrStdout(), "\n%s freed.\n", fileSize(freed))
if left > 0 {
// Each node decides: one that keeps a path it cannot prove it
// fetched is the feature working. Its agent's log names each.
fmt.Fprintf(cmd.OutOrStdout(), "%s the nodes kept; each node's agent log says why "+
"(journalctl -u llmgw-agent).\n", fileSize(left))
}
// "asked", not "removed": each node decides, and one that refuses a path it
// cannot prove it fetched is the feature working rather than failing.
fmt.Fprintf(cmd.OutOrStdout(), "\n%s requested. The nodes verify each path before deleting it.\n",
fileSize(freed))
return nil
}
+21 -8
View File
@@ -643,17 +643,30 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
menuItem{"Delete model files nothing uses", func(m *consoleModel) tea.Cmd {
return m.call("Unused files on "+n.ID, http.MethodPost, "/api/v1/weights/collect", map[string]string{"node": n.ID},
func(raw json.RawMessage) string {
var rs []struct {
Files int `json:"files"`
Bytes int64 `json:"bytes"`
}
// What the node no longer lists, and what it kept.
var rs []adminapi.CollectResult
_ = json.Unmarshal(raw, &rs)
var files int
var b int64
var files, left int
var b, lb int64
var errs []string
for _, r := range rs {
files, b = files+r.Files, b+r.Bytes
files, b, left, lb = files+r.Files, b+r.Bytes, left+r.Left, lb+r.LeftBytes
if r.Error != "" {
errs = append(errs, r.Error)
}
}
return fmt.Sprintf("%d files, %.1f GB freed.", files, float64(b)/(1<<30))
if len(rs) == 0 {
return "Nothing on " + n.ID + " that no model uses."
}
say := fmt.Sprintf("%d files removed, %.1f GB freed.", files, float64(b)/(1<<30))
if left > 0 {
say += fmt.Sprintf(" %d files (%.1f GB) are still there: the node kept them, and its agent's log "+
"says why (journalctl -u llmgw-agent there).", left, float64(lb)/(1<<30))
}
if len(errs) > 0 {
say += " " + strings.Join(errs, "; ")
}
return say
})
}},
menuItem{"Remove from the cluster…", func(m *consoleModel) tea.Cmd {