Deleting unused model files says what went, and sets go whole
A sweep counted every file the nodes were asked to delete as freed as soon as it asked, and a node refused a set's directory as not empty, so tens of gigabytes were reported freed and nothing was. The agent removes a set's directory whole (a link only as a link), and the control plane waits for each node to list its files again, saying what it no longer lists and what it kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3bb52742a4
commit
cba4897987
8 files changed
+293
-46
No files matched your search
+19
-15
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -87,13 +88,10 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var results []struct {
|
||||
Node string `json:"node"`
|
||||
Files int `json:"files"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
if err := c.post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil {
|
||||
var results []adminapi.CollectResult
|
||||
// The control plane waits for the nodes to list their files without the
|
||||
// deleted ones: half a minute at most.
|
||||
if err := c.patient(time.Minute).post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil {
|
||||
return err
|
||||
}
|
||||
if asJSON {
|
||||
@@ -103,7 +101,7 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Nothing to reclaim.")
|
||||
return nil
|
||||
}
|
||||
var freed int64
|
||||
var freed, left int64
|
||||
for _, r := range results {
|
||||
if r.Error != "" {
|
||||
// Reported rather than fatal: the other nodes' space is still worth
|
||||
@@ -111,14 +109,20 @@ func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", r.Node, r.Error)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: asked to remove %d files, %s\n",
|
||||
r.Node, r.Files, fileSize(r.Bytes))
|
||||
freed += r.Bytes
|
||||
line := fmt.Sprintf("%s: removed %d files, %s", r.Node, r.Files, fileSize(r.Bytes))
|
||||
if r.Left > 0 {
|
||||
line += fmt.Sprintf("; %d still there, %s", r.Left, fileSize(r.LeftBytes))
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), line)
|
||||
freed, left = freed+r.Bytes, left+r.LeftBytes
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\n%s freed.\n", fileSize(freed))
|
||||
if left > 0 {
|
||||
// Each node decides: one that keeps a path it cannot prove it
|
||||
// fetched is the feature working. Its agent's log names each.
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s the nodes kept; each node's agent log says why "+
|
||||
"(journalctl -u llmgw-agent).\n", fileSize(left))
|
||||
}
|
||||
// "asked", not "removed": each node decides, and one that refuses a path it
|
||||
// cannot prove it fetched is the feature working rather than failing.
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\n%s requested. The nodes verify each path before deleting it.\n",
|
||||
fileSize(freed))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -643,17 +643,30 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
|
||||
menuItem{"Delete model files nothing uses", func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Unused files on "+n.ID, http.MethodPost, "/api/v1/weights/collect", map[string]string{"node": n.ID},
|
||||
func(raw json.RawMessage) string {
|
||||
var rs []struct {
|
||||
Files int `json:"files"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
// What the node no longer lists, and what it kept.
|
||||
var rs []adminapi.CollectResult
|
||||
_ = json.Unmarshal(raw, &rs)
|
||||
var files int
|
||||
var b int64
|
||||
var files, left int
|
||||
var b, lb int64
|
||||
var errs []string
|
||||
for _, r := range rs {
|
||||
files, b = files+r.Files, b+r.Bytes
|
||||
files, b, left, lb = files+r.Files, b+r.Bytes, left+r.Left, lb+r.LeftBytes
|
||||
if r.Error != "" {
|
||||
errs = append(errs, r.Error)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%d files, %.1f GB freed.", files, float64(b)/(1<<30))
|
||||
if len(rs) == 0 {
|
||||
return "Nothing on " + n.ID + " that no model uses."
|
||||
}
|
||||
say := fmt.Sprintf("%d files removed, %.1f GB freed.", files, float64(b)/(1<<30))
|
||||
if left > 0 {
|
||||
say += fmt.Sprintf(" %d files (%.1f GB) are still there: the node kept them, and its agent's log "+
|
||||
"says why (journalctl -u llmgw-agent there).", left, float64(lb)/(1<<30))
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
say += " " + strings.Join(errs, "; ")
|
||||
}
|
||||
return say
|
||||
})
|
||||
}},
|
||||
menuItem{"Remove from the cluster…", func(m *consoleModel) tea.Cmd {
|
||||
|
||||
Reference in new issue
Block a user