Disk was the one resource nothing reported. `statfs` was already being called on every model directory, but only to ask whether the filesystem was a network one — the free space it returns in the same struct was thrown away. So the only way to find out why a node was filling up was to ssh to it, which is what this system exists to avoid. Reported per filesystem rather than per configured directory. Free space belongs to the disk, not the path: two model directories on one disk are one pool of bytes, and counting them separately would let the control plane add 24 GB to 24 GB and place a 40 GB model on a node that has 24. Shared filesystems are summed separately from local ones and left out of a node's own total. Space on a filer belongs to everyone mounting it, so twenty nodes would otherwise each report the same free terabyte, and a scheduler would place twenty models into room for one. Nothing reported means nothing is known, everywhere: an agent too old to send storage and a node with no model directories both send nothing, and both must read as unknown rather than as a full disk. `ctl nodes` and the dashboard show a dash, the metric is absent rather than zero, and a missing directory — which is what an unmounted filer looks like — is skipped rather than reported as having no space. Shown as free out of total, because the first is useless without the second: 8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root partition, and only the second stops a model being placed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b5177b70f6
commit
753f4e3265
13 files changed
+515
-83
No files matched your search
+20
-2
@@ -149,10 +149,11 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n",
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
|
||||
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
@@ -221,6 +222,23 @@ func memColumn(gb int) string {
|
||||
return fmt.Sprintf("%dG", gb)
|
||||
}
|
||||
|
||||
// diskColumn renders free space against the size it is free out of.
|
||||
//
|
||||
// Both, because they answer different questions and the first is useless
|
||||
// without the second: 8 GB free is comfortable on a 10 TB filer and nearly
|
||||
// full on a 10 GB root partition, and it is the second case that stops a
|
||||
// model from being placed.
|
||||
//
|
||||
// A dash for a node that reported nothing, which an older agent and a node
|
||||
// with no model directories both do. It has to read as "not known" rather
|
||||
// than as "no space", or every unconfigured node looks like it is full.
|
||||
func diskColumn(freeGB, totalGB int) string {
|
||||
if totalGB <= 0 {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprintf("%dG/%dG", freeGB, totalGB)
|
||||
}
|
||||
|
||||
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "models",
|
||||
|
||||
Reference in new issue
Block a user