Disk was the one resource nothing reported. `statfs` was already being called on every model directory, but only to ask whether the filesystem was a network one — the free space it returns in the same struct was thrown away. So the only way to find out why a node was filling up was to ssh to it, which is what this system exists to avoid. Reported per filesystem rather than per configured directory. Free space belongs to the disk, not the path: two model directories on one disk are one pool of bytes, and counting them separately would let the control plane add 24 GB to 24 GB and place a 40 GB model on a node that has 24. Shared filesystems are summed separately from local ones and left out of a node's own total. Space on a filer belongs to everyone mounting it, so twenty nodes would otherwise each report the same free terabyte, and a scheduler would place twenty models into room for one. Nothing reported means nothing is known, everywhere: an agent too old to send storage and a node with no model directories both send nothing, and both must read as unknown rather than as a full disk. `ctl nodes` and the dashboard show a dash, the metric is absent rather than zero, and a missing directory — which is what an unmounted filer looks like — is skipped rather than reported as having no space. Shown as free out of total, because the first is useless without the second: 8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root partition, and only the second stops a model being placed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b5177b70f6
commit
753f4e3265
13 files changed
+515
-83
No files matched your search
@@ -51,6 +51,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
// holds has nothing to do with its cards, and a node with none is exactly
|
||||
// where a CPU profile will look for them.
|
||||
inv.Weights = ScanWeights(opts.ModelDirs)
|
||||
// And the disk they sit on. A node can hold a model today and be unable to
|
||||
// take the next one, which the file list alone never shows.
|
||||
inv.Storage = ScanStorage(opts.ModelDirs)
|
||||
|
||||
if !opts.SkipGPUs {
|
||||
switch {
|
||||
|
||||
@@ -114,6 +114,9 @@ type Inventory struct {
|
||||
// Weights are the model files this node can see. Empty means it was not
|
||||
// asked to look.
|
||||
Weights []LocalWeights `json:"weights,omitempty"`
|
||||
// Storage is the disk behind those weights, one entry per filesystem.
|
||||
// Empty means the same thing it means for Weights: not asked.
|
||||
Storage []Storage `json:"storage,omitempty"`
|
||||
}
|
||||
|
||||
// Classes returns the distinct GPU classes present, sorted.
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Storage is one filesystem a node keeps weights on.
|
||||
//
|
||||
// One entry per filesystem rather than per configured directory, because free
|
||||
// space belongs to the disk and not to the path. Two model directories on the
|
||||
// same disk are one pool of bytes, and reporting them separately would let the
|
||||
// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24.
|
||||
type Storage struct {
|
||||
// Path is one directory on this filesystem — the first one configured that
|
||||
// landed here, so the operator recognises it.
|
||||
Path string `json:"path"`
|
||||
// FreeBytes is what a non-root process can still write. Deliberately not
|
||||
// the root reserve: the agent does not run as root, so the reserve is space
|
||||
// it cannot have.
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
// TotalBytes is the filesystem's size, so a reading of "8 GB free" can be
|
||||
// told apart on a 10 TB filer and on a 10 GB root partition.
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
// Shared marks a filesystem the node did not have to fill. Free space on
|
||||
// one is shared with every other node using it, so two nodes reporting
|
||||
// 500 GB free on the same filer have 500 GB between them, not a terabyte.
|
||||
Shared bool `json:"shared,omitempty"`
|
||||
}
|
||||
|
||||
// ScanStorage reports the filesystems behind the given model directories.
|
||||
//
|
||||
// Empty in, empty out, and a directory that cannot be read is skipped rather
|
||||
// than reported as full: a mount that has not come back yet looks exactly like
|
||||
// a missing directory, and claiming zero free bytes would take the node out of
|
||||
// placement for a reason that is not true. Absent is not zero here, as
|
||||
// everywhere else in this system.
|
||||
func ScanStorage(dirs []string) []Storage {
|
||||
var out []Storage
|
||||
seen := map[uint64]int{} // filesystem id -> index in out
|
||||
|
||||
for _, dir := range dirs {
|
||||
if dir == "" {
|
||||
continue
|
||||
}
|
||||
dev, ok := filesystemID(dir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if i, dup := seen[dev]; dup {
|
||||
// Same disk, already counted. Keep the first path: it is the one
|
||||
// the operator wrote first and so the one they will recognise.
|
||||
_ = i
|
||||
continue
|
||||
}
|
||||
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs(dir, &st); err != nil {
|
||||
continue
|
||||
}
|
||||
// Bsize is int32 on some platforms and int64 on others, which is why
|
||||
// every field here is widened explicitly rather than assumed.
|
||||
block := int64(st.Bsize)
|
||||
seen[dev] = len(out)
|
||||
out = append(out, Storage{
|
||||
Path: dir,
|
||||
FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose
|
||||
TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose
|
||||
Shared: isNetworkFS(dir),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// filesystemID identifies the disk a path is on.
|
||||
//
|
||||
// By device number rather than by the statfs filesystem id, because the field
|
||||
// holding that id is spelled differently on Linux and on macOS and this has to
|
||||
// compile on both — the agent runs on one and is developed on the other.
|
||||
func filesystemID(path string) (uint64, bool) {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) {
|
||||
// The reason this matters: free space belongs to the disk, not to the path.
|
||||
// Counting two directories separately lets the control plane add 24 GB to
|
||||
// 24 GB and place a 40 GB model on a node that has 24.
|
||||
base := t.TempDir()
|
||||
a := filepath.Join(base, "models-a")
|
||||
b := filepath.Join(base, "models-b")
|
||||
for _, d := range []string{a, b} {
|
||||
if err := os.Mkdir(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
got := ScanStorage([]string{a, b})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Path != a {
|
||||
t.Errorf("kept %q rather than the first directory configured", got[0].Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageReportsSomethingPlausible(t *testing.T) {
|
||||
got := ScanStorage([]string{t.TempDir()})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a real directory reported %d filesystems", len(got))
|
||||
}
|
||||
s := got[0]
|
||||
if s.TotalBytes <= 0 {
|
||||
t.Error("a filesystem with no size at all")
|
||||
}
|
||||
if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes {
|
||||
t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) {
|
||||
// A mount that has not come back yet looks exactly like a missing
|
||||
// directory. Reporting it as zero free bytes would take the node out of
|
||||
// placement for a reason that is not true; reporting nothing says only
|
||||
// that nothing is known, which is the honest answer.
|
||||
got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")})
|
||||
if len(got) != 0 {
|
||||
t.Errorf("a missing directory was reported as a filesystem: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoDirectoriesConstrainNothing(t *testing.T) {
|
||||
if got := ScanStorage(nil); len(got) != 0 {
|
||||
t.Errorf("scanning nothing found %+v", got)
|
||||
}
|
||||
if got := ScanStorage([]string{""}); len(got) != 0 {
|
||||
t.Errorf("an empty path found %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user