Ilker ManapandClaude Opus 5.5
cba4897987
A sweep counted every file the nodes were asked to delete as freed as soon as it asked, and a node refused a set's directory as not empty, so tens of gigabytes were reported freed and nothing was. The agent removes a set's directory whole (a link only as a link), and the control plane waits for each node to list its files again, saying what it no longer lists and what it kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
72 lines
2.5 KiB
Go
72 lines
2.5 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
// deletingDispatcher deletes what it is asked to as a node would, by its
|
|
// files dropping out of what the node lists — but for refused, which it
|
|
// keeps, as an agent keeps a file it cannot prove it fetched.
|
|
type deletingDispatcher struct {
|
|
nullDispatcher
|
|
inv *inventory.Store
|
|
refused string
|
|
}
|
|
|
|
func (d *deletingDispatcher) RemoveWeights(_ context.Context, node string, paths []string) error {
|
|
n, _ := d.inv.Get(node)
|
|
n.Weights = slices.DeleteFunc(n.Weights, func(w discovery.LocalWeights) bool {
|
|
return w.Path != d.refused && slices.Contains(paths, w.Path)
|
|
})
|
|
d.inv.Upsert(n)
|
|
return nil
|
|
}
|
|
|
|
// What a sweep says it freed is what the node no longer lists: a file it
|
|
// kept is said to be left, not counted as freed.
|
|
func TestASweepSaysWhatWasDeletedNotWhatWasAsked(t *testing.T) {
|
|
confirmWait = 1500 * time.Millisecond
|
|
t.Cleanup(func() { confirmWait = 30 * time.Second })
|
|
|
|
gone := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/old/v1/old.gguf")
|
|
set := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/big/v1")
|
|
kept := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/stuck/v1/stuck.gguf")
|
|
cat, _ := catalog.New(nil)
|
|
d := &deletingDispatcher{refused: kept}
|
|
ctl := control.New(control.DefaultConfig(), catalog.NewLive(cat), d, nullSink{}, slog.New(slog.DiscardHandler))
|
|
d.inv = ctl.Inventory()
|
|
node := gpuNode("n1", 1)
|
|
node.Weights = []discovery.LocalWeights{{Path: gone, SizeBytes: 1 << 30}, {Path: set, SizeBytes: 20 << 30},
|
|
{Path: kept, SizeBytes: 4 << 30}}
|
|
ctl.Inventory().Upsert(node)
|
|
a := New(ctl)
|
|
|
|
rec := httptest.NewRecorder()
|
|
a.collectWeights(rec, httptest.NewRequest(http.MethodPost, "/api/v1/weights/collect", strings.NewReader(`{"node":"n1"}`)))
|
|
var got []CollectResult
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
|
t.Fatalf("%v: %s", err, rec.Body.String())
|
|
}
|
|
if len(got) != 1 {
|
|
t.Fatalf("results %+v", got)
|
|
}
|
|
r := got[0]
|
|
if r.Files != 2 || r.Bytes != 21<<30 || r.Left != 1 || r.LeftBytes != 4<<30 {
|
|
t.Errorf("result %+v: want 2 files and 21 GB freed, 1 file of 4 GB left", r)
|
|
}
|
|
}
|