Ilker ManapandClaude Opus 5
774e134ed1
CI / test (push) Successful in 1m39s
Two halves of the same problem: nothing stopped a model being placed on a node that could not hold it, and nothing ever removed the weights of a model that had been deleted from the catalog. Placement now has a disk check, shaped exactly like the memory one it sits beside, with one addition: it is skipped on a node that already holds the weights. A node holding the file needs no room for it, and without that the node best placed to run a model — because it already has it — is the one most likely to be rejected, since holding the file is what used the space. Reclaiming is the dangerous half, so the whole design is about what it will not do. llmgw deletes only files it fetched itself, and that is a calculation rather than a heuristic: the cache name is a hash of the source, so the set of wanted files can be computed from the catalog exactly. Anything without that name was put there by an operator — scp, a shared mount, a copy by hand — and is none of this system's business however unreferenced it looks. A model that took a week to copy must not be removable by a command someone ran to tidy up. The node checks every path again before unlinking it, and that duplication is the point. The control plane knows the catalog; the node knows its disk. A control plane that is wrong, newer than the agent, or tampered with must not be able to talk a node into deleting something, so the agent refuses anything outside its own cache directory or without the cache name — including a path that tries to traverse out of it. Listing is the default and deleting takes --collect. The summary says "asked to remove", because each node decides, and one that refuses a path it cannot prove it fetched is this working rather than failing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
108 lines
3.3 KiB
Go
108 lines
3.3 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"sort"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
|
)
|
|
|
|
// OrphanView is cached weights on one node that no catalog model accounts for.
|
|
type OrphanView struct {
|
|
Node string `json:"node"`
|
|
Paths []string `json:"paths"`
|
|
// Bytes is what deleting them would free, as far as the node reported. A
|
|
// file whose size was not reported counts as zero rather than being left
|
|
// out, so the list and the number always describe the same set.
|
|
Bytes int64 `json:"bytes"`
|
|
}
|
|
|
|
// Orphans finds cached weights nothing in the catalog points at.
|
|
//
|
|
// Computed here from what nodes already report, so listing costs no round trip
|
|
// and a node that is offline simply contributes nothing — its files are not
|
|
// deletable while it is gone anyway.
|
|
//
|
|
// Only files llmgw fetched are ever considered. Everything else in a model
|
|
// directory was put there by the operator and is none of this system's
|
|
// business, however unreferenced it looks.
|
|
func (a *API) Orphans() []OrphanView {
|
|
var wanted []string
|
|
for _, m := range a.ctl.Catalog().Models() {
|
|
wanted = append(wanted, m.Source)
|
|
}
|
|
|
|
out := make([]OrphanView, 0)
|
|
for _, n := range a.ctl.Inventory().Nodes() {
|
|
size := map[string]int64{}
|
|
have := make([]string, 0, len(n.Weights))
|
|
for _, w := range n.Weights {
|
|
have = append(have, w.Path)
|
|
size[w.Path] = w.SizeBytes
|
|
}
|
|
|
|
paths := weights.Orphans(have, wanted)
|
|
if len(paths) == 0 {
|
|
continue
|
|
}
|
|
v := OrphanView{Node: n.ID, Paths: paths}
|
|
for _, p := range paths {
|
|
v.Bytes += size[p]
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
|
return out
|
|
}
|
|
|
|
func (a *API) listOrphans(w http.ResponseWriter, _ *http.Request) {
|
|
writeJSON(w, http.StatusOK, a.Orphans())
|
|
}
|
|
|
|
// collectWeights deletes them.
|
|
//
|
|
// A proposal, not an instruction: each node checks every path again before
|
|
// unlinking it, and refuses anything it cannot prove it fetched. The two
|
|
// checks are deliberate duplication — the control plane knows the catalog and
|
|
// the node knows its disk, and only the node can be sure.
|
|
func (a *API) collectWeights(w http.ResponseWriter, r *http.Request) {
|
|
rm := a.ctl.WeightRemover()
|
|
if rm == nil {
|
|
writeError(w, http.StatusNotImplemented,
|
|
"this control plane cannot ask nodes to delete files")
|
|
return
|
|
}
|
|
|
|
var body struct {
|
|
// Node limits it to one node. Empty means every node, which is the
|
|
// common case after removing a model from the catalog.
|
|
Node string `json:"node,omitempty"`
|
|
}
|
|
// An empty body is the "all nodes" case and must not be an error.
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
|
|
type result struct {
|
|
Node string `json:"node"`
|
|
Files int `json:"files"`
|
|
Bytes int64 `json:"bytes"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
out := make([]result, 0)
|
|
for _, o := range a.Orphans() {
|
|
if body.Node != "" && o.Node != body.Node {
|
|
continue
|
|
}
|
|
res := result{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes}
|
|
// A node that has gone since the list was built is reported rather than
|
|
// failing the whole sweep: the other nodes' space is still worth
|
|
// freeing, and this one will be offered again next time.
|
|
if err := rm.RemoveWeights(r.Context(), o.Node, o.Paths); err != nil {
|
|
res.Error = err.Error()
|
|
res.Files, res.Bytes = 0, 0
|
|
}
|
|
out = append(out, res)
|
|
}
|
|
writeJSON(w, http.StatusOK, out)
|
|
}
|