Ilker ManapandClaude Opus 5.5
561a536477
CI / test (push) Successful in 9m33s
A set is fetched into a directory named from its address without the trailing slash; Orphans named it from the catalog's address with it, so every set in the catalog was offered for deletion: 330 GB of six models on the PoC. One function, cacheSource, now gives both the name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
111 lines
4.0 KiB
Go
111 lines
4.0 KiB
Go
package weights
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// cacheNamePattern matches a filename this package created.
|
|
//
|
|
// The eight-byte hash of the source, hex encoded, then a dash, then the
|
|
// original filename — exactly what cachePath writes. It is the only mark of
|
|
// ownership there is, and it has to be exact: everything else in a model
|
|
// directory was put there by the operator, and deleting one of those files
|
|
// could cost a week of someone's bandwidth.
|
|
var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`)
|
|
|
|
// CachePathFor is where a source would be cached in the given directory.
|
|
//
|
|
// Exported so the control plane can work out which files in a node's reported
|
|
// list are ones llmgw fetched for a model it still wants, without asking the
|
|
// node anything. The agent and the control plane must agree on this name
|
|
// exactly, which is why there is one function and not two.
|
|
func CachePathFor(dir, source string) string {
|
|
sum := sha256.Sum256([]byte(source))
|
|
name := filepath.Base(source)
|
|
if name == "" || name == "." || name == "/" {
|
|
name = "weights"
|
|
}
|
|
return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name)
|
|
}
|
|
|
|
// cacheSource is the source a cache name is computed from: a set's address
|
|
// without its trailing slash, as its directory was named when it was fetched.
|
|
// Computed from the slash-ended address, every set in the catalog looked like
|
|
// nobody's, and a sweep offered 330 GB of models still in use for deletion.
|
|
func cacheSource(source string) string {
|
|
if IsSet(source) {
|
|
return strings.TrimSuffix(source, "/")
|
|
}
|
|
return source
|
|
}
|
|
|
|
// Owned reports whether a path is one this package created.
|
|
//
|
|
// The dividing line of the whole feature. A file without the cache name was
|
|
// put there by the operator — by scp, by a shared mount, by hand — and llmgw
|
|
// has no business deleting it however unreferenced it looks. A file with the
|
|
// cache name is one llmgw downloaded, and llmgw can take it back.
|
|
func Owned(path string) bool {
|
|
return cacheNamePattern.MatchString(filepath.Base(path))
|
|
}
|
|
|
|
// Orphans lists cached files that no wanted source accounts for.
|
|
//
|
|
// `have` is what a node reports on disk and `wanted` is every source in the
|
|
// catalog; both are passed in so this is a calculation rather than a scan, and
|
|
// so it can be run on the control plane against a node's reported inventory.
|
|
//
|
|
// Only files this package created are ever returned. A source that is not
|
|
// fetched — a plain path, or a model id an engine resolves itself — produces
|
|
// no cache name and so protects nothing and deletes nothing, which is correct:
|
|
// those files are not in the cache to begin with.
|
|
func Orphans(have []string, wanted []string) []string {
|
|
keep := map[string]struct{}{}
|
|
for _, src := range wanted {
|
|
if !isRemote(src) {
|
|
// Nothing was fetched for it, so nothing in the cache belongs to
|
|
// it. Computing a name here would reserve a file that does not
|
|
// exist and could never be an orphan anyway.
|
|
continue
|
|
}
|
|
// By base name rather than full path: the control plane does not know
|
|
// each node's cache directory, and the name is what identifies the
|
|
// file wherever the node keeps it.
|
|
keep[filepath.Base(CachePathFor("", cacheSource(src)))] = struct{}{}
|
|
}
|
|
|
|
var out []string
|
|
seen := map[string]struct{}{}
|
|
for _, path := range have {
|
|
base := filepath.Base(path)
|
|
if !Owned(base) {
|
|
continue
|
|
}
|
|
if _, wanted := keep[base]; wanted {
|
|
continue
|
|
}
|
|
if _, dup := seen[path]; dup {
|
|
continue
|
|
}
|
|
seen[path] = struct{}{}
|
|
out = append(out, path)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// PartialName reports whether a path is the temporary file a fetch writes to.
|
|
//
|
|
// Those are not orphans in the same sense — a fetch may be running right now —
|
|
// so they are named separately rather than swept up with the rest. A `.part`
|
|
// left behind by a killed agent is real garbage, but only its own node can
|
|
// tell whether it is stale, and nothing here can.
|
|
func PartialName(path string) bool {
|
|
return strings.HasSuffix(path, ".part")
|
|
}
|