package weights import ( "crypto/sha256" "encoding/hex" "path/filepath" "regexp" "sort" "strings" ) // cacheNamePattern matches a filename this package created. // // The eight-byte hash of the source, hex encoded, then a dash, then the // original filename — exactly what cachePath writes. It is the only mark of // ownership there is, and it has to be exact: everything else in a model // directory was put there by the operator, and deleting one of those files // could cost a week of someone's bandwidth. var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`) // CachePathFor is where a source would be cached in the given directory. // // Exported so the control plane can work out which files in a node's reported // list are ones llmgw fetched for a model it still wants, without asking the // node anything. The agent and the control plane must agree on this name // exactly, which is why there is one function and not two. func CachePathFor(dir, source string) string { sum := sha256.Sum256([]byte(source)) name := filepath.Base(source) if name == "" || name == "." || name == "/" { name = "weights" } return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name) } // Owned reports whether a path is one this package created. // // The dividing line of the whole feature. A file without the cache name was // put there by the operator — by scp, by a shared mount, by hand — and llmgw // has no business deleting it however unreferenced it looks. A file with the // cache name is one llmgw downloaded, and llmgw can take it back. func Owned(path string) bool { return cacheNamePattern.MatchString(filepath.Base(path)) } // Orphans lists cached files that no wanted source accounts for. // // `have` is what a node reports on disk and `wanted` is every source in the // catalog; both are passed in so this is a calculation rather than a scan, and // so it can be run on the control plane against a node's reported inventory. // // Only files this package created are ever returned. A source that is not // fetched — a plain path, or a model id an engine resolves itself — produces // no cache name and so protects nothing and deletes nothing, which is correct: // those files are not in the cache to begin with. func Orphans(have []string, wanted []string) []string { keep := map[string]struct{}{} for _, src := range wanted { if !isRemote(src) { // Nothing was fetched for it, so nothing in the cache belongs to // it. Computing a name here would reserve a file that does not // exist and could never be an orphan anyway. continue } // By base name rather than full path: the control plane does not know // each node's cache directory, and the name is what identifies the // file wherever the node keeps it. keep[filepath.Base(CachePathFor("", src))] = struct{}{} } var out []string seen := map[string]struct{}{} for _, path := range have { base := filepath.Base(path) if !Owned(base) { continue } if _, wanted := keep[base]; wanted { continue } if _, dup := seen[path]; dup { continue } seen[path] = struct{}{} out = append(out, path) } sort.Strings(out) return out } // PartialName reports whether a path is the temporary file a fetch writes to. // // Those are not orphans in the same sense — a fetch may be running right now — // so they are named separately rather than swept up with the rest. A `.part` // left behind by a killed agent is real garbage, but only its own node can // tell whether it is stale, and nothing here can. func PartialName(path string) bool { return strings.HasSuffix(path, ".part") }