Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s

Two defects, both of the same kind: something the CLI knew and the rest of
the system did not.

A profile can be edited while a replica is running against the old one. The
reconciler deliberately leaves it alone — churning a replica that is serving
costs more than the imperfection — but there was no way to see it, so the
cluster could disagree with its own configuration indefinitely and read as
healthy. Profile now carries a fingerprint, a replica records the one it was
placed with, and both the API and `ctl replicas` say "stale profile" when
they differ. An empty fingerprint is not stale: it means a replica adopted
from an older agent, which is unknown rather than wrong.

Adding a model from the admin page skipped the source check `ctl model add`
has done all along, so a typo or a mirror that is down became a replica
failing on a node minutes later, with an error naming the node. The check
moves out of the CLI into a package both use.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 22:42:26 +02:00
1 parent 832d0a652c
commit 40f5c930c4
14 files changed
+281 -49

No files matched your search

+80
View File
@@ -0,0 +1,80 @@
package sourcecheck
import (
"context"
"fmt"
"net/http"
"strings"
"time"
)
// Check asks whether a model's weights are actually there.
//
// A source that does not resolve is caught today by a replica failing to start,
// minutes later, on a node — and the error names the node rather than the typo.
// Asking now turns that into a sentence while the operator still has the file
// open.
//
// Only URLs. A path is checked on the node that will use it, by the node, which
// is the only machine whose answer means anything; a bare model id is resolved
// by the engine and is not this function's business.
//
// A package of its own rather than a helper in the CLI, because the admin page
// adds models too and was not checking anything. Two ways in that disagree
// about whether the weights exist is the kind of difference nobody finds until
// a replica fails on a node, hours later, naming the node.
func Check(ctx context.Context, source string) (size int64, err error) {
url, ok := HTTPSource(source)
if !ok {
return 0, nil
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return 0, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return 0, fmt.Errorf("cannot reach %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("%s answered %s", url, resp.Status)
}
// Absent when the server does not send it, which is not an error: a
// resumable fetch does not need the length in advance, and refusing here
// would reject a mirror that is merely terse. Go reports that as -1;
// normalised to 0 so callers have one spelling of "unknown".
if resp.ContentLength < 0 {
return 0, nil
}
return resp.ContentLength, nil
}
// HTTPSource returns the URL to check, and whether there is one.
//
// hf: is expanded the way the fetcher expands it, so the check asks about the
// same file the download will ask for rather than about a string that looks
// similar.
func HTTPSource(source string) (string, bool) {
switch {
case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"):
return source, true
case strings.HasPrefix(source, "hf:"):
rest := strings.TrimPrefix(source, "hf:")
parts := strings.SplitN(rest, "/", 3)
if len(parts) != 3 {
// A repository rather than a file. vLLM and SGLang download those
// themselves, so there is nothing here to check.
return "", false
}
return fmt.Sprintf("https://huggingface.co/%s/%s/resolve/main/%s",
parts[0], parts[1], parts[2]), true
default:
return "", false
}
}
+103
View File
@@ -0,0 +1,103 @@
package sourcecheck
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestOnlyURLsAreChecked(t *testing.T) {
// A path is checked by the node that will use it, which is the only
// machine whose answer means anything — the control plane may not even
// have that filesystem. A bare model id is the engine's business.
for _, src := range []string{
"/var/lib/llmgw-models/x.gguf", "./x.gguf", "Qwen/Qwen2.5-7B-Instruct",
} {
if _, ok := HTTPSource(src); ok {
t.Errorf("%q was treated as a URL to check", src)
}
if _, err := Check(context.Background(), src); err != nil {
t.Errorf("checking %q returned %v; it should be skipped", src, err)
}
}
}
func TestAnHFFileIsCheckedAtTheURLTheFetcherWouldUse(t *testing.T) {
// Asking about a string that merely looks similar would be worse than not
// asking: it would pass while the download later fails.
got, ok := HTTPSource("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf")
if !ok {
t.Fatal("an hf: file was not recognised")
}
want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/q4.gguf"
if got != want {
t.Errorf("hf: resolved to %q, want %q", got, want)
}
}
func TestAnHFRepositoryIsNotChecked(t *testing.T) {
// vLLM and SGLang download whole repositories themselves; there is no one
// file to ask about.
if _, ok := HTTPSource("hf:Qwen/Qwen2.5-7B-Instruct"); ok {
t.Error("a repository was treated as a single file")
}
}
func TestAReachableSourceReportsItsSize(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodHead {
t.Errorf("the check used %s, not HEAD — it must not download the file", r.Method)
}
w.Header().Set("Content-Length", "1073741824")
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
size, err := Check(context.Background(), srv.URL+"/model.gguf")
if err != nil {
t.Fatal(err)
}
if size != 1<<30 {
t.Errorf("size = %d", size)
}
}
func TestAMissingFileSaysSoWithTheURL(t *testing.T) {
// The whole point: a typo becomes a sentence while the operator still has
// the file open, rather than a replica failing minutes later on a node
// with an error that names the node.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "not found", http.StatusNotFound)
}))
defer srv.Close()
_, err := Check(context.Background(), srv.URL+"/typo.gguf")
if err == nil {
t.Fatal("a 404 was accepted")
}
if !strings.Contains(err.Error(), "typo.gguf") {
t.Errorf("the error does not name the URL: %v", err)
}
}
func TestAServerThatSendsNoLengthIsStillFine(t *testing.T) {
// A resumable fetch does not need the length in advance, and refusing here
// would reject a mirror that is merely terse.
// A server that does not know the length omits the header rather than
// sending a negative one, which is what a real mirror streaming from a
// pipe does.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
size, err := Check(context.Background(), srv.URL+"/x.gguf")
if err != nil {
t.Errorf("a terse server was rejected: %v", err)
}
if size != 0 {
t.Errorf("an unknown length came back as %d, want 0", size)
}
}