Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s

Two defects, both of the same kind: something the CLI knew and the rest of
the system did not.

A profile can be edited while a replica is running against the old one. The
reconciler deliberately leaves it alone — churning a replica that is serving
costs more than the imperfection — but there was no way to see it, so the
cluster could disagree with its own configuration indefinitely and read as
healthy. Profile now carries a fingerprint, a replica records the one it was
placed with, and both the API and `ctl replicas` say "stale profile" when
they differ. An empty fingerprint is not stale: it means a replica adopted
from an older agent, which is unknown rather than wrong.

Adding a model from the admin page skipped the source check `ctl model add`
has done all along, so a typo or a mirror that is down became a replica
failing on a node minutes later, with an error naming the node. The check
moves out of the CLI into a package both use.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 22:42:26 +02:00
1 parent 832d0a652c
commit 40f5c930c4
14 files changed
+281 -49

No files matched your search

+8 -1
View File
@@ -344,8 +344,15 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
for _, s := range r.Shards {
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
}
// The state says stale rather than a column of its own: most
// replicas are not, a column would be empty for all of them,
// and when one is it is the first thing worth knowing about it.
state := r.State
if r.Stale {
state += " (stale profile)"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
r.ID, r.Model, r.State, r.Pool, join(nodes),
r.ID, r.Model, state, r.Pool, join(nodes),
(time.Duration(r.AgeSec) * time.Second).String(),
idleColumn(r.IdleSec), r.Endpoint)
// A line rather than a column, because most replicas are not
+28
View File
@@ -3,6 +3,8 @@ package cli
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
@@ -44,3 +46,29 @@ func TestTheFilenameIsTakenFromTheURLWhenNotGiven(t *testing.T) {
}
}
}
func TestAVersionAnswersHeadAsWellAsGet(t *testing.T) {
// A sharded model's catalog source is the version directory, and the serve
// path checks a source with HEAD before accepting it. If the manifest
// answered only GET, every multi-file model would be refused as missing.
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "llama", "v1"), 0o755); err != nil {
t.Fatal(err)
}
for _, name := range []string{"model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"} {
if err := os.WriteFile(filepath.Join(root, "llama", "v1", name), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
h := readOnly(withManifest(root, http.FileServer(http.Dir(root))))
w := httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/llama/v1/", nil))
if w.Code != http.StatusOK {
t.Fatalf("a version should answer HEAD; got %d", w.Code)
}
if got := w.Header().Get("Content-Type"); got != "application/json" {
t.Errorf("expected a manifest, got %q", got)
}
}
+3 -2
View File
@@ -13,6 +13,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
)
// newModelCmd adds and removes models without a restart.
@@ -70,13 +71,13 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
// minutes later on a node — with an error naming the node rather
// than the typo.
if !skipCheck {
size, err := checkSource(cmd.Context(), m.Source)
size, err := sourcecheck.Check(cmd.Context(), m.Source)
if err != nil {
return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+
"which is right when the mirror is not up yet or the nodes "+
"reach it by an address this machine does not.", err)
}
if _, isURL := httpSource(m.Source); isURL {
if _, isURL := sourcecheck.HTTPSource(m.Source); isURL {
fmt.Fprintln(cmd.OutOrStdout(), sourceNote(m.Source, size))
}
}
+17
View File
@@ -0,0 +1,17 @@
package cli
import "fmt"
// sourceNote is what to print about a checked source.
//
// It says where the answer came from, because the control plane's network is
// not the nodes' network: a mirror this machine can reach may be unreachable
// from a node behind a different firewall, and the reverse. The check catches
// typos and a mirror that is down, which is most of what goes wrong, and it
// should not be read as proof that the fetch will work.
func sourceNote(source string, size int64) string {
if size > 0 {
return fmt.Sprintf(" source reachable from here: %s (%s)", source, fileSize(size))
}
return fmt.Sprintf(" source reachable from here: %s (size not advertised)", source)
}
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"strings"
"testing"
)
func TestTheNoteSaysWhereTheAnswerCameFrom(t *testing.T) {
// The control plane's network is not the nodes' network. The check catches
// typos and a mirror that is down; it is not proof the fetch will work,
// and the wording must not suggest it is.
note := sourceNote("http://models.internal/x.gguf", 1<<30)
if !strings.Contains(note, "from here") {
t.Errorf("the note overstates what was checked: %s", note)
}
}
+35 -3
View File
@@ -123,9 +123,16 @@ type ModelView struct {
// ReplicaView is a replica as `ctl replicas` shows it.
type ReplicaView struct {
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
// Stale is true when the profile has been edited since this replica was
// placed, so it is running a definition the catalog no longer holds.
//
// Reported rather than acted on. Moving a replica that is serving costs
// more than the imperfection, and the operator who made the edit is the
// one who should decide whether it is worth a restart.
Stale bool `json:"stale,omitempty"`
State string `json:"state"`
Pool string `json:"pool,omitempty"`
Fabric string `json:"fabric,omitempty"`
@@ -290,6 +297,7 @@ func (a *API) Replicas(model string) []ReplicaView {
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
Endpoint: rep.Endpoint, Err: rep.Err,
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
IdleSec: -1,
}
@@ -675,3 +683,27 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
func writeError(w http.ResponseWriter, code int, msg string) {
writeJSON(w, code, map[string]string{"error": msg})
}
// profileChanged reports whether a replica is running a profile the catalog no
// longer holds.
//
// A replica with no recorded fingerprint is not stale: that is every replica
// adopted from an agent older than the field, and calling those stale would
// mark a whole fleet as wrong on the first upgrade. Nor is one whose profile
// has been deleted outright — the model or the profile going away is a
// different thing, and the replica's own state already says it.
func (a *API) profileChanged(model, profile, was string) bool {
if was == "" {
return false
}
m, ok := a.ctl.Catalog().Resolve(model)
if !ok {
return false
}
for _, p := range m.Profiles {
if p.Name == profile {
return p.Fingerprint() != was
}
}
return false
}
+45
View File
@@ -622,3 +622,48 @@ func TestABadSinceIsRefusedBeforeTheQuery(t *testing.T) {
t.Error("nothing should have reached the store")
}
}
func TestAReplicaRunningAnEditedProfileIsMarkedStale(t *testing.T) {
models := testModels()
a, ctl := setup(t, models)
// Placed from the profile as it was, then the profile is edited and keeps
// its name — which is what makes the two indistinguishable without a
// fingerprint.
was := models[0].Profiles[0].Fingerprint()
edited := models[0]
edited.Profiles[0].MinMemoryGB = 64
built, err := catalog.New([]catalog.Model{edited})
if err != nil {
t.Fatal(err)
}
ctl.ReplaceCatalog(t.Context(), built)
if !a.profileChanged("llama", "tp8", was) {
t.Error("an edited profile should read as changed")
}
if a.profileChanged("llama", "tp8", edited.Profiles[0].Fingerprint()) {
t.Error("the profile in force is not a change")
}
}
func TestAReplicaWithNoRecordedProfileIsNotStale(t *testing.T) {
a, _ := setup(t, testModels())
// Every replica adopted from an agent older than the field. Marking those
// stale would call a whole fleet wrong on the first upgrade.
if a.profileChanged("llama", "tp8", "") {
t.Error("unknown is not stale")
}
}
func TestAProfileThatIsGoneIsNotReportedAsStale(t *testing.T) {
a, _ := setup(t, testModels())
// The model or the profile disappearing is a different thing, and the
// replica's own state already says it.
if a.profileChanged("llama", "no-such-profile", "abc") {
t.Error("a missing profile is not a changed one")
}
if a.profileChanged("no-such-model", "tp8", "abc") {
t.Error("a missing model is not a changed profile")
}
}
+24
View File
@@ -3,6 +3,9 @@
package catalog
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"strings"
@@ -444,3 +447,24 @@ func (m Model) Simulated() bool {
}
return true
}
// Fingerprint identifies a profile's contents.
//
// The name is not enough. A profile can be edited — a memory requirement
// raised, a node label added, an engine argument changed — and keep its name,
// so a replica running the old definition is indistinguishable from one
// running the new. The reconciler does not move a running replica when its
// profile changes, which is right: churning a replica that is serving costs
// more than the imperfection. What was missing is any way to see that it had
// happened, and the answer to "why is this one on the wrong node" was in
// somebody's memory of what they edited an hour ago.
func (p Profile) Fingerprint() string {
b, err := json.Marshal(p)
if err != nil {
// A profile that will not marshal cannot be compared, and saying so by
// never matching is better than saying two differing ones are the same.
return "unmarshalable"
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
+1 -1
View File
@@ -812,7 +812,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name,
ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(),
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
WantShards: len(pl.Nodes),
+12
View File
@@ -104,6 +104,18 @@ type Replica struct {
ID string
Model string
Profile string
// ProfileFP is the profile's contents when this replica was placed.
//
// Compared with the catalog's to tell a replica running the definition in
// force from one running whatever it was edited from. A running replica is
// deliberately not moved when its profile changes — churn costs more than
// the imperfection — so without this there is nothing that says the two
// have diverged, and "why is this one on the wrong node" is answered from
// memory of what somebody edited an hour ago.
//
// Empty on a replica adopted from an agent that predates it, which reads
// as unknown rather than as stale.
ProfileFP string
// Fabric is set only for multi-node replicas.
Fabric string
Pool string
@@ -1,4 +1,4 @@
package cli
package sourcecheck
import (
"context"
@@ -8,7 +8,7 @@ import (
"time"
)
// checkSource asks whether a model's weights are actually there.
// Check asks whether a model's weights are actually there.
//
// A source that does not resolve is caught today by a replica failing to start,
// minutes later, on a node — and the error names the node rather than the typo.
@@ -18,8 +18,13 @@ import (
// Only URLs. A path is checked on the node that will use it, by the node, which
// is the only machine whose answer means anything; a bare model id is resolved
// by the engine and is not this function's business.
func checkSource(ctx context.Context, source string) (size int64, err error) {
url, ok := httpSource(source)
//
// A package of its own rather than a helper in the CLI, because the admin page
// adds models too and was not checking anything. Two ways in that disagree
// about whether the weights exist is the kind of difference nobody finds until
// a replica fails on a node, hours later, naming the node.
func Check(ctx context.Context, source string) (size int64, err error) {
url, ok := HTTPSource(source)
if !ok {
return 0, nil
}
@@ -50,12 +55,12 @@ func checkSource(ctx context.Context, source string) (size int64, err error) {
return resp.ContentLength, nil
}
// httpSource returns the URL to check, and whether there is one.
// HTTPSource returns the URL to check, and whether there is one.
//
// hf: is expanded the way the fetcher expands it, so the check asks about the
// same file the download will ask for rather than about a string that looks
// similar.
func httpSource(source string) (string, bool) {
func HTTPSource(source string) (string, bool) {
switch {
case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"):
return source, true
@@ -73,17 +78,3 @@ func httpSource(source string) (string, bool) {
return "", false
}
}
// sourceNote is what to print about a checked source.
//
// It says where the answer came from, because the control plane's network is
// not the nodes' network: a mirror this machine can reach may be unreachable
// from a node behind a different firewall, and the reverse. The check catches
// typos and a mirror that is down, which is most of what goes wrong, and it
// should not be read as proof that the fetch will work.
func sourceNote(source string, size int64) string {
if size > 0 {
return fmt.Sprintf(" source reachable from here: %s (%s)", source, fileSize(size))
}
return fmt.Sprintf(" source reachable from here: %s (size not advertised)", source)
}
@@ -1,4 +1,4 @@
package cli
package sourcecheck
import (
"context"
@@ -15,10 +15,10 @@ func TestOnlyURLsAreChecked(t *testing.T) {
for _, src := range []string{
"/var/lib/llmgw-models/x.gguf", "./x.gguf", "Qwen/Qwen2.5-7B-Instruct",
} {
if _, ok := httpSource(src); ok {
if _, ok := HTTPSource(src); ok {
t.Errorf("%q was treated as a URL to check", src)
}
if _, err := checkSource(context.Background(), src); err != nil {
if _, err := Check(context.Background(), src); err != nil {
t.Errorf("checking %q returned %v; it should be skipped", src, err)
}
}
@@ -27,7 +27,7 @@ func TestOnlyURLsAreChecked(t *testing.T) {
func TestAnHFFileIsCheckedAtTheURLTheFetcherWouldUse(t *testing.T) {
// Asking about a string that merely looks similar would be worse than not
// asking: it would pass while the download later fails.
got, ok := httpSource("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf")
got, ok := HTTPSource("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf")
if !ok {
t.Fatal("an hf: file was not recognised")
}
@@ -40,7 +40,7 @@ func TestAnHFFileIsCheckedAtTheURLTheFetcherWouldUse(t *testing.T) {
func TestAnHFRepositoryIsNotChecked(t *testing.T) {
// vLLM and SGLang download whole repositories themselves; there is no one
// file to ask about.
if _, ok := httpSource("hf:Qwen/Qwen2.5-7B-Instruct"); ok {
if _, ok := HTTPSource("hf:Qwen/Qwen2.5-7B-Instruct"); ok {
t.Error("a repository was treated as a single file")
}
}
@@ -55,7 +55,7 @@ func TestAReachableSourceReportsItsSize(t *testing.T) {
}))
defer srv.Close()
size, err := checkSource(context.Background(), srv.URL+"/model.gguf")
size, err := Check(context.Background(), srv.URL+"/model.gguf")
if err != nil {
t.Fatal(err)
}
@@ -73,7 +73,7 @@ func TestAMissingFileSaysSoWithTheURL(t *testing.T) {
}))
defer srv.Close()
_, err := checkSource(context.Background(), srv.URL+"/typo.gguf")
_, err := Check(context.Background(), srv.URL+"/typo.gguf")
if err == nil {
t.Fatal("a 404 was accepted")
}
@@ -93,7 +93,7 @@ func TestAServerThatSendsNoLengthIsStillFine(t *testing.T) {
}))
defer srv.Close()
size, err := checkSource(context.Background(), srv.URL+"/x.gguf")
size, err := Check(context.Background(), srv.URL+"/x.gguf")
if err != nil {
t.Errorf("a terse server was rejected: %v", err)
}
@@ -101,13 +101,3 @@ func TestAServerThatSendsNoLengthIsStillFine(t *testing.T) {
t.Errorf("an unknown length came back as %d, want 0", size)
}
}
func TestTheNoteSaysWhereTheAnswerCameFrom(t *testing.T) {
// The control plane's network is not the nodes' network. The check catches
// typos and a mirror that is down; it is not proof the fetch will work,
// and the wording must not suggest it is.
note := sourceNote("http://models.internal/x.gguf", 1<<30)
if !strings.Contains(note, "from here") {
t.Errorf("the note overstates what was checked: %s", note)
}
}
+10
View File
@@ -12,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -189,6 +190,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
if err != nil {
return "", err
}
// The same check `llmgw ctl model add` makes. This page builds
// the source out of a row the mirror reported, so it is right
// far more often than something typed — but a file removed
// between the listing and the click, or a mirror that has gone
// down since, both end as a replica failing on a node minutes
// later with an error naming the node. Asking costs one HEAD.
if _, err := sourcecheck.Check(ctx, model.Source); err != nil {
return "", err
}
if err := d.Catalog.Add(ctx, model, admin.Email); err != nil {
return "", err
}
+63 -4
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
@@ -60,6 +61,17 @@ func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
}, true)
}
// reachableSource stands in for a mirror holding the file. The serve handler
// checks the source before the catalog sees it, the same as the CLI does, so a
// test that means to exercise the form has to offer something that answers.
func reachableSource(t *testing.T) string {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
t.Cleanup(srv.Close)
return srv.URL + "/coder/v1/m.gguf"
}
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
cat := &fakeCatalog{}
body := mirrorGet(t, servePage(t, []mirror.Status{
@@ -160,9 +172,10 @@ func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
"source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"},
"source": {source}, "engine": {"llamacpp"},
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
"ctx_size": {"16384"}, "profile": {"cpu-large"},
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
@@ -207,8 +220,9 @@ func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
source := reachableSource(t)
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
@@ -316,8 +330,9 @@ func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
func TestADescriptionReachesTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
"description": {"code completion and review; 7B, fits the large node"},
})
@@ -334,11 +349,55 @@ func TestADescriptionReachesTheCatalog(t *testing.T) {
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
})
if len(cat.added) != 1 || cat.added[0].Description != "" {
t.Errorf("a description is optional: %+v", cat.added)
}
}
func TestAnUnreachableSourceIsRefusedBeforeTheCatalog(t *testing.T) {
// A port nothing is listening on: refused at once rather than after a
// timeout, and the same shape of failure as a mirror that is down.
srv := httptest.NewServer(http.HandlerFunc(
func(http.ResponseWriter, *http.Request) {}))
dead := srv.URL
srv.Close()
cat := &fakeCatalog{}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {dead + "/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if len(cat.added) != 0 {
t.Error("a model whose weights are not there should not reach the catalog")
}
// The address, so the operator can see which one it could not reach; the
// page used to accept this and leave the failure to a node, minutes later.
if !strings.Contains(body, "cannot reach") {
t.Errorf("the reason should name the source; got %s", body)
}
}
func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) {
gone := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "gone", http.StatusNotFound) }))
t.Cleanup(gone.Close)
cat := &fakeCatalog{}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {gone.URL + "/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if len(cat.added) != 0 {
t.Error("a file removed between the listing and the click should be caught")
}
if !strings.Contains(body, "404") {
t.Errorf("the server's answer should be shown; got %s", body)
}
}