Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s

Two defects, both of the same kind: something the CLI knew and the rest of
the system did not.

A profile can be edited while a replica is running against the old one. The
reconciler deliberately leaves it alone — churning a replica that is serving
costs more than the imperfection — but there was no way to see it, so the
cluster could disagree with its own configuration indefinitely and read as
healthy. Profile now carries a fingerprint, a replica records the one it was
placed with, and both the API and `ctl replicas` say "stale profile" when
they differ. An empty fingerprint is not stale: it means a replica adopted
from an older agent, which is unknown rather than wrong.

Adding a model from the admin page skipped the source check `ctl model add`
has done all along, so a typo or a mirror that is down became a replica
failing on a node minutes later, with an error naming the node. The check
moves out of the CLI into a package both use.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 22:42:26 +02:00
1 parent 832d0a652c
commit 40f5c930c4
14 files changed
+281 -49

No files matched your search

+35 -3
View File
@@ -123,9 +123,16 @@ type ModelView struct {
// ReplicaView is a replica as `ctl replicas` shows it.
type ReplicaView struct {
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
// Stale is true when the profile has been edited since this replica was
// placed, so it is running a definition the catalog no longer holds.
//
// Reported rather than acted on. Moving a replica that is serving costs
// more than the imperfection, and the operator who made the edit is the
// one who should decide whether it is worth a restart.
Stale bool `json:"stale,omitempty"`
State string `json:"state"`
Pool string `json:"pool,omitempty"`
Fabric string `json:"fabric,omitempty"`
@@ -290,6 +297,7 @@ func (a *API) Replicas(model string) []ReplicaView {
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
Endpoint: rep.Endpoint, Err: rep.Err,
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
IdleSec: -1,
}
@@ -675,3 +683,27 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
func writeError(w http.ResponseWriter, code int, msg string) {
writeJSON(w, code, map[string]string{"error": msg})
}
// profileChanged reports whether a replica is running a profile the catalog no
// longer holds.
//
// A replica with no recorded fingerprint is not stale: that is every replica
// adopted from an agent older than the field, and calling those stale would
// mark a whole fleet as wrong on the first upgrade. Nor is one whose profile
// has been deleted outright — the model or the profile going away is a
// different thing, and the replica's own state already says it.
func (a *API) profileChanged(model, profile, was string) bool {
if was == "" {
return false
}
m, ok := a.ctl.Catalog().Resolve(model)
if !ok {
return false
}
for _, p := range m.Profiles {
if p.Name == profile {
return p.Fingerprint() != was
}
}
return false
}
+45
View File
@@ -622,3 +622,48 @@ func TestABadSinceIsRefusedBeforeTheQuery(t *testing.T) {
t.Error("nothing should have reached the store")
}
}
func TestAReplicaRunningAnEditedProfileIsMarkedStale(t *testing.T) {
models := testModels()
a, ctl := setup(t, models)
// Placed from the profile as it was, then the profile is edited and keeps
// its name — which is what makes the two indistinguishable without a
// fingerprint.
was := models[0].Profiles[0].Fingerprint()
edited := models[0]
edited.Profiles[0].MinMemoryGB = 64
built, err := catalog.New([]catalog.Model{edited})
if err != nil {
t.Fatal(err)
}
ctl.ReplaceCatalog(t.Context(), built)
if !a.profileChanged("llama", "tp8", was) {
t.Error("an edited profile should read as changed")
}
if a.profileChanged("llama", "tp8", edited.Profiles[0].Fingerprint()) {
t.Error("the profile in force is not a change")
}
}
func TestAReplicaWithNoRecordedProfileIsNotStale(t *testing.T) {
a, _ := setup(t, testModels())
// Every replica adopted from an agent older than the field. Marking those
// stale would call a whole fleet wrong on the first upgrade.
if a.profileChanged("llama", "tp8", "") {
t.Error("unknown is not stale")
}
}
func TestAProfileThatIsGoneIsNotReportedAsStale(t *testing.T) {
a, _ := setup(t, testModels())
// The model or the profile disappearing is a different thing, and the
// replica's own state already says it.
if a.profileChanged("llama", "no-such-profile", "abc") {
t.Error("a missing profile is not a changed one")
}
if a.profileChanged("no-such-model", "tp8", "abc") {
t.Error("a missing model is not a changed profile")
}
}