Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s

Two defects, both of the same kind: something the CLI knew and the rest of
the system did not.

A profile can be edited while a replica is running against the old one. The
reconciler deliberately leaves it alone — churning a replica that is serving
costs more than the imperfection — but there was no way to see it, so the
cluster could disagree with its own configuration indefinitely and read as
healthy. Profile now carries a fingerprint, a replica records the one it was
placed with, and both the API and `ctl replicas` say "stale profile" when
they differ. An empty fingerprint is not stale: it means a replica adopted
from an older agent, which is unknown rather than wrong.

Adding a model from the admin page skipped the source check `ctl model add`
has done all along, so a typo or a mirror that is down became a replica
failing on a node minutes later, with an error naming the node. The check
moves out of the CLI into a package both use.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 22:42:26 +02:00
1 parent 832d0a652c
commit 40f5c930c4
14 files changed
+281 -49

No files matched your search

+35 -3
View File
@@ -123,9 +123,16 @@ type ModelView struct {
// ReplicaView is a replica as `ctl replicas` shows it.
type ReplicaView struct {
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
// Stale is true when the profile has been edited since this replica was
// placed, so it is running a definition the catalog no longer holds.
//
// Reported rather than acted on. Moving a replica that is serving costs
// more than the imperfection, and the operator who made the edit is the
// one who should decide whether it is worth a restart.
Stale bool `json:"stale,omitempty"`
State string `json:"state"`
Pool string `json:"pool,omitempty"`
Fabric string `json:"fabric,omitempty"`
@@ -290,6 +297,7 @@ func (a *API) Replicas(model string) []ReplicaView {
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
Endpoint: rep.Endpoint, Err: rep.Err,
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
IdleSec: -1,
}
@@ -675,3 +683,27 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
func writeError(w http.ResponseWriter, code int, msg string) {
writeJSON(w, code, map[string]string{"error": msg})
}
// profileChanged reports whether a replica is running a profile the catalog no
// longer holds.
//
// A replica with no recorded fingerprint is not stale: that is every replica
// adopted from an agent older than the field, and calling those stale would
// mark a whole fleet as wrong on the first upgrade. Nor is one whose profile
// has been deleted outright — the model or the profile going away is a
// different thing, and the replica's own state already says it.
func (a *API) profileChanged(model, profile, was string) bool {
if was == "" {
return false
}
m, ok := a.ctl.Catalog().Resolve(model)
if !ok {
return false
}
for _, p := range m.Profiles {
if p.Name == profile {
return p.Fingerprint() != was
}
}
return false
}
+45
View File
@@ -622,3 +622,48 @@ func TestABadSinceIsRefusedBeforeTheQuery(t *testing.T) {
t.Error("nothing should have reached the store")
}
}
func TestAReplicaRunningAnEditedProfileIsMarkedStale(t *testing.T) {
models := testModels()
a, ctl := setup(t, models)
// Placed from the profile as it was, then the profile is edited and keeps
// its name — which is what makes the two indistinguishable without a
// fingerprint.
was := models[0].Profiles[0].Fingerprint()
edited := models[0]
edited.Profiles[0].MinMemoryGB = 64
built, err := catalog.New([]catalog.Model{edited})
if err != nil {
t.Fatal(err)
}
ctl.ReplaceCatalog(t.Context(), built)
if !a.profileChanged("llama", "tp8", was) {
t.Error("an edited profile should read as changed")
}
if a.profileChanged("llama", "tp8", edited.Profiles[0].Fingerprint()) {
t.Error("the profile in force is not a change")
}
}
func TestAReplicaWithNoRecordedProfileIsNotStale(t *testing.T) {
a, _ := setup(t, testModels())
// Every replica adopted from an agent older than the field. Marking those
// stale would call a whole fleet wrong on the first upgrade.
if a.profileChanged("llama", "tp8", "") {
t.Error("unknown is not stale")
}
}
func TestAProfileThatIsGoneIsNotReportedAsStale(t *testing.T) {
a, _ := setup(t, testModels())
// The model or the profile disappearing is a different thing, and the
// replica's own state already says it.
if a.profileChanged("llama", "no-such-profile", "abc") {
t.Error("a missing profile is not a changed one")
}
if a.profileChanged("no-such-model", "tp8", "abc") {
t.Error("a missing model is not a changed profile")
}
}
+24
View File
@@ -3,6 +3,9 @@
package catalog
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"strings"
@@ -444,3 +447,24 @@ func (m Model) Simulated() bool {
}
return true
}
// Fingerprint identifies a profile's contents.
//
// The name is not enough. A profile can be edited — a memory requirement
// raised, a node label added, an engine argument changed — and keep its name,
// so a replica running the old definition is indistinguishable from one
// running the new. The reconciler does not move a running replica when its
// profile changes, which is right: churning a replica that is serving costs
// more than the imperfection. What was missing is any way to see that it had
// happened, and the answer to "why is this one on the wrong node" was in
// somebody's memory of what they edited an hour ago.
func (p Profile) Fingerprint() string {
b, err := json.Marshal(p)
if err != nil {
// A profile that will not marshal cannot be compared, and saying so by
// never matching is better than saying two differing ones are the same.
return "unmarshalable"
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
+1 -1
View File
@@ -812,7 +812,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name,
ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(),
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
WantShards: len(pl.Nodes),
+12
View File
@@ -104,6 +104,18 @@ type Replica struct {
ID string
Model string
Profile string
// ProfileFP is the profile's contents when this replica was placed.
//
// Compared with the catalog's to tell a replica running the definition in
// force from one running whatever it was edited from. A running replica is
// deliberately not moved when its profile changes — churn costs more than
// the imperfection — so without this there is nothing that says the two
// have diverged, and "why is this one on the wrong node" is answered from
// memory of what somebody edited an hour ago.
//
// Empty on a replica adopted from an agent that predates it, which reads
// as unknown rather than as stale.
ProfileFP string
// Fabric is set only for multi-node replicas.
Fabric string
Pool string