Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s
CI / test (push) Successful in 1m43s
Two defects, both of the same kind: something the CLI knew and the rest of the system did not. A profile can be edited while a replica is running against the old one. The reconciler deliberately leaves it alone — churning a replica that is serving costs more than the imperfection — but there was no way to see it, so the cluster could disagree with its own configuration indefinitely and read as healthy. Profile now carries a fingerprint, a replica records the one it was placed with, and both the API and `ctl replicas` say "stale profile" when they differ. An empty fingerprint is not stale: it means a replica adopted from an older agent, which is unknown rather than wrong. Adding a model from the admin page skipped the source check `ctl model add` has done all along, so a typo or a mirror that is down became a replica failing on a node minutes later, with an error naming the node. The check moves out of the CLI into a package both use. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
832d0a652c
commit
40f5c930c4
14 files changed
+281
-49
No files matched your search
@@ -123,9 +123,16 @@ type ModelView struct {
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
type ReplicaView struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
// Stale is true when the profile has been edited since this replica was
|
||||
// placed, so it is running a definition the catalog no longer holds.
|
||||
//
|
||||
// Reported rather than acted on. Moving a replica that is serving costs
|
||||
// more than the imperfection, and the operator who made the edit is the
|
||||
// one who should decide whether it is worth a restart.
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
State string `json:"state"`
|
||||
Pool string `json:"pool,omitempty"`
|
||||
Fabric string `json:"fabric,omitempty"`
|
||||
@@ -290,6 +297,7 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
|
||||
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
|
||||
Endpoint: rep.Endpoint, Err: rep.Err,
|
||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||
IdleSec: -1,
|
||||
}
|
||||
@@ -675,3 +683,27 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
|
||||
func writeError(w http.ResponseWriter, code int, msg string) {
|
||||
writeJSON(w, code, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
// profileChanged reports whether a replica is running a profile the catalog no
|
||||
// longer holds.
|
||||
//
|
||||
// A replica with no recorded fingerprint is not stale: that is every replica
|
||||
// adopted from an agent older than the field, and calling those stale would
|
||||
// mark a whole fleet as wrong on the first upgrade. Nor is one whose profile
|
||||
// has been deleted outright — the model or the profile going away is a
|
||||
// different thing, and the replica's own state already says it.
|
||||
func (a *API) profileChanged(model, profile, was string) bool {
|
||||
if was == "" {
|
||||
return false
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(model)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
if p.Name == profile {
|
||||
return p.Fingerprint() != was
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -622,3 +622,48 @@ func TestABadSinceIsRefusedBeforeTheQuery(t *testing.T) {
|
||||
t.Error("nothing should have reached the store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaRunningAnEditedProfileIsMarkedStale(t *testing.T) {
|
||||
models := testModels()
|
||||
a, ctl := setup(t, models)
|
||||
|
||||
// Placed from the profile as it was, then the profile is edited and keeps
|
||||
// its name — which is what makes the two indistinguishable without a
|
||||
// fingerprint.
|
||||
was := models[0].Profiles[0].Fingerprint()
|
||||
edited := models[0]
|
||||
edited.Profiles[0].MinMemoryGB = 64
|
||||
built, err := catalog.New([]catalog.Model{edited})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl.ReplaceCatalog(t.Context(), built)
|
||||
|
||||
if !a.profileChanged("llama", "tp8", was) {
|
||||
t.Error("an edited profile should read as changed")
|
||||
}
|
||||
if a.profileChanged("llama", "tp8", edited.Profiles[0].Fingerprint()) {
|
||||
t.Error("the profile in force is not a change")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaWithNoRecordedProfileIsNotStale(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
// Every replica adopted from an agent older than the field. Marking those
|
||||
// stale would call a whole fleet wrong on the first upgrade.
|
||||
if a.profileChanged("llama", "tp8", "") {
|
||||
t.Error("unknown is not stale")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileThatIsGoneIsNotReportedAsStale(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
// The model or the profile disappearing is a different thing, and the
|
||||
// replica's own state already says it.
|
||||
if a.profileChanged("llama", "no-such-profile", "abc") {
|
||||
t.Error("a missing profile is not a changed one")
|
||||
}
|
||||
if a.profileChanged("no-such-model", "tp8", "abc") {
|
||||
t.Error("a missing model is not a changed profile")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user