On-demand loading worked right up to the moment the cluster was full and then stopped meaning anything: the model somebody had just asked for could not be placed, and the cards were held by a model nobody had asked for in an hour. That is the only situation in which any of it matters. Deliberately narrow. Only idle replicas of other on-demand models — a model at min 1 is a promise that it stays up, and breaking it to serve a different request would make every guarantee in the catalog conditional on load. Only single-node replicas, since evicting a multi-node one frees several machines and costs tens of minutes to rebuild. One at a time, because freeing everything that qualifies would empty a cluster for a request that needed one node. And never a replica with no reported usage: after a router restart that is all of them, and unknown is not idle. Writing the test found a gap the feature had rather than a bug in the test. Two on-demand models on a cluster with room for one would take turns evicting each other, at a model load per cycle — worse for both than one of them waiting. A replica is now safe from eviction for a few minutes after it starts serving, which bounds that. Which in turn found a latent bug: deriveState stamped ReadyAt with time.Now() inside a package that has an injectable clock, so the new lifetime rule could not be tested at all — the fake clock said three minutes and the wall clock said eight months. Invisible while nothing compared ReadyAt to anything. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7fe59bd5fb
commit
9b32531ce3
6 files changed
+412
-8
No files matched your search
@@ -515,6 +515,15 @@ models used daily — keep those at `min: 1`. And stopping a replica throws away
|
||||
the KV cache prefix routing spent its effort filling, so an idle timeout wants
|
||||
to be minutes rather than seconds.
|
||||
|
||||
When the cluster is full, a model somebody asks for will stop an idle one to
|
||||
make room. Only idle replicas of other on-demand models: a model at `min: 1` is
|
||||
a promise that it stays up, and breaking that to serve a different request would
|
||||
make every guarantee in the catalog conditional on load. Only single-node
|
||||
replicas, because evicting a multi-node one frees several machines and costs
|
||||
tens of minutes to rebuild — a decision worth making deliberately. One at a
|
||||
time. And never a replica that has just started, which is what stops two models
|
||||
on a cluster with room for one from taking turns evicting each other.
|
||||
|
||||
For a GPU model, name the class and the width instead:
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
// evictionCandidate is a replica that could be stopped to make room.
|
||||
type evictionCandidate struct {
|
||||
ID string
|
||||
Model string
|
||||
Idle time.Duration
|
||||
GPUs int
|
||||
// Nodes is how many machines the replica spans. More than one is left
|
||||
// alone — see makeRoomFor.
|
||||
Nodes int
|
||||
}
|
||||
|
||||
// makeRoomFor stops an idle replica so a wanted model can be placed.
|
||||
//
|
||||
// Without this, on-demand loading works right up to the moment the cluster is
|
||||
// full and then stops meaning anything: the model somebody just asked for
|
||||
// cannot be placed, and the cards are held by a model nobody has asked for in
|
||||
// an hour. That is the exact situation the feature exists to resolve, and it is
|
||||
// the only situation in which it matters.
|
||||
//
|
||||
// Deliberately conservative, on three counts.
|
||||
//
|
||||
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
|
||||
// stays up, and quietly breaking that to serve a different request would make
|
||||
// every guarantee in the catalog conditional on load.
|
||||
//
|
||||
// Only single-node replicas. Evicting a multi-node replica frees several
|
||||
// machines at once and takes tens of minutes to rebuild, which is a decision
|
||||
// worth making deliberately rather than as a side effect of one request.
|
||||
//
|
||||
// One at a time. Freeing everything that qualifies would empty a cluster to
|
||||
// satisfy a request that may only have needed one node, and the next pass can
|
||||
// always free more.
|
||||
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
||||
candidates := r.evictable(m.Name)
|
||||
if len(candidates) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// The longest idle first. It is the replica least likely to be wanted
|
||||
// again, and the one whose KV cache is worth the least.
|
||||
best := candidates[0]
|
||||
r.log.Info("evicting an idle replica to make room",
|
||||
"stopping", best.ID, "idle_model", best.Model,
|
||||
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
||||
r.stopReplica(ctx, best.ID)
|
||||
return true
|
||||
}
|
||||
|
||||
// evictable lists replicas that could be stopped for a model that needs room,
|
||||
// longest idle first.
|
||||
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
||||
cat := r.cat.Load()
|
||||
now := r.now()
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
var out []evictionCandidate
|
||||
for _, rep := range r.replicas {
|
||||
if rep.Model == wantedBy || !rep.State.Counts() {
|
||||
continue
|
||||
}
|
||||
other, ok := cat.Resolve(rep.Model)
|
||||
if !ok || !other.Scaling.OnDemand() {
|
||||
continue
|
||||
}
|
||||
// Unknown is not idle. A replica the router has said nothing about —
|
||||
// after a restart, that is all of them — must not be evicted on the
|
||||
// strength of a fact nobody has reported.
|
||||
if rep.LastUsedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
idle := now.Sub(rep.LastUsedAt)
|
||||
if idle < r.cfg.EvictionGrace {
|
||||
continue
|
||||
}
|
||||
// A replica that has only just started is left alone however idle it
|
||||
// looks. Two on-demand models on a cluster with room for one would
|
||||
// otherwise take turns evicting each other, paying a model load each
|
||||
// time — and a load is expensive enough that waiting is strictly
|
||||
// better than winning.
|
||||
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
|
||||
continue
|
||||
}
|
||||
nodes := map[string]struct{}{}
|
||||
gpus := 0
|
||||
for _, sh := range rep.Shards {
|
||||
nodes[sh.NodeID] = struct{}{}
|
||||
gpus += len(sh.GPUIndices)
|
||||
}
|
||||
if len(nodes) > 1 {
|
||||
continue
|
||||
}
|
||||
out = append(out, evictionCandidate{
|
||||
ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
|
||||
return out
|
||||
}
|
||||
|
||||
// EvictionReason explains why room could not be made, for placement failures.
|
||||
func (r *Reconciler) EvictionReason(model string) string {
|
||||
if c := r.evictable(model); len(c) > 0 {
|
||||
return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
|
||||
}
|
||||
return "nothing idle enough to stop"
|
||||
}
|
||||
@@ -66,6 +66,21 @@ type Config struct {
|
||||
PortRange [2]int
|
||||
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
||||
MgmtInterface string
|
||||
// EvictionGrace is how long a replica must have gone unasked-for before it
|
||||
// may be stopped to make room for another model.
|
||||
//
|
||||
// Shorter than IdleTimeout on purpose: a replica idle for five minutes is
|
||||
// not worth reclaiming on its own, and is worth reclaiming when something
|
||||
// else needs the cards. Still minutes rather than seconds, because stopping
|
||||
// one throws away the KV cache prefix routing spent filling.
|
||||
EvictionGrace time.Duration
|
||||
// MinLifetime is how long a replica is safe from eviction after it starts
|
||||
// serving, whatever else wants its cards.
|
||||
//
|
||||
// The thing that stops two models on a cluster with room for one from
|
||||
// taking turns evicting each other. A load costs minutes; oscillating at a
|
||||
// load per cycle is worse for both models than one of them waiting.
|
||||
MinLifetime time.Duration
|
||||
// IdleTimeout is how long an on-demand model stays loaded with nobody
|
||||
// asking for it. A model may override it.
|
||||
IdleTimeout time.Duration
|
||||
@@ -99,6 +114,8 @@ func DefaultConfig() Config {
|
||||
LostAfter: 60 * time.Second,
|
||||
SettleFor: 60 * time.Second,
|
||||
IdleTimeout: 15 * time.Minute,
|
||||
EvictionGrace: 2 * time.Minute,
|
||||
MinLifetime: 5 * time.Minute,
|
||||
DrainGrace: 5 * time.Minute,
|
||||
PortRange: [2]int{20000, 20999},
|
||||
MgmtInterface: "eth0",
|
||||
@@ -119,6 +136,12 @@ func (c *Config) normalize() {
|
||||
if c.IdleTimeout <= 0 {
|
||||
c.IdleTimeout = d.IdleTimeout
|
||||
}
|
||||
if c.EvictionGrace <= 0 {
|
||||
c.EvictionGrace = d.EvictionGrace
|
||||
}
|
||||
if c.MinLifetime <= 0 {
|
||||
c.MinLifetime = d.MinLifetime
|
||||
}
|
||||
if c.SettleFor < 0 {
|
||||
c.SettleFor = 0
|
||||
} else if c.SettleFor == 0 {
|
||||
@@ -729,6 +752,17 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
r.mu.Unlock()
|
||||
|
||||
if res.Placement == nil {
|
||||
// Full, with a model somebody has just asked for and cards held by one
|
||||
// nobody has asked for in a while. Stopping one is the whole point of
|
||||
// on-demand loading; without it the feature works until the cluster is
|
||||
// full, which is the only time it matters.
|
||||
//
|
||||
// Not retried here. The replica is stopped, its GPUs are released on
|
||||
// the pass that reaps it, and the next pass places this model — which
|
||||
// keeps the stop-then-start ordering in one place rather than two.
|
||||
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
|
||||
return
|
||||
}
|
||||
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
|
||||
return
|
||||
}
|
||||
@@ -1076,7 +1110,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
}
|
||||
}
|
||||
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
|
||||
rep.deriveState()
|
||||
rep.deriveState(r.now())
|
||||
|
||||
r.log.Info("adopted replica from the fleet",
|
||||
"replica", rep.ID, "model", rep.Model,
|
||||
@@ -1132,7 +1166,7 @@ func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []Ad
|
||||
lost = append(lost, sh.ID)
|
||||
}
|
||||
if changed {
|
||||
rep.deriveState()
|
||||
rep.deriveState(r.now())
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
@@ -1203,7 +1237,7 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
|
||||
}
|
||||
|
||||
prev := rep.State
|
||||
rep.deriveState()
|
||||
rep.deriveState(r.now())
|
||||
if rep.State == Ready && prev != Ready {
|
||||
// A model that comes up clears its backoff; the next failure starts
|
||||
// counting from scratch rather than from an old streak.
|
||||
|
||||
@@ -1431,3 +1431,237 @@ func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) {
|
||||
t.Errorf("an always-on model was evicted for being idle: %d live", live)
|
||||
}
|
||||
}
|
||||
|
||||
// --- eviction under pressure ---
|
||||
|
||||
// twoOnDemandModels builds a catalog where each model fills the whole node, so
|
||||
// the second can only run if the first stops.
|
||||
func twoOnDemandModels(t *testing.T) *catalog.Catalog {
|
||||
t.Helper()
|
||||
mk := func(name string) catalog.Model {
|
||||
m := testModel(1)
|
||||
m.Name = name
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: time.Hour}
|
||||
return m
|
||||
}
|
||||
built, err := catalog.New([]catalog.Model{mk("llama"), mk("second")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return built
|
||||
}
|
||||
|
||||
func TestAWantedModelEvictsAnIdleOneWhenTheClusterIsFull(t *testing.T) {
|
||||
// On-demand loading works right up to the moment the cluster is full and
|
||||
// then stops meaning anything: the model somebody just asked for cannot be
|
||||
// placed, and the cards are held by one nobody has asked for in an hour.
|
||||
// That is the only situation in which any of this matters.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
first := h.r.Replicas()
|
||||
if len(first) != 1 {
|
||||
t.Fatalf("the first model did not start: %d replicas", len(first))
|
||||
}
|
||||
h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()})
|
||||
|
||||
// It goes quiet. Past the eviction grace and the minimum lifetime, so it is
|
||||
// genuinely a candidate rather than merely recent.
|
||||
h.advance(30 * time.Minute)
|
||||
|
||||
// Asked for directly, rather than through a Reconcile pass, because two
|
||||
// on-demand models both wanting the only free node is a race the scheduler
|
||||
// resolves in whatever order the catalog iterates — and what is being
|
||||
// tested here is that room gets made, not who wins.
|
||||
second := mustResolve(t, h, "second")
|
||||
if !h.r.makeRoomFor(t.Context(), second) {
|
||||
t.Fatal("no room was made for a model that could not be placed")
|
||||
}
|
||||
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.Model == "llama" && rep.State.Counts() {
|
||||
t.Error("the idle replica was not evicted")
|
||||
}
|
||||
}
|
||||
// And its GPUs are back, which is what the wanted model needed.
|
||||
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
||||
t.Errorf("node has %d free GPUs after the eviction, want 4", free)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementFailureTriggersEviction(t *testing.T) {
|
||||
// The wiring: scaleUp is what notices there is nowhere to put a model, and
|
||||
// is the only place that knows to try making room.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("setup: %d replicas", len(reps))
|
||||
}
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(30 * time.Minute)
|
||||
|
||||
// The full node has no room, so placing this must free some.
|
||||
h.r.scaleUp(t.Context(), mustResolve(t, h, "second"), nil)
|
||||
|
||||
if free := len(h.inv.FreeGPUs("1")); free == 0 {
|
||||
t.Error("a placement failure did not make room")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAlwaysOnModelIsNeverEvicted(t *testing.T) {
|
||||
// min 1 is a promise that it stays up. Quietly breaking it to serve another
|
||||
// request would make every guarantee in the catalog conditional on load.
|
||||
alwaysOn := testModel(1) // min 1
|
||||
onDemand := testModel(1)
|
||||
onDemand.Name = "second"
|
||||
onDemand.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
||||
built, err := catalog.New([]catalog.Model{alwaysOn, onDemand})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := newHarness(t, alwaysOn, gpuNode("1", 4))
|
||||
h.r.cat.Store(built)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("the always-on model did not start")
|
||||
}
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(24 * time.Hour)
|
||||
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Errorf("an always-on model was offered for eviction: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaNothingHasReportedIsNeverEvicted(t *testing.T) {
|
||||
// Unknown is not idle. After a router restart that is every replica, and
|
||||
// evicting on the strength of a fact nobody has reported would empty the
|
||||
// cluster at exactly the wrong moment.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
|
||||
h.advance(24 * time.Hour) // idle by any measure, but nothing has said so
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Errorf("a replica with no reported usage was offered for eviction: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMultiNodeReplicaIsLeftAlone(t *testing.T) {
|
||||
// Evicting one frees several machines at once and takes tens of minutes to
|
||||
// rebuild — a decision worth making deliberately rather than as a side
|
||||
// effect of one request.
|
||||
m := testModel(1, catalog.Profile{
|
||||
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
||||
})
|
||||
m.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
||||
other := testModel(1)
|
||||
other.Name = "second"
|
||||
other.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
||||
built, err := catalog.New([]catalog.Model{m, other})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
||||
h.r.cat.Store(built)
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
||||
t.Fatalf("expected a two-node replica, got %+v", reps)
|
||||
}
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(24 * time.Hour)
|
||||
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Errorf("a multi-node replica was offered for eviction: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyOneReplicaIsEvictedPerPass(t *testing.T) {
|
||||
// Freeing everything that qualifies would empty a cluster to satisfy a
|
||||
// request that may only have needed one node.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Want("llama")
|
||||
for range 3 {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
reps := h.r.Replicas()
|
||||
used := map[string]time.Time{}
|
||||
for _, rep := range reps {
|
||||
used[rep.ID] = h.clock.now()
|
||||
}
|
||||
h.r.RecordUsage(used)
|
||||
h.advance(time.Hour)
|
||||
|
||||
before := len(h.r.evictable("second"))
|
||||
if before < 1 {
|
||||
t.Skip("nothing evictable to begin with")
|
||||
}
|
||||
h.r.makeRoomFor(t.Context(), mustResolve(t, h, "second"))
|
||||
|
||||
var stopped int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if !rep.State.Counts() {
|
||||
stopped++
|
||||
}
|
||||
}
|
||||
if stopped > 1 {
|
||||
t.Errorf("%d replicas were stopped in one pass", stopped)
|
||||
}
|
||||
}
|
||||
|
||||
func mustResolve(t *testing.T, h *harness, name string) catalog.Model {
|
||||
t.Helper()
|
||||
m, ok := h.r.cat.Load().Resolve(name)
|
||||
if !ok {
|
||||
t.Fatalf("model %q is not in the catalog", name)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
|
||||
// What stops two models on a cluster with room for one from taking turns
|
||||
// evicting each other, paying a model load each time. A load is expensive
|
||||
// enough that waiting is strictly better than winning.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("setup: %d replicas", len(reps))
|
||||
}
|
||||
|
||||
// Used once, then idle well past the eviction grace — but only just
|
||||
// started, which is the case this protects.
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(DefaultConfig().EvictionGrace + time.Minute)
|
||||
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Errorf("a replica younger than the minimum lifetime was offered: %+v", got)
|
||||
}
|
||||
|
||||
// Once it has been up long enough, it is fair game.
|
||||
h.advance(DefaultConfig().MinLifetime)
|
||||
if got := h.r.evictable("second"); len(got) == 0 {
|
||||
t.Error("a long-running idle replica was never offered for eviction")
|
||||
}
|
||||
}
|
||||
@@ -157,7 +157,14 @@ func (r *Replica) hasLiveShard() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *Replica) deriveState() {
|
||||
// deriveState recomputes the replica's state from its shards.
|
||||
//
|
||||
// Takes the time rather than reading the clock, because ReadyAt is now
|
||||
// something decisions are made against — a replica is safe from eviction for a
|
||||
// while after it starts serving — and a field stamped from the wall clock
|
||||
// inside a package with an injectable one cannot be reasoned about in a test.
|
||||
// It was invisible while nothing compared ReadyAt to anything.
|
||||
func (r *Replica) deriveState(now time.Time) {
|
||||
if r.State == Draining || r.State == Stopped {
|
||||
return
|
||||
}
|
||||
@@ -198,7 +205,7 @@ func (r *Replica) deriveState() {
|
||||
}
|
||||
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
|
||||
if r.State != Ready {
|
||||
r.ReadyAt = time.Now()
|
||||
r.ReadyAt = now
|
||||
}
|
||||
r.State = Ready
|
||||
return
|
||||
|
||||
@@ -350,7 +350,7 @@ func TestAReplicaMissingAShardIsNotReady(t *testing.T) {
|
||||
ID: "r1", WantShards: 2,
|
||||
Shards: []Shard{{ID: "r1-r1", Rank: 1, State: supervisor.Ready}},
|
||||
}
|
||||
rep.deriveState()
|
||||
rep.deriveState(time.Now())
|
||||
|
||||
if rep.State == Ready {
|
||||
t.Fatal("a replica with one of two shards is Ready")
|
||||
@@ -368,7 +368,7 @@ func TestACompleteReplicaIsReady(t *testing.T) {
|
||||
{ID: "r1-r1", Rank: 1, State: supervisor.Ready},
|
||||
},
|
||||
}
|
||||
rep.deriveState()
|
||||
rep.deriveState(time.Now())
|
||||
if rep.State != Ready {
|
||||
t.Errorf("state = %s, want ready", rep.State)
|
||||
}
|
||||
@@ -383,7 +383,7 @@ func TestAReplicaStillComingUpIsNotFailed(t *testing.T) {
|
||||
{ID: "r1-r1", Rank: 1, State: supervisor.Loading},
|
||||
},
|
||||
}
|
||||
rep.deriveState()
|
||||
rep.deriveState(time.Now())
|
||||
if rep.State != Starting {
|
||||
t.Errorf("state = %s, want starting", rep.State)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user