CI's test job has failed intermittently — two of three runs on some days — while every run here passed, including six repetitions on one CPU under the race detector with eight packages at once. The difference is a shared, loaded Linux runner, and the tests most exposed to it are the ones that sleep a fixed time and then assert: - admission: idle admission bounded at 50ms of wall clock against a 1ms tick, and a queued request checked for after a 50ms sleep; - scrape: a 10ms scrape timeout, which on a slow runner reports the engine unreachable and then the violation a second time, in a test that counts one; - supervisor: log lines, three lines of output and a removed state file each checked after a fixed sleep, and an exited worker bounded at 1.5s; - agent: a heartbeat checked for after four intervals. Each now waits for its condition up to a deadline of seconds, and the two wall-clock bounds are set against what they exist to distinguish — the admission queue's tick raised to ten seconds, the exited worker bounded below the five-second grace it must not wait out — rather than against how fast this laptop is. Without the CI logs this cannot be proven to be the failure; these are the tests that fail when the machine is slow, and they no longer do. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
37a80a9b44
commit
b661bfead8
4 files changed
+57
-17
No files matched your search
@@ -400,11 +400,16 @@ func TestHeartbeatKeepsNodeAlive(t *testing.T) {
|
||||
n, _ := h.inv.Get("n1")
|
||||
first := n.LastSeen
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
n, _ = h.inv.Get("n1")
|
||||
if !n.LastSeen.After(first) {
|
||||
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
||||
// Waited for rather than slept through: four heartbeat intervals was not
|
||||
// always enough on a loaded runner for one to have landed.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if n, _ = h.inv.Get("n1"); n.LastSeen.After(first) {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
||||
}
|
||||
|
||||
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user