Jobs are served by the leader, and a moment of fitting nowhere is waited out
CI / test (push) Failing after 8m47s

A 3D job leased by the standby control plane, whose router has no endpoints
and which a job replayed in the process never leaves, was told "loading"
for seven minutes while the leader served the jobs behind it. Jobs and
batches are claimed only while leading.

Five jobs asked for while cuda01's agent restarted failed for good as
"cannot be placed": re-registering, the node had no engines for a few
seconds. A job asks again for five minutes before that is its end.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 23:53:33 +02:00
1 parent 9b8ec89247
commit 95c61daf1b
6 files changed
+107 -3

No files matched your search

+4
View File
@@ -46,6 +46,10 @@ APIs change between minors without a deprecation period.
once as "cannot be placed"; it now waits, and a job with it.
- A model evicted for being idle was placed again on the next pass, ahead of
the one it made room for.
- A job or batch leased by a standby control plane, whose router has no
endpoints, waited on "loading" for good; they are served only by the
leader. A job told its model fits nowhere asks again for five minutes
before failing, which an agent re-registering makes true for seconds.
## v0.14.3 — 2026-10-10
+4 -1
View File
@@ -2086,7 +2086,10 @@ curl -X POST http://gateway:8080/v1/3d/generations/job_…/cancel -H "Authorizat
- **How it is served:** the request is kept as it was sent and replayed by
whichever gateway holds the job's lease, through the same path a
client's request takes — limits, admission in the key's band, guardrails,
accounting. It is one usage record under the job's id, tagged
accounting — by the leading control plane, since a standby has nothing
to send it to. A model said to fit nowhere is asked for again for five
minutes before that is the job's error: an agent re-registering says so
for a few seconds. It is one usage record under the job's id, tagged
`job=<id>`, charged like any request; `GET` shows the usage once it is
done. A model still loading, a rate limit or a full queue is waited out
for as long as the job has: nobody is holding a connection, which is the
+1 -1
View File
@@ -39,7 +39,7 @@ func (r *Router) runBatches(ctx context.Context) {
t := time.NewTicker(batchPoll)
defer t.Stop()
for {
for {
for r.leading() {
now := time.Now()
b, ok, err := r.store.ClaimBatch(ctx, owner, now, now.Add(batchLease))
if err != nil {
+61
View File
@@ -299,3 +299,64 @@ func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) {
t.Errorf("job %v", got)
}
}
type countingUnservable struct {
unservableActivator
asked atomic.Int32
}
func (c *countingUnservable) Unservable(model string) (string, bool) {
c.asked.Add(1)
return c.unservableActivator.Unservable(model)
}
func TestAJobSaidToFitNowhereKeepsAskingAWhileBeforeFailing(t *testing.T) {
// An agent restarting re-registers without its engines for a few
// seconds; five 3D jobs asked for in those seconds failed for good.
grace, retry := jobUnplacedGrace, jobUnplacedRetry
jobUnplacedGrace, jobUnplacedRetry = 300*time.Millisecond, 20*time.Millisecond
t.Cleanup(func() { jobUnplacedGrace, jobUnplacedRetry = grace, retry })
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
act := &countingUnservable{unservableActivator: unservableActivator{
fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
why: map[string]string{"sleepy": "sleepy cannot be placed on any node: needs 15 GB"},
}}
rt.SetActivator(act)
j := queueJob(t, rt, "job_u", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
start := time.Now()
rt.wakeJobs()
got := waitJob(t, rt.Handler(), secret, j.ID, jobDone)
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "needs 15 GB") {
t.Errorf("job %v", got)
}
if took := time.Since(start); took < jobUnplacedGrace {
t.Errorf("failed after %s, before the grace of %s", took, jobUnplacedGrace)
}
if n := act.asked.Load(); n < 3 {
t.Errorf("asked %d times in the grace", n)
}
}
func TestAStandbyLeavesJobsToTheLeader(t *testing.T) {
// A standby's router has no endpoints; a 3D job it leased was told
// "loading" for seven minutes while the leader served the jobs behind it.
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
var lead atomic.Bool
rt.SetLeader(lead.Load)
j := queueJob(t, rt, "job_s", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
rt.wakeJobs()
time.Sleep(200 * time.Millisecond)
if got, _ := rt.store.JobByID(t.Context(), j.ID); got.Status != store.JobQueued {
t.Fatalf("a standby took the job: %s", got.Status)
}
lead.Store(true)
rt.wakeJobs()
if got := waitJob(t, rt.Handler(), secret, j.ID, jobDone); got["status"] != "completed" {
t.Errorf("the leader did not serve it: %v", got)
}
}
+24 -1
View File
@@ -29,6 +29,16 @@ const (
jobMaxResult = 256 << 20
)
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
// "Fits nowhere" is said for a moment by a cluster changing under it — an
// agent restarting re-registers without its engines for a few seconds — and
// a job failed by that moment was failed by nothing wrong with it.
var (
jobUnplacedGrace = 5 * time.Minute
jobUnplacedRetry = 15 * time.Second
)
// wakeJobs has the worker look for work now rather than at its next poll.
func (r *Router) wakeJobs() {
select {
@@ -43,7 +53,7 @@ func (r *Router) runJobs(ctx context.Context) {
t := time.NewTicker(jobPoll)
defer t.Stop()
for {
for {
for r.leading() {
now := time.Now()
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
if err != nil {
@@ -152,6 +162,7 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
}()
handler := r.handleInference(j.Endpoint)
var unplaced time.Time // when it was first said to fit nowhere
for attempt := 1; ; attempt++ {
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
if err != nil {
@@ -200,6 +211,18 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
}
continue
}
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
if unplaced.IsZero() {
unplaced = time.Now()
}
if time.Since(unplaced) < jobUnplacedGrace {
select {
case <-rctx.Done():
case <-time.After(jobUnplacedRetry):
}
continue
}
}
if status >= 300 {
return fail(errorMessage(w.buf.Bytes()))
}
+13
View File
@@ -682,6 +682,19 @@ func (r *Router) Stop() {
// SetLeader tells /leader how to know whether this process leads.
func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) }
// leading says whether this process leads, which a process with no rival
// always does.
//
// Jobs and batches are served only where it does. A standby's router has no
// endpoints — its reconciler placed nothing — and a client's request reaches
// the leader by being forwarded at the door, which a job replayed inside the
// process never passes: a 3D job leased by the standby was told "loading"
// for as long as anyone let it, while the leader served the jobs behind it.
func (r *Router) leading() bool {
f := r.isLeader.Load()
return f == nil || (*f)()
}
// Registry exposes the endpoint registry, for tests and admin endpoints.
func (r *Router) Registry() *endpoint.Registry { return r.reg }