From 95c61daf1be345e48f930752fe5cb93c76a20521 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 10 Oct 2026 23:53:09 +0200 Subject: [PATCH] Jobs are served by the leader, and a moment of fitting nowhere is waited out A 3D job leased by the standby control plane, whose router has no endpoints and which a job replayed in the process never leaves, was told "loading" for seven minutes while the leader served the jobs behind it. Jobs and batches are claimed only while leading. Five jobs asked for while cuda01's agent restarted failed for good as "cannot be placed": re-registering, the node had no engines for a few seconds. A job asks again for five minutes before that is its end. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 +++ docs/install.md | 5 ++- internal/router/batchworker.go | 2 +- internal/router/jobs_test.go | 61 ++++++++++++++++++++++++++++++++++ internal/router/jobworker.go | 25 +++++++++++++- internal/router/router.go | 13 ++++++++ 6 files changed, 107 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab84c0..d2ca1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,10 @@ APIs change between minors without a deprecation period. once as "cannot be placed"; it now waits, and a job with it. - A model evicted for being idle was placed again on the next pass, ahead of the one it made room for. +- A job or batch leased by a standby control plane, whose router has no + endpoints, waited on "loading" for good; they are served only by the + leader. A job told its model fits nowhere asks again for five minutes + before failing, which an agent re-registering makes true for seconds. ## v0.14.3 — 2026-10-10 diff --git a/docs/install.md b/docs/install.md index 6efe44e..1437276 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2086,7 +2086,10 @@ curl -X POST http://gateway:8080/v1/3d/generations/job_…/cancel -H "Authorizat - **How it is served:** the request is kept as it was sent and replayed by whichever gateway holds the job's lease, through the same path a client's request takes — limits, admission in the key's band, guardrails, - accounting. It is one usage record under the job's id, tagged + accounting — by the leading control plane, since a standby has nothing + to send it to. A model said to fit nowhere is asked for again for five + minutes before that is the job's error: an agent re-registering says so + for a few seconds. It is one usage record under the job's id, tagged `job=`, charged like any request; `GET` shows the usage once it is done. A model still loading, a rate limit or a full queue is waited out for as long as the job has: nobody is holding a connection, which is the diff --git a/internal/router/batchworker.go b/internal/router/batchworker.go index 381c74c..1da0e1c 100644 --- a/internal/router/batchworker.go +++ b/internal/router/batchworker.go @@ -39,7 +39,7 @@ func (r *Router) runBatches(ctx context.Context) { t := time.NewTicker(batchPoll) defer t.Stop() for { - for { + for r.leading() { now := time.Now() b, ok, err := r.store.ClaimBatch(ctx, owner, now, now.Add(batchLease)) if err != nil { diff --git a/internal/router/jobs_test.go b/internal/router/jobs_test.go index d79bd61..a9faf49 100644 --- a/internal/router/jobs_test.go +++ b/internal/router/jobs_test.go @@ -299,3 +299,64 @@ func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) { t.Errorf("job %v", got) } } + +type countingUnservable struct { + unservableActivator + asked atomic.Int32 +} + +func (c *countingUnservable) Unservable(model string) (string, bool) { + c.asked.Add(1) + return c.unservableActivator.Unservable(model) +} + +func TestAJobSaidToFitNowhereKeepsAskingAWhileBeforeFailing(t *testing.T) { + // An agent restarting re-registers without its engines for a few + // seconds; five 3D jobs asked for in those seconds failed for good. + grace, retry := jobUnplacedGrace, jobUnplacedRetry + jobUnplacedGrace, jobUnplacedRetry = 300*time.Millisecond, 20*time.Millisecond + t.Cleanup(func() { jobUnplacedGrace, jobUnplacedRetry = grace, retry }) + + c := newJobComfy(t) + rt, secret := jobRouter(t, c) + act := &countingUnservable{unservableActivator: unservableActivator{ + fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}}, + why: map[string]string{"sleepy": "sleepy cannot be placed on any node: needs 15 GB"}, + }} + rt.SetActivator(act) + j := queueJob(t, rt, "job_u", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`) + start := time.Now() + rt.wakeJobs() + + got := waitJob(t, rt.Handler(), secret, j.ID, jobDone) + if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "needs 15 GB") { + t.Errorf("job %v", got) + } + if took := time.Since(start); took < jobUnplacedGrace { + t.Errorf("failed after %s, before the grace of %s", took, jobUnplacedGrace) + } + if n := act.asked.Load(); n < 3 { + t.Errorf("asked %d times in the grace", n) + } +} + +func TestAStandbyLeavesJobsToTheLeader(t *testing.T) { + // A standby's router has no endpoints; a 3D job it leased was told + // "loading" for seven minutes while the leader served the jobs behind it. + c := newJobComfy(t) + rt, secret := jobRouter(t, c) + var lead atomic.Bool + rt.SetLeader(lead.Load) + j := queueJob(t, rt, "job_s", store.JobQueued, `{"model":"flux","prompt":"a cube"}`) + rt.wakeJobs() + time.Sleep(200 * time.Millisecond) + if got, _ := rt.store.JobByID(t.Context(), j.ID); got.Status != store.JobQueued { + t.Fatalf("a standby took the job: %s", got.Status) + } + + lead.Store(true) + rt.wakeJobs() + if got := waitJob(t, rt.Handler(), secret, j.ID, jobDone); got["status"] != "completed" { + t.Errorf("the leader did not serve it: %v", got) + } +} diff --git a/internal/router/jobworker.go b/internal/router/jobworker.go index 102ed32..af7abf4 100644 --- a/internal/router/jobworker.go +++ b/internal/router/jobworker.go @@ -29,6 +29,16 @@ const ( jobMaxResult = 256 << 20 ) +// jobUnplacedGrace is how long a job keeps asking for a model said to fit +// nowhere before that is its end, and jobUnplacedRetry how often it asks. +// "Fits nowhere" is said for a moment by a cluster changing under it — an +// agent restarting re-registers without its engines for a few seconds — and +// a job failed by that moment was failed by nothing wrong with it. +var ( + jobUnplacedGrace = 5 * time.Minute + jobUnplacedRetry = 15 * time.Second +) + // wakeJobs has the worker look for work now rather than at its next poll. func (r *Router) wakeJobs() { select { @@ -43,7 +53,7 @@ func (r *Router) runJobs(ctx context.Context) { t := time.NewTicker(jobPoll) defer t.Stop() for { - for { + for r.leading() { now := time.Now() j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease)) if err != nil { @@ -152,6 +162,7 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error { }() handler := r.handleInference(j.Endpoint) + var unplaced time.Time // when it was first said to fit nowhere for attempt := 1; ; attempt++ { hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input)) if err != nil { @@ -200,6 +211,18 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error { } continue } + if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) { + if unplaced.IsZero() { + unplaced = time.Now() + } + if time.Since(unplaced) < jobUnplacedGrace { + select { + case <-rctx.Done(): + case <-time.After(jobUnplacedRetry): + } + continue + } + } if status >= 300 { return fail(errorMessage(w.buf.Bytes())) } diff --git a/internal/router/router.go b/internal/router/router.go index 17060df..cfea90c 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -682,6 +682,19 @@ func (r *Router) Stop() { // SetLeader tells /leader how to know whether this process leads. func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) } +// leading says whether this process leads, which a process with no rival +// always does. +// +// Jobs and batches are served only where it does. A standby's router has no +// endpoints — its reconciler placed nothing — and a client's request reaches +// the leader by being forwarded at the door, which a job replayed inside the +// process never passes: a 3D job leased by the standby was told "loading" +// for as long as anyone let it, while the leader served the jobs behind it. +func (r *Router) leading() bool { + f := r.isLeader.Load() + return f == nil || (*f)() +} + // Registry exposes the endpoint registry, for tests and admin endpoints. func (r *Router) Registry() *endpoint.Registry { return r.reg }