diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab84c0..d2ca1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,10 @@ APIs change between minors without a deprecation period. once as "cannot be placed"; it now waits, and a job with it. - A model evicted for being idle was placed again on the next pass, ahead of the one it made room for. +- A job or batch leased by a standby control plane, whose router has no + endpoints, waited on "loading" for good; they are served only by the + leader. A job told its model fits nowhere asks again for five minutes + before failing, which an agent re-registering makes true for seconds. ## v0.14.3 — 2026-10-10 diff --git a/docs/install.md b/docs/install.md index 6efe44e..1437276 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2086,7 +2086,10 @@ curl -X POST http://gateway:8080/v1/3d/generations/job_…/cancel -H "Authorizat - **How it is served:** the request is kept as it was sent and replayed by whichever gateway holds the job's lease, through the same path a client's request takes — limits, admission in the key's band, guardrails, - accounting. It is one usage record under the job's id, tagged + accounting — by the leading control plane, since a standby has nothing + to send it to. A model said to fit nowhere is asked for again for five + minutes before that is the job's error: an agent re-registering says so + for a few seconds. It is one usage record under the job's id, tagged `job=`, charged like any request; `GET` shows the usage once it is done. A model still loading, a rate limit or a full queue is waited out for as long as the job has: nobody is holding a connection, which is the diff --git a/internal/router/batchworker.go b/internal/router/batchworker.go index 381c74c..1da0e1c 100644 --- a/internal/router/batchworker.go +++ b/internal/router/batchworker.go @@ -39,7 +39,7 @@ func (r *Router) runBatches(ctx context.Context) { t := time.NewTicker(batchPoll) defer t.Stop() for { - for { + for r.leading() { now := time.Now() b, ok, err := r.store.ClaimBatch(ctx, owner, now, now.Add(batchLease)) if err != nil { diff --git a/internal/router/jobs_test.go b/internal/router/jobs_test.go index d79bd61..a9faf49 100644 --- a/internal/router/jobs_test.go +++ b/internal/router/jobs_test.go @@ -299,3 +299,64 @@ func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) { t.Errorf("job %v", got) } } + +type countingUnservable struct { + unservableActivator + asked atomic.Int32 +} + +func (c *countingUnservable) Unservable(model string) (string, bool) { + c.asked.Add(1) + return c.unservableActivator.Unservable(model) +} + +func TestAJobSaidToFitNowhereKeepsAskingAWhileBeforeFailing(t *testing.T) { + // An agent restarting re-registers without its engines for a few + // seconds; five 3D jobs asked for in those seconds failed for good. + grace, retry := jobUnplacedGrace, jobUnplacedRetry + jobUnplacedGrace, jobUnplacedRetry = 300*time.Millisecond, 20*time.Millisecond + t.Cleanup(func() { jobUnplacedGrace, jobUnplacedRetry = grace, retry }) + + c := newJobComfy(t) + rt, secret := jobRouter(t, c) + act := &countingUnservable{unservableActivator: unservableActivator{ + fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}}, + why: map[string]string{"sleepy": "sleepy cannot be placed on any node: needs 15 GB"}, + }} + rt.SetActivator(act) + j := queueJob(t, rt, "job_u", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`) + start := time.Now() + rt.wakeJobs() + + got := waitJob(t, rt.Handler(), secret, j.ID, jobDone) + if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "needs 15 GB") { + t.Errorf("job %v", got) + } + if took := time.Since(start); took < jobUnplacedGrace { + t.Errorf("failed after %s, before the grace of %s", took, jobUnplacedGrace) + } + if n := act.asked.Load(); n < 3 { + t.Errorf("asked %d times in the grace", n) + } +} + +func TestAStandbyLeavesJobsToTheLeader(t *testing.T) { + // A standby's router has no endpoints; a 3D job it leased was told + // "loading" for seven minutes while the leader served the jobs behind it. + c := newJobComfy(t) + rt, secret := jobRouter(t, c) + var lead atomic.Bool + rt.SetLeader(lead.Load) + j := queueJob(t, rt, "job_s", store.JobQueued, `{"model":"flux","prompt":"a cube"}`) + rt.wakeJobs() + time.Sleep(200 * time.Millisecond) + if got, _ := rt.store.JobByID(t.Context(), j.ID); got.Status != store.JobQueued { + t.Fatalf("a standby took the job: %s", got.Status) + } + + lead.Store(true) + rt.wakeJobs() + if got := waitJob(t, rt.Handler(), secret, j.ID, jobDone); got["status"] != "completed" { + t.Errorf("the leader did not serve it: %v", got) + } +} diff --git a/internal/router/jobworker.go b/internal/router/jobworker.go index 102ed32..af7abf4 100644 --- a/internal/router/jobworker.go +++ b/internal/router/jobworker.go @@ -29,6 +29,16 @@ const ( jobMaxResult = 256 << 20 ) +// jobUnplacedGrace is how long a job keeps asking for a model said to fit +// nowhere before that is its end, and jobUnplacedRetry how often it asks. +// "Fits nowhere" is said for a moment by a cluster changing under it — an +// agent restarting re-registers without its engines for a few seconds — and +// a job failed by that moment was failed by nothing wrong with it. +var ( + jobUnplacedGrace = 5 * time.Minute + jobUnplacedRetry = 15 * time.Second +) + // wakeJobs has the worker look for work now rather than at its next poll. func (r *Router) wakeJobs() { select { @@ -43,7 +53,7 @@ func (r *Router) runJobs(ctx context.Context) { t := time.NewTicker(jobPoll) defer t.Stop() for { - for { + for r.leading() { now := time.Now() j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease)) if err != nil { @@ -152,6 +162,7 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error { }() handler := r.handleInference(j.Endpoint) + var unplaced time.Time // when it was first said to fit nowhere for attempt := 1; ; attempt++ { hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input)) if err != nil { @@ -200,6 +211,18 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error { } continue } + if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) { + if unplaced.IsZero() { + unplaced = time.Now() + } + if time.Since(unplaced) < jobUnplacedGrace { + select { + case <-rctx.Done(): + case <-time.After(jobUnplacedRetry): + } + continue + } + } if status >= 300 { return fail(errorMessage(w.buf.Bytes())) } diff --git a/internal/router/router.go b/internal/router/router.go index 17060df..cfea90c 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -682,6 +682,19 @@ func (r *Router) Stop() { // SetLeader tells /leader how to know whether this process leads. func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) } +// leading says whether this process leads, which a process with no rival +// always does. +// +// Jobs and batches are served only where it does. A standby's router has no +// endpoints — its reconciler placed nothing — and a client's request reaches +// the leader by being forwarded at the door, which a job replayed inside the +// process never passes: a 3D job leased by the standby was told "loading" +// for as long as anyone let it, while the leader served the jobs behind it. +func (r *Router) leading() bool { + f := r.isLeader.Load() + return f == nil || (*f)() +} + // Registry exposes the endpoint registry, for tests and admin endpoints. func (r *Router) Registry() *endpoint.Registry { return r.reg }