Jobs are served by the leader, and a moment of fitting nowhere is waited out
CI / test (push) Failing after 8m47s
CI / test (push) Failing after 8m47s
A 3D job leased by the standby control plane, whose router has no endpoints and which a job replayed in the process never leaves, was told "loading" for seven minutes while the leader served the jobs behind it. Jobs and batches are claimed only while leading. Five jobs asked for while cuda01's agent restarted failed for good as "cannot be placed": re-registering, the node had no engines for a few seconds. A job asks again for five minutes before that is its end. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9b8ec89247
commit
95c61daf1b
6 files changed
+107
-3
No files matched your search
@@ -46,6 +46,10 @@ APIs change between minors without a deprecation period.
|
||||
once as "cannot be placed"; it now waits, and a job with it.
|
||||
- A model evicted for being idle was placed again on the next pass, ahead of
|
||||
the one it made room for.
|
||||
- A job or batch leased by a standby control plane, whose router has no
|
||||
endpoints, waited on "loading" for good; they are served only by the
|
||||
leader. A job told its model fits nowhere asks again for five minutes
|
||||
before failing, which an agent re-registering makes true for seconds.
|
||||
|
||||
## v0.14.3 — 2026-10-10
|
||||
|
||||
|
||||
+4
-1
@@ -2086,7 +2086,10 @@ curl -X POST http://gateway:8080/v1/3d/generations/job_…/cancel -H "Authorizat
|
||||
- **How it is served:** the request is kept as it was sent and replayed by
|
||||
whichever gateway holds the job's lease, through the same path a
|
||||
client's request takes — limits, admission in the key's band, guardrails,
|
||||
accounting. It is one usage record under the job's id, tagged
|
||||
accounting — by the leading control plane, since a standby has nothing
|
||||
to send it to. A model said to fit nowhere is asked for again for five
|
||||
minutes before that is the job's error: an agent re-registering says so
|
||||
for a few seconds. It is one usage record under the job's id, tagged
|
||||
`job=<id>`, charged like any request; `GET` shows the usage once it is
|
||||
done. A model still loading, a rate limit or a full queue is waited out
|
||||
for as long as the job has: nobody is holding a connection, which is the
|
||||
|
||||
@@ -39,7 +39,7 @@ func (r *Router) runBatches(ctx context.Context) {
|
||||
t := time.NewTicker(batchPoll)
|
||||
defer t.Stop()
|
||||
for {
|
||||
for {
|
||||
for r.leading() {
|
||||
now := time.Now()
|
||||
b, ok, err := r.store.ClaimBatch(ctx, owner, now, now.Add(batchLease))
|
||||
if err != nil {
|
||||
|
||||
@@ -299,3 +299,64 @@ func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) {
|
||||
t.Errorf("job %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type countingUnservable struct {
|
||||
unservableActivator
|
||||
asked atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingUnservable) Unservable(model string) (string, bool) {
|
||||
c.asked.Add(1)
|
||||
return c.unservableActivator.Unservable(model)
|
||||
}
|
||||
|
||||
func TestAJobSaidToFitNowhereKeepsAskingAWhileBeforeFailing(t *testing.T) {
|
||||
// An agent restarting re-registers without its engines for a few
|
||||
// seconds; five 3D jobs asked for in those seconds failed for good.
|
||||
grace, retry := jobUnplacedGrace, jobUnplacedRetry
|
||||
jobUnplacedGrace, jobUnplacedRetry = 300*time.Millisecond, 20*time.Millisecond
|
||||
t.Cleanup(func() { jobUnplacedGrace, jobUnplacedRetry = grace, retry })
|
||||
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
act := &countingUnservable{unservableActivator: unservableActivator{
|
||||
fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
||||
why: map[string]string{"sleepy": "sleepy cannot be placed on any node: needs 15 GB"},
|
||||
}}
|
||||
rt.SetActivator(act)
|
||||
j := queueJob(t, rt, "job_u", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
|
||||
start := time.Now()
|
||||
rt.wakeJobs()
|
||||
|
||||
got := waitJob(t, rt.Handler(), secret, j.ID, jobDone)
|
||||
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "needs 15 GB") {
|
||||
t.Errorf("job %v", got)
|
||||
}
|
||||
if took := time.Since(start); took < jobUnplacedGrace {
|
||||
t.Errorf("failed after %s, before the grace of %s", took, jobUnplacedGrace)
|
||||
}
|
||||
if n := act.asked.Load(); n < 3 {
|
||||
t.Errorf("asked %d times in the grace", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandbyLeavesJobsToTheLeader(t *testing.T) {
|
||||
// A standby's router has no endpoints; a 3D job it leased was told
|
||||
// "loading" for seven minutes while the leader served the jobs behind it.
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
var lead atomic.Bool
|
||||
rt.SetLeader(lead.Load)
|
||||
j := queueJob(t, rt, "job_s", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
||||
rt.wakeJobs()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if got, _ := rt.store.JobByID(t.Context(), j.ID); got.Status != store.JobQueued {
|
||||
t.Fatalf("a standby took the job: %s", got.Status)
|
||||
}
|
||||
|
||||
lead.Store(true)
|
||||
rt.wakeJobs()
|
||||
if got := waitJob(t, rt.Handler(), secret, j.ID, jobDone); got["status"] != "completed" {
|
||||
t.Errorf("the leader did not serve it: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,16 @@ const (
|
||||
jobMaxResult = 256 << 20
|
||||
)
|
||||
|
||||
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
|
||||
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
|
||||
// "Fits nowhere" is said for a moment by a cluster changing under it — an
|
||||
// agent restarting re-registers without its engines for a few seconds — and
|
||||
// a job failed by that moment was failed by nothing wrong with it.
|
||||
var (
|
||||
jobUnplacedGrace = 5 * time.Minute
|
||||
jobUnplacedRetry = 15 * time.Second
|
||||
)
|
||||
|
||||
// wakeJobs has the worker look for work now rather than at its next poll.
|
||||
func (r *Router) wakeJobs() {
|
||||
select {
|
||||
@@ -43,7 +53,7 @@ func (r *Router) runJobs(ctx context.Context) {
|
||||
t := time.NewTicker(jobPoll)
|
||||
defer t.Stop()
|
||||
for {
|
||||
for {
|
||||
for r.leading() {
|
||||
now := time.Now()
|
||||
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
|
||||
if err != nil {
|
||||
@@ -152,6 +162,7 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
||||
}()
|
||||
|
||||
handler := r.handleInference(j.Endpoint)
|
||||
var unplaced time.Time // when it was first said to fit nowhere
|
||||
for attempt := 1; ; attempt++ {
|
||||
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
|
||||
if err != nil {
|
||||
@@ -200,6 +211,18 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
|
||||
if unplaced.IsZero() {
|
||||
unplaced = time.Now()
|
||||
}
|
||||
if time.Since(unplaced) < jobUnplacedGrace {
|
||||
select {
|
||||
case <-rctx.Done():
|
||||
case <-time.After(jobUnplacedRetry):
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if status >= 300 {
|
||||
return fail(errorMessage(w.buf.Bytes()))
|
||||
}
|
||||
|
||||
@@ -682,6 +682,19 @@ func (r *Router) Stop() {
|
||||
// SetLeader tells /leader how to know whether this process leads.
|
||||
func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) }
|
||||
|
||||
// leading says whether this process leads, which a process with no rival
|
||||
// always does.
|
||||
//
|
||||
// Jobs and batches are served only where it does. A standby's router has no
|
||||
// endpoints — its reconciler placed nothing — and a client's request reaches
|
||||
// the leader by being forwarded at the door, which a job replayed inside the
|
||||
// process never passes: a 3D job leased by the standby was told "loading"
|
||||
// for as long as anyone let it, while the leader served the jobs behind it.
|
||||
func (r *Router) leading() bool {
|
||||
f := r.isLeader.Load()
|
||||
return f == nil || (*f)()
|
||||
}
|
||||
|
||||
// Registry exposes the endpoint registry, for tests and admin endpoints.
|
||||
func (r *Router) Registry() *endpoint.Registry { return r.reg }
|
||||
|
||||
|
||||
Reference in new issue
Block a user