A job waiting for the card says what holds it and about how long
CI / test (push) Failing after 6m16s

A 3D job waiting for the picture model to give up the one card had an empty
progress; its client could not tell waiting from stuck. The control plane
says which model holds the cards and when it lets go, plus the waiting
model's last start; the 503 carries it as headers, and a job keeps it as
progress {state: waiting_gpu, held_by, eta_seconds}.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 22:25:27 +02:00
1 parent cf2edfd995
commit b1490dbf30
12 files changed
+252 -8

No files matched your search

+35
View File
@@ -2,6 +2,7 @@ package router
import (
"context"
"fmt"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
@@ -26,6 +27,40 @@ type unservable interface {
Unservable(model string) (string, bool)
}
// waiter is an activator that can say which model holds the cards an
// on-demand one waits for, and about how long the wait is.
type waiter interface {
WaitFor(model string) (heldBy string, eta time.Duration, ok bool)
}
// waitingErr is errLoading for a model waiting on another to give up its
// cards: who holds them, and the wait when it can be told.
type waitingErr struct {
heldBy string
eta time.Duration
}
func (e *waitingErr) Error() string {
if e.eta > 0 {
return fmt.Sprintf("%s: waiting for %s to give up the GPU; expected in about %d s",
errLoading, e.heldBy, int(e.eta.Seconds()))
}
return fmt.Sprintf("%s: waiting for %s to give up the GPU", errLoading, e.heldBy)
}
func (e *waitingErr) Unwrap() error { return errLoading }
// loadingErr is why a wanted model is not serving yet: waiting on another
// model's cards when the activator can say so, else simply loading.
func (r *Router) loadingErr(model string) error {
if w, ok := r.activator.(waiter); ok {
if by, eta, waiting := w.WaitFor(model); waiting {
return &waitingErr{heldBy: by, eta: eta}
}
}
return fmt.Errorf("%w: %s", errLoading, "the model is loading; retry shortly")
}
// whyUnservable is the activator's reason a model cannot be served, if any.
func (r *Router) whyUnservable(model string) (string, bool) {
u, ok := r.activator.(unservable)
+13 -1
View File
@@ -730,7 +730,19 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
// is told so will make it. Without the header a well-behaved client
// backs off exactly as it would from a permanent failure.
status = http.StatusServiceUnavailable
w.Header().Set("Retry-After", "30")
retry := 30
// Waiting on another model's cards: who holds them and for about
// how long, which a job keeps as its progress.
var we *waitingErr
if errors.As(res.err, &we) {
w.Header().Set(waitingForHeader, we.heldBy)
if we.eta > 0 {
secs := int(we.eta.Seconds())
w.Header().Set(etaHeader, strconv.Itoa(secs))
retry = min(max(secs, 5), retry)
}
}
w.Header().Set("Retry-After", strconv.Itoa(retry))
writeError(w, status, "server_error", res.err.Error())
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
+16
View File
@@ -283,3 +283,19 @@ func TestJobsAreTheirOwners(t *testing.T) {
t.Errorf("another project read a job's result: %d", code)
}
}
func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) {
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
rt.cfg.ActivationWait = 10 * time.Millisecond
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
heldBy: "flux-schnell", eta: 90 * time.Second})
j := queueJob(t, rt, "job_w", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
rt.wakeJobs()
got := waitJob(t, rt.Handler(), secret, j.ID, func(j map[string]any) bool { return j["progress"] != nil })
p, _ := got["progress"].(map[string]any)
if got["status"] != "running" || p["state"] != "waiting_gpu" || p["held_by"] != "flux-schnell" || p["eta_seconds"] != 90.0 {
t.Errorf("job %v", got)
}
}
+12
View File
@@ -3,6 +3,7 @@ package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -180,6 +181,17 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
// because waiting does not help.
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
time.Now().Before(j.ExpiresAt) {
if by := w.header.Get(waitingForHeader); by != "" {
// Kept as its progress: whoever asks after it is told what
// it waits on, rather than nothing at all.
p := map[string]any{"state": "waiting_gpu", "held_by": by}
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
p["eta_seconds"] = eta
}
if b, err := json.Marshal(p); err == nil {
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
}
}
secs, _ := strconv.Atoi(wait)
select {
case <-rctx.Done():
+1 -2
View File
@@ -1023,8 +1023,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
// Asked for, still loading. Saying "no endpoint serves this" would
// be a lie, and a client told to retry will succeed where one told
// the model does not exist gives up.
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
"the model is loading; retry shortly")}
return dispatchResult{err: r.loadingErr(model)}
}
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, slices.DeleteFunc(r.servedModels(), r.hiddenModel)))}
}
+33
View File
@@ -808,3 +808,36 @@ func TestABatchOfEmbeddingsIsCheckedTextByText(t *testing.T) {
t.Errorf("a passage over the context was let through: %d", rec.Code)
}
}
type waitingActivator struct {
fakeActivator
heldBy string
eta time.Duration
}
func (a *waitingActivator) WaitFor(string) (string, time.Duration, bool) {
return a.heldBy, a.eta, true
}
func TestAModelWaitingOnAnothersCardsSaysWhoAndForHowLong(t *testing.T) {
// A 3D job waiting for the picture model to give up the one card was
// told nothing at all; its client could not tell waiting from stuck.
e := newFakeEngine(t, "m")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
cfg.Router.ActivationWait = 50 * time.Millisecond
rt := newTestRouter(t, cfg)
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
heldBy: "flux-schnell", eta: 12 * time.Second})
rec := post(t, rt.Handler(), map[string]any{"model": "sleepy", "messages": []any{}}, nil)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status %d, expected 503", rec.Code)
}
if rec.Header().Get(waitingForHeader) != "flux-schnell" || rec.Header().Get(etaHeader) != "12" ||
rec.Header().Get("Retry-After") != "12" {
t.Errorf("headers %v", rec.Header())
}
if !strings.Contains(rec.Body.String(), "waiting for flux-schnell") {
t.Errorf("body %s", rec.Body.String())
}
}
+8 -4
View File
@@ -11,10 +11,14 @@ import (
// project: OpenAI's metadata (at most 16 keys, as OpenAI allows) and the
// x-llmgw-tags header, which wins for a key both give.
const (
tagsHeader = "x-llmgw-tags"
maxTags = 16
maxTagValue = 512
maxEndUser = 256
tagsHeader = "x-llmgw-tags"
// waitingForHeader and etaHeader say, on a 503, which model holds the
// cards the one asked for waits on, and about how many seconds it waits.
waitingForHeader = "X-Llmgw-Waiting-For"
etaHeader = "X-Llmgw-Eta-Seconds"
maxTags = 16
maxTagValue = 512
maxEndUser = 256
)
var tagKey = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)