A job waiting for the card says what holds it and about how long
CI / test (push) Failing after 6m16s
CI / test (push) Failing after 6m16s
A 3D job waiting for the picture model to give up the one card had an empty
progress; its client could not tell waiting from stuck. The control plane
says which model holds the cards and when it lets go, plus the waiting
model's last start; the 503 carries it as headers, and a job keeps it as
progress {state: waiting_gpu, held_by, eta_seconds}.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cf2edfd995
commit
b1490dbf30
12 files changed
+252
-8
No files matched your search
@@ -2,6 +2,7 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
@@ -26,6 +27,40 @@ type unservable interface {
|
||||
Unservable(model string) (string, bool)
|
||||
}
|
||||
|
||||
// waiter is an activator that can say which model holds the cards an
|
||||
// on-demand one waits for, and about how long the wait is.
|
||||
type waiter interface {
|
||||
WaitFor(model string) (heldBy string, eta time.Duration, ok bool)
|
||||
}
|
||||
|
||||
// waitingErr is errLoading for a model waiting on another to give up its
|
||||
// cards: who holds them, and the wait when it can be told.
|
||||
type waitingErr struct {
|
||||
heldBy string
|
||||
eta time.Duration
|
||||
}
|
||||
|
||||
func (e *waitingErr) Error() string {
|
||||
if e.eta > 0 {
|
||||
return fmt.Sprintf("%s: waiting for %s to give up the GPU; expected in about %d s",
|
||||
errLoading, e.heldBy, int(e.eta.Seconds()))
|
||||
}
|
||||
return fmt.Sprintf("%s: waiting for %s to give up the GPU", errLoading, e.heldBy)
|
||||
}
|
||||
|
||||
func (e *waitingErr) Unwrap() error { return errLoading }
|
||||
|
||||
// loadingErr is why a wanted model is not serving yet: waiting on another
|
||||
// model's cards when the activator can say so, else simply loading.
|
||||
func (r *Router) loadingErr(model string) error {
|
||||
if w, ok := r.activator.(waiter); ok {
|
||||
if by, eta, waiting := w.WaitFor(model); waiting {
|
||||
return &waitingErr{heldBy: by, eta: eta}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%w: %s", errLoading, "the model is loading; retry shortly")
|
||||
}
|
||||
|
||||
// whyUnservable is the activator's reason a model cannot be served, if any.
|
||||
func (r *Router) whyUnservable(model string) (string, bool) {
|
||||
u, ok := r.activator.(unservable)
|
||||
|
||||
@@ -730,7 +730,19 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
|
||||
// is told so will make it. Without the header a well-behaved client
|
||||
// backs off exactly as it would from a permanent failure.
|
||||
status = http.StatusServiceUnavailable
|
||||
w.Header().Set("Retry-After", "30")
|
||||
retry := 30
|
||||
// Waiting on another model's cards: who holds them and for about
|
||||
// how long, which a job keeps as its progress.
|
||||
var we *waitingErr
|
||||
if errors.As(res.err, &we) {
|
||||
w.Header().Set(waitingForHeader, we.heldBy)
|
||||
if we.eta > 0 {
|
||||
secs := int(we.eta.Seconds())
|
||||
w.Header().Set(etaHeader, strconv.Itoa(secs))
|
||||
retry = min(max(secs, 5), retry)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
||||
writeError(w, status, "server_error", res.err.Error())
|
||||
case errors.Is(res.err, errNoCapacity):
|
||||
status = http.StatusServiceUnavailable
|
||||
|
||||
@@ -283,3 +283,19 @@ func TestJobsAreTheirOwners(t *testing.T) {
|
||||
t.Errorf("another project read a job's result: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
rt.cfg.ActivationWait = 10 * time.Millisecond
|
||||
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
||||
heldBy: "flux-schnell", eta: 90 * time.Second})
|
||||
j := queueJob(t, rt, "job_w", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
|
||||
rt.wakeJobs()
|
||||
|
||||
got := waitJob(t, rt.Handler(), secret, j.ID, func(j map[string]any) bool { return j["progress"] != nil })
|
||||
p, _ := got["progress"].(map[string]any)
|
||||
if got["status"] != "running" || p["state"] != "waiting_gpu" || p["held_by"] != "flux-schnell" || p["eta_seconds"] != 90.0 {
|
||||
t.Errorf("job %v", got)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package router
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -180,6 +181,17 @@ func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
||||
// because waiting does not help.
|
||||
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
|
||||
time.Now().Before(j.ExpiresAt) {
|
||||
if by := w.header.Get(waitingForHeader); by != "" {
|
||||
// Kept as its progress: whoever asks after it is told what
|
||||
// it waits on, rather than nothing at all.
|
||||
p := map[string]any{"state": "waiting_gpu", "held_by": by}
|
||||
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
|
||||
p["eta_seconds"] = eta
|
||||
}
|
||||
if b, err := json.Marshal(p); err == nil {
|
||||
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
|
||||
}
|
||||
}
|
||||
secs, _ := strconv.Atoi(wait)
|
||||
select {
|
||||
case <-rctx.Done():
|
||||
|
||||
@@ -1023,8 +1023,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
// Asked for, still loading. Saying "no endpoint serves this" would
|
||||
// be a lie, and a client told to retry will succeed where one told
|
||||
// the model does not exist gives up.
|
||||
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
|
||||
"the model is loading; retry shortly")}
|
||||
return dispatchResult{err: r.loadingErr(model)}
|
||||
}
|
||||
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, slices.DeleteFunc(r.servedModels(), r.hiddenModel)))}
|
||||
}
|
||||
|
||||
@@ -808,3 +808,36 @@ func TestABatchOfEmbeddingsIsCheckedTextByText(t *testing.T) {
|
||||
t.Errorf("a passage over the context was let through: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type waitingActivator struct {
|
||||
fakeActivator
|
||||
heldBy string
|
||||
eta time.Duration
|
||||
}
|
||||
|
||||
func (a *waitingActivator) WaitFor(string) (string, time.Duration, bool) {
|
||||
return a.heldBy, a.eta, true
|
||||
}
|
||||
|
||||
func TestAModelWaitingOnAnothersCardsSaysWhoAndForHowLong(t *testing.T) {
|
||||
// A 3D job waiting for the picture model to give up the one card was
|
||||
// told nothing at all; its client could not tell waiting from stuck.
|
||||
e := newFakeEngine(t, "m")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
|
||||
cfg.Router.ActivationWait = 50 * time.Millisecond
|
||||
rt := newTestRouter(t, cfg)
|
||||
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
||||
heldBy: "flux-schnell", eta: 12 * time.Second})
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "sleepy", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status %d, expected 503", rec.Code)
|
||||
}
|
||||
if rec.Header().Get(waitingForHeader) != "flux-schnell" || rec.Header().Get(etaHeader) != "12" ||
|
||||
rec.Header().Get("Retry-After") != "12" {
|
||||
t.Errorf("headers %v", rec.Header())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "waiting for flux-schnell") {
|
||||
t.Errorf("body %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -11,10 +11,14 @@ import (
|
||||
// project: OpenAI's metadata (at most 16 keys, as OpenAI allows) and the
|
||||
// x-llmgw-tags header, which wins for a key both give.
|
||||
const (
|
||||
tagsHeader = "x-llmgw-tags"
|
||||
maxTags = 16
|
||||
maxTagValue = 512
|
||||
maxEndUser = 256
|
||||
tagsHeader = "x-llmgw-tags"
|
||||
// waitingForHeader and etaHeader say, on a 503, which model holds the
|
||||
// cards the one asked for waits on, and about how many seconds it waits.
|
||||
waitingForHeader = "X-Llmgw-Waiting-For"
|
||||
etaHeader = "X-Llmgw-Eta-Seconds"
|
||||
maxTags = 16
|
||||
maxTagValue = 512
|
||||
maxEndUser = 256
|
||||
)
|
||||
|
||||
var tagKey = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)
|
||||
|
||||
Reference in new issue
Block a user