CI / test (push) Failing after 8m47s
A 3D job leased by the standby control plane, whose router has no endpoints and which a job replayed in the process never leaves, was told "loading" for seven minutes while the leader served the jobs behind it. Jobs and batches are claimed only while leading. Five jobs asked for while cuda01's agent restarted failed for good as "cannot be placed": re-registering, the node had no engines for a few seconds. A job asks again for five minutes before that is its end. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
300 lines
9.3 KiB
Go
300 lines
9.3 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// The worker that serves jobs. Any gateway on the store may run one: a job
|
|
// is worked on by whichever holds its lease, and one whose gateway died is
|
|
// taken by another when the lease runs out.
|
|
|
|
const (
|
|
jobLease = 2 * time.Minute
|
|
jobPoll = 5 * time.Second
|
|
// jobCancelCheck is how often a running job looks for a cancel.
|
|
jobCancelCheck = 2 * time.Second
|
|
// jobMaxResult bounds what one job may answer: a mesh, not a film.
|
|
jobMaxResult = 256 << 20
|
|
)
|
|
|
|
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
|
|
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
|
|
// "Fits nowhere" is said for a moment by a cluster changing under it — an
|
|
// agent restarting re-registers without its engines for a few seconds — and
|
|
// a job failed by that moment was failed by nothing wrong with it.
|
|
var (
|
|
jobUnplacedGrace = 5 * time.Minute
|
|
jobUnplacedRetry = 15 * time.Second
|
|
)
|
|
|
|
// wakeJobs has the worker look for work now rather than at its next poll.
|
|
func (r *Router) wakeJobs() {
|
|
select {
|
|
case r.jobWake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (r *Router) runJobs(ctx context.Context) {
|
|
host, _ := os.Hostname()
|
|
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
|
|
t := time.NewTicker(jobPoll)
|
|
defer t.Stop()
|
|
for {
|
|
for r.leading() {
|
|
now := time.Now()
|
|
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
|
|
if err != nil {
|
|
if ctx.Err() == nil {
|
|
r.log.Warn("could not look for jobs to serve", "err", err)
|
|
}
|
|
break
|
|
}
|
|
if !ok {
|
|
break
|
|
}
|
|
r.workJob(ctx, owner, j)
|
|
}
|
|
// A job's record outlives its file by nothing: both go at the
|
|
// retention.
|
|
_, _, _, days := r.batchCfg()
|
|
if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 {
|
|
r.log.Info("jobs past their retention forgotten", "jobs", n)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
case <-r.jobWake:
|
|
}
|
|
}
|
|
}
|
|
|
|
// workJob takes one job as far as it goes, holding its lease the while.
|
|
func (r *Router) workJob(ctx context.Context, owner string, j store.Job) {
|
|
jctx, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }()
|
|
// The lease kept while working; lost, the work stops and another
|
|
// gateway has it.
|
|
go func() {
|
|
t := time.NewTicker(jobLease / 4)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-jctx.Done():
|
|
return
|
|
case <-t.C:
|
|
if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held {
|
|
stop()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil {
|
|
r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err)
|
|
}
|
|
}
|
|
|
|
// serveJob replays the job's request and keeps what came back. A failure
|
|
// of the gateway's own (the store unreachable) is returned, and the job is
|
|
// picked up again; a refusal of the request is the job's end.
|
|
func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
|
now := time.Now()
|
|
switch {
|
|
case j.Status == store.JobCancelling:
|
|
j.Status, j.FinishedAt = store.JobCancelled, now
|
|
return r.store.UpdateJob(ctx, *j)
|
|
case now.After(j.ExpiresAt):
|
|
j.Status, j.FinishedAt = store.JobExpired, now
|
|
j.Error = fmt.Sprintf("not finished within %s", jobWindow)
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
if j.Status == store.JobQueued {
|
|
j.Status, j.StartedAt = store.JobRunning, now
|
|
if err := r.store.UpdateJob(ctx, *j); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// A job found running was another gateway's, which died with it. It is
|
|
// run again from the start: the engine's queue does not outlive the
|
|
// request that was waiting on it.
|
|
fail := func(msg string) error {
|
|
j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now()
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
p, err := r.principalOfJob(ctx, *j)
|
|
if err != nil {
|
|
return fail("the key or account this job was made with can no longer be used: " + err.Error())
|
|
}
|
|
|
|
// A cancel asked for while it runs ends the request, which takes the
|
|
// work out of the engine's queue.
|
|
rctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
go func() {
|
|
t := time.NewTicker(jobCancelCheck)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-rctx.Done():
|
|
return
|
|
case <-t.C:
|
|
if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
handler := r.handleInference(j.Endpoint)
|
|
var unplaced time.Time // when it was first said to fit nowhere
|
|
for attempt := 1; ; attempt++ {
|
|
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
|
|
if err != nil {
|
|
return fail(err.Error())
|
|
}
|
|
hreq.Header.Set("Content-Type", "application/json")
|
|
hreq.Header.Set("X-Request-Id", j.ID)
|
|
hreq.Header.Set(tagsHeader, "job="+j.ID)
|
|
w := &bufferWriter{header: http.Header{}, max: jobMaxResult}
|
|
handler(w, hreq)
|
|
if rctx.Err() != nil {
|
|
if ctx.Err() != nil {
|
|
// The lease was lost or the gateway is stopping: another
|
|
// gateway's to pick up.
|
|
return ctx.Err()
|
|
}
|
|
j.Status, j.FinishedAt = store.JobCancelled, time.Now()
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
status := w.status
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
// Waited out, for as long as the job has: a model loading, a rate
|
|
// limit, a queue full. Nobody is holding a connection, which is the
|
|
// point of a job. Not a spent budget, which says no Retry-After
|
|
// because waiting does not help.
|
|
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
|
|
time.Now().Before(j.ExpiresAt) {
|
|
if by := w.header.Get(waitingForHeader); by != "" {
|
|
// Kept as its progress: whoever asks after it is told what
|
|
// it waits on, rather than nothing at all.
|
|
p := map[string]any{"state": "waiting_gpu", "held_by": by}
|
|
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
|
|
p["eta_seconds"] = eta
|
|
}
|
|
if b, err := json.Marshal(p); err == nil {
|
|
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
|
|
}
|
|
}
|
|
secs, _ := strconv.Atoi(wait)
|
|
select {
|
|
case <-rctx.Done():
|
|
continue // said above
|
|
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
|
|
}
|
|
continue
|
|
}
|
|
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
|
|
if unplaced.IsZero() {
|
|
unplaced = time.Now()
|
|
}
|
|
if time.Since(unplaced) < jobUnplacedGrace {
|
|
select {
|
|
case <-rctx.Done():
|
|
case <-time.After(jobUnplacedRetry):
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if status >= 300 {
|
|
return fail(errorMessage(w.buf.Bytes()))
|
|
}
|
|
ct := w.header.Get("Content-Type")
|
|
if ct == "" {
|
|
ct = http.DetectContentType(w.buf.Bytes())
|
|
}
|
|
_, _, _, days := r.batchCfg()
|
|
done := time.Now()
|
|
f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput,
|
|
Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)}
|
|
if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
// Said done while it may still be said: the last word heard was the
|
|
// node running, and a finished job saying it is running reads as
|
|
// one that stopped halfway.
|
|
_ = r.store.SetJobProgress(ctx, j.ID, `{"state":"done","percent":100}`, "")
|
|
j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
}
|
|
|
|
// extensionFor names a result's file by what it is.
|
|
func extensionFor(contentType string) string {
|
|
ct, _, _ := strings.Cut(contentType, ";")
|
|
switch strings.TrimSpace(ct) {
|
|
case "application/json":
|
|
return ".json"
|
|
case "model/gltf-binary":
|
|
return ".glb"
|
|
case "model/gltf+json":
|
|
return ".gltf"
|
|
case "model/obj":
|
|
return ".obj"
|
|
case "image/png":
|
|
return ".png"
|
|
case "image/jpeg":
|
|
return ".jpg"
|
|
case "image/webp":
|
|
return ".webp"
|
|
}
|
|
return ".bin"
|
|
}
|
|
|
|
// principalOfJob is who a job acts for, as they stand now: the key it was
|
|
// made with, or — for one made from the portal — the person whose session
|
|
// made it, while they may still use the gateway and the project the job is
|
|
// charged to.
|
|
func (r *Router) principalOfJob(ctx context.Context, j store.Job) (auth.Principal, error) {
|
|
uid, session := strings.CutPrefix(j.KeyID, "session:")
|
|
if !session {
|
|
return r.principalOfKey(ctx, j.KeyID, "")
|
|
}
|
|
u, err := r.store.UserByID(ctx, uid)
|
|
if err != nil {
|
|
return auth.Principal{}, err
|
|
}
|
|
if u.State != store.UserActive {
|
|
return auth.Principal{}, fmt.Errorf("the account that made it is %s", u.State)
|
|
}
|
|
project, _ := strings.CutPrefix(j.Owner, "project:")
|
|
if !strings.HasPrefix(j.Owner, "project:") {
|
|
project = ""
|
|
} else if ok, err := r.store.CanUseProject(ctx, uid, project); err != nil || !ok {
|
|
return auth.Principal{}, errors.New("its account is no longer a member of the project it is charged to")
|
|
}
|
|
eff, err := r.store.ResolveFor(ctx, j.KeyID, uid, project)
|
|
if err != nil {
|
|
return auth.Principal{}, err
|
|
}
|
|
return auth.Principal{KeyID: j.KeyID, KeyPrefix: "session", UserID: uid, ProjectID: eff.ProjectID,
|
|
TeamID: eff.TeamID, OrgID: eff.OrgID, Band: eff.Band, RoutingProfile: eff.RoutingProfile, Limits: eff}, nil
|
|
}
|