Files
llmgw/internal/router/jobworker.go
T
Ilker ManapandClaude Opus 5.5 0460f52dd2 Jobs run side by side, one per model at a time
One job at a time, a 3D job waiting for the card held up another model's
job that could have run (seen on the PoC, BACKLOG H7). The leader's worker
claims jobs while fewer than eight run, passing over the models that have
one running, and starts each in its own goroutine; one ending wakes it for
the next. Lease and leader-only rules are as they were.

BACKLOG: H5–H7 marked, H8 (presets) and H9 (what the PoC taught) written.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 10:06:20 +02:00

352 lines
11 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The worker that serves jobs. Any gateway on the store may run one: a job
// is worked on by whichever holds its lease, and one whose gateway died is
// taken by another when the lease runs out.
const (
jobLease = 2 * time.Minute
jobPoll = 5 * time.Second
// jobCancelCheck is how often a running job looks for a cancel.
jobCancelCheck = 2 * time.Second
// jobMaxResult bounds what one job may answer: a mesh, a short video.
jobMaxResult = 256 << 20
// jobsPerModel is how many of one model's jobs run at once: its engine
// works through them in turn anyway. jobsAtOnce bounds them all.
jobsPerModel = 1
jobsAtOnce = 8
)
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
// "Fits nowhere" is said for a moment by a cluster changing under it — an
// agent restarting re-registers without its engines for a few seconds — and
// a job failed by that moment was failed by nothing wrong with it.
var (
jobUnplacedGrace = 5 * time.Minute
jobUnplacedRetry = 15 * time.Second
)
// wakeJobs has the worker look for work now rather than at its next poll.
func (r *Router) wakeJobs() {
select {
case r.jobWake <- struct{}{}:
default:
}
}
func (r *Router) runJobs(ctx context.Context) {
host, _ := os.Hostname()
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
t := time.NewTicker(jobPoll)
defer t.Stop()
// Jobs run side by side, one per model at a time and jobsAtOnce in all:
// one at a time, a job waiting for the card held up another model's job
// that could have run.
var mu sync.Mutex
running := map[string]int{}
total := 0
for {
for r.leading() {
mu.Lock()
if total >= jobsAtOnce {
mu.Unlock()
break
}
var skip []string
for m, n := range running {
if n >= jobsPerModel {
skip = append(skip, m)
}
}
mu.Unlock()
now := time.Now()
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease), skip...)
if err != nil {
if ctx.Err() == nil {
r.log.Warn("could not look for jobs to serve", "err", err)
}
break
}
if !ok {
break
}
mu.Lock()
running[j.Model]++
total++
mu.Unlock()
go func(j store.Job) {
r.workJob(ctx, owner, j)
mu.Lock()
if running[j.Model]--; running[j.Model] <= 0 {
delete(running, j.Model)
}
total--
mu.Unlock()
r.wakeJobs() // room for the next
}(j)
}
// A job's record outlives its file by nothing: both go at the
// retention.
_, _, _, days := r.batchCfg()
if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 {
r.log.Info("jobs past their retention forgotten", "jobs", n)
}
select {
case <-ctx.Done():
return
case <-t.C:
case <-r.jobWake:
}
}
}
// workJob takes one job as far as it goes, holding its lease the while.
func (r *Router) workJob(ctx context.Context, owner string, j store.Job) {
jctx, stop := context.WithCancel(ctx)
defer stop()
defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }()
// The lease kept while working; lost, the work stops and another
// gateway has it.
go func() {
t := time.NewTicker(jobLease / 4)
defer t.Stop()
for {
select {
case <-jctx.Done():
return
case <-t.C:
if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held {
stop()
return
}
}
}
}()
if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil {
r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err)
}
}
// serveJob replays the job's request and keeps what came back. A failure
// of the gateway's own (the store unreachable) is returned, and the job is
// picked up again; a refusal of the request is the job's end.
func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
now := time.Now()
switch {
case j.Status == store.JobCancelling:
j.Status, j.FinishedAt = store.JobCancelled, now
return r.store.UpdateJob(ctx, *j)
case now.After(j.ExpiresAt):
j.Status, j.FinishedAt = store.JobExpired, now
j.Error = fmt.Sprintf("not finished within %s", jobWindow)
return r.store.UpdateJob(ctx, *j)
}
if j.Status == store.JobQueued {
j.Status, j.StartedAt = store.JobRunning, now
if err := r.store.UpdateJob(ctx, *j); err != nil {
return err
}
}
// A job found running was another gateway's, which died with it. It is
// run again from the start: the engine's queue does not outlive the
// request that was waiting on it.
fail := func(msg string) error {
j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now()
return r.store.UpdateJob(ctx, *j)
}
p, err := r.principalOfJob(ctx, *j)
if err != nil {
return fail("the key or account this job was made with can no longer be used: " + err.Error())
}
// A cancel asked for while it runs ends the request, which takes the
// work out of the engine's queue.
rctx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
t := time.NewTicker(jobCancelCheck)
defer t.Stop()
for {
select {
case <-rctx.Done():
return
case <-t.C:
if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling {
cancel()
return
}
}
}
}()
handler := r.handleInference(j.Endpoint)
var unplaced time.Time // when it was first said to fit nowhere
for attempt := 1; ; attempt++ {
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
if err != nil {
return fail(err.Error())
}
hreq.Header.Set("Content-Type", "application/json")
hreq.Header.Set("X-Request-Id", j.ID)
hreq.Header.Set(tagsHeader, "job="+j.ID)
w := &bufferWriter{header: http.Header{}, max: jobMaxResult}
handler(w, hreq)
if rctx.Err() != nil {
if ctx.Err() != nil {
// The lease was lost or the gateway is stopping: another
// gateway's to pick up.
return ctx.Err()
}
j.Status, j.FinishedAt = store.JobCancelled, time.Now()
return r.store.UpdateJob(ctx, *j)
}
status := w.status
if status == 0 {
status = http.StatusOK
}
// Waited out, for as long as the job has: a model loading, a rate
// limit, a queue full. Nobody is holding a connection, which is the
// point of a job. Not a spent budget, which says no Retry-After
// because waiting does not help.
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
time.Now().Before(j.ExpiresAt) {
if by := w.header.Get(waitingForHeader); by != "" {
// Kept as its progress: whoever asks after it is told what
// it waits on, rather than nothing at all.
p := map[string]any{"state": "waiting_gpu", "held_by": by}
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
p["eta_seconds"] = eta
}
if b, err := json.Marshal(p); err == nil {
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
}
}
secs, _ := strconv.Atoi(wait)
select {
case <-rctx.Done():
continue // said above
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
}
continue
}
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
if unplaced.IsZero() {
unplaced = time.Now()
}
if time.Since(unplaced) < jobUnplacedGrace {
select {
case <-rctx.Done():
case <-time.After(jobUnplacedRetry):
}
continue
}
}
if status >= 300 {
return fail(errorMessage(w.buf.Bytes()))
}
ct := w.header.Get("Content-Type")
if ct == "" {
ct = http.DetectContentType(w.buf.Bytes())
}
_, _, _, days := r.batchCfg()
done := time.Now()
f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput,
Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)}
if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil {
return err
}
// Said done while it may still be said: the last word heard was the
// node running, and a finished job saying it is running reads as
// one that stopped halfway.
_ = r.store.SetJobProgress(ctx, j.ID, `{"state":"done","percent":100}`, "")
j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done
return r.store.UpdateJob(ctx, *j)
}
}
// extensionFor names a result's file by what it is.
func extensionFor(contentType string) string {
ct, _, _ := strings.Cut(contentType, ";")
switch strings.TrimSpace(ct) {
case "application/json":
return ".json"
case "model/gltf-binary":
return ".glb"
case "model/gltf+json":
return ".gltf"
case "model/obj":
return ".obj"
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/webp":
return ".webp"
case "image/gif":
return ".gif"
case "video/mp4":
return ".mp4"
case "video/webm":
return ".webm"
case "video/quicktime":
return ".mov"
case "audio/wav", "audio/wave", "audio/x-wav":
return ".wav"
case "audio/mpeg":
return ".mp3"
case "audio/flac":
return ".flac"
case "audio/ogg":
return ".ogg"
}
return ".bin"
}
// principalOfJob is who a job acts for, as they stand now: the key it was
// made with, or — for one made from the portal — the person whose session
// made it, while they may still use the gateway and the project the job is
// charged to.
func (r *Router) principalOfJob(ctx context.Context, j store.Job) (auth.Principal, error) {
uid, session := strings.CutPrefix(j.KeyID, "session:")
if !session {
return r.principalOfKey(ctx, j.KeyID, "")
}
u, err := r.store.UserByID(ctx, uid)
if err != nil {
return auth.Principal{}, err
}
if u.State != store.UserActive {
return auth.Principal{}, fmt.Errorf("the account that made it is %s", u.State)
}
project, _ := strings.CutPrefix(j.Owner, "project:")
if !strings.HasPrefix(j.Owner, "project:") {
project = ""
} else if ok, err := r.store.CanUseProject(ctx, uid, project); err != nil || !ok {
return auth.Principal{}, errors.New("its account is no longer a member of the project it is charged to")
}
eff, err := r.store.ResolveFor(ctx, j.KeyID, uid, project)
if err != nil {
return auth.Principal{}, err
}
return auth.Principal{KeyID: j.KeyID, KeyPrefix: "session", UserID: uid, ProjectID: eff.ProjectID,
TeamID: eff.TeamID, OrgID: eff.OrgID, Band: eff.Band, RoutingProfile: eff.RoutingProfile, Limits: eff}, nil
}