Files
llmgw/internal/router/jobworker.go
T
Ilker ManapandClaude Opus 5.5 b6bfc33ef7
CI / test (push) Successful in 9m7s
Any ComfyUI workflow runs as a job, from the inputs its catalog entry declares
A model's catalog entry now says what its workflow takes and makes:
comfyui.inputs, each a placeholder of its name with a type (text, int,
float, bool, choice, image, mask, audio, video, file), bounds, choices
and a default; and comfyui.output, which gains video, audio and files
beside pictures and meshes. POST /v1/workflows/runs with {model, inputs}
checks a request against them before a job is written — a required input
left out, an unknown one, a number out of bounds, a song sent as a
picture, another owner's file id are each refused with what was wrong —
and the job worker replays it to a replica, where the files go into
ComfyUI's input directory, the workflow is filled and run, and the first
file of the model's output kind is the job's result. /v1/models lists
each ComfyUI model's workflow, so a client can build its form.

A PNG result carries the licence in an iTXt chunk, as a GLB does in
asset.extras. A file whose server named no type is typed by its name
before its bytes, so a WAV is audio/wav, not audio/wave.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-11 14:28:15 +02:00

316 lines
9.6 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The worker that serves jobs. Any gateway on the store may run one: a job
// is worked on by whichever holds its lease, and one whose gateway died is
// taken by another when the lease runs out.
const (
jobLease = 2 * time.Minute
jobPoll = 5 * time.Second
// jobCancelCheck is how often a running job looks for a cancel.
jobCancelCheck = 2 * time.Second
// jobMaxResult bounds what one job may answer: a mesh, a short video.
jobMaxResult = 256 << 20
)
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
// "Fits nowhere" is said for a moment by a cluster changing under it — an
// agent restarting re-registers without its engines for a few seconds — and
// a job failed by that moment was failed by nothing wrong with it.
var (
jobUnplacedGrace = 5 * time.Minute
jobUnplacedRetry = 15 * time.Second
)
// wakeJobs has the worker look for work now rather than at its next poll.
func (r *Router) wakeJobs() {
select {
case r.jobWake <- struct{}{}:
default:
}
}
func (r *Router) runJobs(ctx context.Context) {
host, _ := os.Hostname()
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
t := time.NewTicker(jobPoll)
defer t.Stop()
for {
for r.leading() {
now := time.Now()
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
if err != nil {
if ctx.Err() == nil {
r.log.Warn("could not look for jobs to serve", "err", err)
}
break
}
if !ok {
break
}
r.workJob(ctx, owner, j)
}
// A job's record outlives its file by nothing: both go at the
// retention.
_, _, _, days := r.batchCfg()
if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 {
r.log.Info("jobs past their retention forgotten", "jobs", n)
}
select {
case <-ctx.Done():
return
case <-t.C:
case <-r.jobWake:
}
}
}
// workJob takes one job as far as it goes, holding its lease the while.
func (r *Router) workJob(ctx context.Context, owner string, j store.Job) {
jctx, stop := context.WithCancel(ctx)
defer stop()
defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }()
// The lease kept while working; lost, the work stops and another
// gateway has it.
go func() {
t := time.NewTicker(jobLease / 4)
defer t.Stop()
for {
select {
case <-jctx.Done():
return
case <-t.C:
if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held {
stop()
return
}
}
}
}()
if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil {
r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err)
}
}
// serveJob replays the job's request and keeps what came back. A failure
// of the gateway's own (the store unreachable) is returned, and the job is
// picked up again; a refusal of the request is the job's end.
func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
now := time.Now()
switch {
case j.Status == store.JobCancelling:
j.Status, j.FinishedAt = store.JobCancelled, now
return r.store.UpdateJob(ctx, *j)
case now.After(j.ExpiresAt):
j.Status, j.FinishedAt = store.JobExpired, now
j.Error = fmt.Sprintf("not finished within %s", jobWindow)
return r.store.UpdateJob(ctx, *j)
}
if j.Status == store.JobQueued {
j.Status, j.StartedAt = store.JobRunning, now
if err := r.store.UpdateJob(ctx, *j); err != nil {
return err
}
}
// A job found running was another gateway's, which died with it. It is
// run again from the start: the engine's queue does not outlive the
// request that was waiting on it.
fail := func(msg string) error {
j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now()
return r.store.UpdateJob(ctx, *j)
}
p, err := r.principalOfJob(ctx, *j)
if err != nil {
return fail("the key or account this job was made with can no longer be used: " + err.Error())
}
// A cancel asked for while it runs ends the request, which takes the
// work out of the engine's queue.
rctx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
t := time.NewTicker(jobCancelCheck)
defer t.Stop()
for {
select {
case <-rctx.Done():
return
case <-t.C:
if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling {
cancel()
return
}
}
}
}()
handler := r.handleInference(j.Endpoint)
var unplaced time.Time // when it was first said to fit nowhere
for attempt := 1; ; attempt++ {
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
if err != nil {
return fail(err.Error())
}
hreq.Header.Set("Content-Type", "application/json")
hreq.Header.Set("X-Request-Id", j.ID)
hreq.Header.Set(tagsHeader, "job="+j.ID)
w := &bufferWriter{header: http.Header{}, max: jobMaxResult}
handler(w, hreq)
if rctx.Err() != nil {
if ctx.Err() != nil {
// The lease was lost or the gateway is stopping: another
// gateway's to pick up.
return ctx.Err()
}
j.Status, j.FinishedAt = store.JobCancelled, time.Now()
return r.store.UpdateJob(ctx, *j)
}
status := w.status
if status == 0 {
status = http.StatusOK
}
// Waited out, for as long as the job has: a model loading, a rate
// limit, a queue full. Nobody is holding a connection, which is the
// point of a job. Not a spent budget, which says no Retry-After
// because waiting does not help.
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
time.Now().Before(j.ExpiresAt) {
if by := w.header.Get(waitingForHeader); by != "" {
// Kept as its progress: whoever asks after it is told what
// it waits on, rather than nothing at all.
p := map[string]any{"state": "waiting_gpu", "held_by": by}
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
p["eta_seconds"] = eta
}
if b, err := json.Marshal(p); err == nil {
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
}
}
secs, _ := strconv.Atoi(wait)
select {
case <-rctx.Done():
continue // said above
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
}
continue
}
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
if unplaced.IsZero() {
unplaced = time.Now()
}
if time.Since(unplaced) < jobUnplacedGrace {
select {
case <-rctx.Done():
case <-time.After(jobUnplacedRetry):
}
continue
}
}
if status >= 300 {
return fail(errorMessage(w.buf.Bytes()))
}
ct := w.header.Get("Content-Type")
if ct == "" {
ct = http.DetectContentType(w.buf.Bytes())
}
_, _, _, days := r.batchCfg()
done := time.Now()
f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput,
Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)}
if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil {
return err
}
// Said done while it may still be said: the last word heard was the
// node running, and a finished job saying it is running reads as
// one that stopped halfway.
_ = r.store.SetJobProgress(ctx, j.ID, `{"state":"done","percent":100}`, "")
j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done
return r.store.UpdateJob(ctx, *j)
}
}
// extensionFor names a result's file by what it is.
func extensionFor(contentType string) string {
ct, _, _ := strings.Cut(contentType, ";")
switch strings.TrimSpace(ct) {
case "application/json":
return ".json"
case "model/gltf-binary":
return ".glb"
case "model/gltf+json":
return ".gltf"
case "model/obj":
return ".obj"
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/webp":
return ".webp"
case "image/gif":
return ".gif"
case "video/mp4":
return ".mp4"
case "video/webm":
return ".webm"
case "video/quicktime":
return ".mov"
case "audio/wav", "audio/wave", "audio/x-wav":
return ".wav"
case "audio/mpeg":
return ".mp3"
case "audio/flac":
return ".flac"
case "audio/ogg":
return ".ogg"
}
return ".bin"
}
// principalOfJob is who a job acts for, as they stand now: the key it was
// made with, or — for one made from the portal — the person whose session
// made it, while they may still use the gateway and the project the job is
// charged to.
func (r *Router) principalOfJob(ctx context.Context, j store.Job) (auth.Principal, error) {
uid, session := strings.CutPrefix(j.KeyID, "session:")
if !session {
return r.principalOfKey(ctx, j.KeyID, "")
}
u, err := r.store.UserByID(ctx, uid)
if err != nil {
return auth.Principal{}, err
}
if u.State != store.UserActive {
return auth.Principal{}, fmt.Errorf("the account that made it is %s", u.State)
}
project, _ := strings.CutPrefix(j.Owner, "project:")
if !strings.HasPrefix(j.Owner, "project:") {
project = ""
} else if ok, err := r.store.CanUseProject(ctx, uid, project); err != nil || !ok {
return auth.Principal{}, errors.New("its account is no longer a member of the project it is charged to")
}
eff, err := r.store.ResolveFor(ctx, j.KeyID, uid, project)
if err != nil {
return auth.Principal{}, err
}
return auth.Principal{KeyID: j.KeyID, KeyPrefix: "session", UserID: uid, ProjectID: eff.ProjectID,
TeamID: eff.TeamID, OrgID: eff.OrgID, Band: eff.Band, RoutingProfile: eff.RoutingProfile, Limits: eff}, nil
}