A model's catalog entry now says what its workflow takes and makes:
comfyui.inputs, each a placeholder of its name with a type (text, int,
float, bool, choice, image, mask, audio, video, file), bounds, choices
and a default; and comfyui.output, which gains video, audio and files
beside pictures and meshes. POST /v1/workflows/runs with {model, inputs}
checks a request against them before a job is written — a required input
left out, an unknown one, a number out of bounds, a song sent as a
picture, another owner's file id are each refused with what was wrong —
and the job worker replays it to a replica, where the files go into
ComfyUI's input directory, the workflow is filled and run, and the first
file of the model's output kind is the job's result. /v1/models lists
each ComfyUI model's workflow, so a client can build its form.
A PNG result carries the licence in an iTXt chunk, as a GLB does in
asset.extras. A file whose server named no type is typed by its name
before its bytes, so a WAV is audio/wav, not audio/wave.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
316 lines
9.6 KiB
Go
316 lines
9.6 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// The worker that serves jobs. Any gateway on the store may run one: a job
|
|
// is worked on by whichever holds its lease, and one whose gateway died is
|
|
// taken by another when the lease runs out.
|
|
|
|
const (
|
|
jobLease = 2 * time.Minute
|
|
jobPoll = 5 * time.Second
|
|
// jobCancelCheck is how often a running job looks for a cancel.
|
|
jobCancelCheck = 2 * time.Second
|
|
// jobMaxResult bounds what one job may answer: a mesh, a short video.
|
|
jobMaxResult = 256 << 20
|
|
)
|
|
|
|
// jobUnplacedGrace is how long a job keeps asking for a model said to fit
|
|
// nowhere before that is its end, and jobUnplacedRetry how often it asks.
|
|
// "Fits nowhere" is said for a moment by a cluster changing under it — an
|
|
// agent restarting re-registers without its engines for a few seconds — and
|
|
// a job failed by that moment was failed by nothing wrong with it.
|
|
var (
|
|
jobUnplacedGrace = 5 * time.Minute
|
|
jobUnplacedRetry = 15 * time.Second
|
|
)
|
|
|
|
// wakeJobs has the worker look for work now rather than at its next poll.
|
|
func (r *Router) wakeJobs() {
|
|
select {
|
|
case r.jobWake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (r *Router) runJobs(ctx context.Context) {
|
|
host, _ := os.Hostname()
|
|
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
|
|
t := time.NewTicker(jobPoll)
|
|
defer t.Stop()
|
|
for {
|
|
for r.leading() {
|
|
now := time.Now()
|
|
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
|
|
if err != nil {
|
|
if ctx.Err() == nil {
|
|
r.log.Warn("could not look for jobs to serve", "err", err)
|
|
}
|
|
break
|
|
}
|
|
if !ok {
|
|
break
|
|
}
|
|
r.workJob(ctx, owner, j)
|
|
}
|
|
// A job's record outlives its file by nothing: both go at the
|
|
// retention.
|
|
_, _, _, days := r.batchCfg()
|
|
if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 {
|
|
r.log.Info("jobs past their retention forgotten", "jobs", n)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
case <-r.jobWake:
|
|
}
|
|
}
|
|
}
|
|
|
|
// workJob takes one job as far as it goes, holding its lease the while.
|
|
func (r *Router) workJob(ctx context.Context, owner string, j store.Job) {
|
|
jctx, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }()
|
|
// The lease kept while working; lost, the work stops and another
|
|
// gateway has it.
|
|
go func() {
|
|
t := time.NewTicker(jobLease / 4)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-jctx.Done():
|
|
return
|
|
case <-t.C:
|
|
if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held {
|
|
stop()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil {
|
|
r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err)
|
|
}
|
|
}
|
|
|
|
// serveJob replays the job's request and keeps what came back. A failure
|
|
// of the gateway's own (the store unreachable) is returned, and the job is
|
|
// picked up again; a refusal of the request is the job's end.
|
|
func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
|
now := time.Now()
|
|
switch {
|
|
case j.Status == store.JobCancelling:
|
|
j.Status, j.FinishedAt = store.JobCancelled, now
|
|
return r.store.UpdateJob(ctx, *j)
|
|
case now.After(j.ExpiresAt):
|
|
j.Status, j.FinishedAt = store.JobExpired, now
|
|
j.Error = fmt.Sprintf("not finished within %s", jobWindow)
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
if j.Status == store.JobQueued {
|
|
j.Status, j.StartedAt = store.JobRunning, now
|
|
if err := r.store.UpdateJob(ctx, *j); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// A job found running was another gateway's, which died with it. It is
|
|
// run again from the start: the engine's queue does not outlive the
|
|
// request that was waiting on it.
|
|
fail := func(msg string) error {
|
|
j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now()
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
p, err := r.principalOfJob(ctx, *j)
|
|
if err != nil {
|
|
return fail("the key or account this job was made with can no longer be used: " + err.Error())
|
|
}
|
|
|
|
// A cancel asked for while it runs ends the request, which takes the
|
|
// work out of the engine's queue.
|
|
rctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
go func() {
|
|
t := time.NewTicker(jobCancelCheck)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-rctx.Done():
|
|
return
|
|
case <-t.C:
|
|
if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
handler := r.handleInference(j.Endpoint)
|
|
var unplaced time.Time // when it was first said to fit nowhere
|
|
for attempt := 1; ; attempt++ {
|
|
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
|
|
if err != nil {
|
|
return fail(err.Error())
|
|
}
|
|
hreq.Header.Set("Content-Type", "application/json")
|
|
hreq.Header.Set("X-Request-Id", j.ID)
|
|
hreq.Header.Set(tagsHeader, "job="+j.ID)
|
|
w := &bufferWriter{header: http.Header{}, max: jobMaxResult}
|
|
handler(w, hreq)
|
|
if rctx.Err() != nil {
|
|
if ctx.Err() != nil {
|
|
// The lease was lost or the gateway is stopping: another
|
|
// gateway's to pick up.
|
|
return ctx.Err()
|
|
}
|
|
j.Status, j.FinishedAt = store.JobCancelled, time.Now()
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
status := w.status
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
// Waited out, for as long as the job has: a model loading, a rate
|
|
// limit, a queue full. Nobody is holding a connection, which is the
|
|
// point of a job. Not a spent budget, which says no Retry-After
|
|
// because waiting does not help.
|
|
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
|
|
time.Now().Before(j.ExpiresAt) {
|
|
if by := w.header.Get(waitingForHeader); by != "" {
|
|
// Kept as its progress: whoever asks after it is told what
|
|
// it waits on, rather than nothing at all.
|
|
p := map[string]any{"state": "waiting_gpu", "held_by": by}
|
|
if eta, err := strconv.Atoi(w.header.Get(etaHeader)); err == nil && eta > 0 {
|
|
p["eta_seconds"] = eta
|
|
}
|
|
if b, err := json.Marshal(p); err == nil {
|
|
_ = r.store.SetJobProgress(ctx, j.ID, string(b), "")
|
|
}
|
|
}
|
|
secs, _ := strconv.Atoi(wait)
|
|
select {
|
|
case <-rctx.Done():
|
|
continue // said above
|
|
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
|
|
}
|
|
continue
|
|
}
|
|
if status == http.StatusServiceUnavailable && time.Now().Before(j.ExpiresAt) {
|
|
if unplaced.IsZero() {
|
|
unplaced = time.Now()
|
|
}
|
|
if time.Since(unplaced) < jobUnplacedGrace {
|
|
select {
|
|
case <-rctx.Done():
|
|
case <-time.After(jobUnplacedRetry):
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if status >= 300 {
|
|
return fail(errorMessage(w.buf.Bytes()))
|
|
}
|
|
ct := w.header.Get("Content-Type")
|
|
if ct == "" {
|
|
ct = http.DetectContentType(w.buf.Bytes())
|
|
}
|
|
_, _, _, days := r.batchCfg()
|
|
done := time.Now()
|
|
f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput,
|
|
Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)}
|
|
if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
// Said done while it may still be said: the last word heard was the
|
|
// node running, and a finished job saying it is running reads as
|
|
// one that stopped halfway.
|
|
_ = r.store.SetJobProgress(ctx, j.ID, `{"state":"done","percent":100}`, "")
|
|
j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done
|
|
return r.store.UpdateJob(ctx, *j)
|
|
}
|
|
}
|
|
|
|
// extensionFor names a result's file by what it is.
|
|
func extensionFor(contentType string) string {
|
|
ct, _, _ := strings.Cut(contentType, ";")
|
|
switch strings.TrimSpace(ct) {
|
|
case "application/json":
|
|
return ".json"
|
|
case "model/gltf-binary":
|
|
return ".glb"
|
|
case "model/gltf+json":
|
|
return ".gltf"
|
|
case "model/obj":
|
|
return ".obj"
|
|
case "image/png":
|
|
return ".png"
|
|
case "image/jpeg":
|
|
return ".jpg"
|
|
case "image/webp":
|
|
return ".webp"
|
|
case "image/gif":
|
|
return ".gif"
|
|
case "video/mp4":
|
|
return ".mp4"
|
|
case "video/webm":
|
|
return ".webm"
|
|
case "video/quicktime":
|
|
return ".mov"
|
|
case "audio/wav", "audio/wave", "audio/x-wav":
|
|
return ".wav"
|
|
case "audio/mpeg":
|
|
return ".mp3"
|
|
case "audio/flac":
|
|
return ".flac"
|
|
case "audio/ogg":
|
|
return ".ogg"
|
|
}
|
|
return ".bin"
|
|
}
|
|
|
|
// principalOfJob is who a job acts for, as they stand now: the key it was
|
|
// made with, or — for one made from the portal — the person whose session
|
|
// made it, while they may still use the gateway and the project the job is
|
|
// charged to.
|
|
func (r *Router) principalOfJob(ctx context.Context, j store.Job) (auth.Principal, error) {
|
|
uid, session := strings.CutPrefix(j.KeyID, "session:")
|
|
if !session {
|
|
return r.principalOfKey(ctx, j.KeyID, "")
|
|
}
|
|
u, err := r.store.UserByID(ctx, uid)
|
|
if err != nil {
|
|
return auth.Principal{}, err
|
|
}
|
|
if u.State != store.UserActive {
|
|
return auth.Principal{}, fmt.Errorf("the account that made it is %s", u.State)
|
|
}
|
|
project, _ := strings.CutPrefix(j.Owner, "project:")
|
|
if !strings.HasPrefix(j.Owner, "project:") {
|
|
project = ""
|
|
} else if ok, err := r.store.CanUseProject(ctx, uid, project); err != nil || !ok {
|
|
return auth.Principal{}, errors.New("its account is no longer a member of the project it is charged to")
|
|
}
|
|
eff, err := r.store.ResolveFor(ctx, j.KeyID, uid, project)
|
|
if err != nil {
|
|
return auth.Principal{}, err
|
|
}
|
|
return auth.Principal{KeyID: j.KeyID, KeyPrefix: "session", UserID: uid, ProjectID: eff.ProjectID,
|
|
TeamID: eff.TeamID, OrgID: eff.OrgID, Band: eff.Band, RoutingProfile: eff.RoutingProfile, Limits: eff}, nil
|
|
}
|