A 3D model is a job: queued with an id, asked after, its result a file
POST /v1/3d/generations answers 202 with a job; GET says its status, progress and result file; cancel stops it in the engine's queue too. The request is kept as sent and replayed by whichever gateway holds the lease through the path a client's request takes, so it is charged and judged like any request (tagged job=<id>), and a model still loading is waited out for as long as the job has. Migration 0044. Nothing makes 3D models yet; the door refuses every model until the ComfyUI adapter learns meshes. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
baaaa4e019
commit
b51cc3cf92
16 files changed
+1198
-10
No files matched your search
@@ -148,7 +148,7 @@ func (r *Router) serveLines(ctx context.Context, b *store.Batch, lines []batchLi
|
||||
for _, d := range done {
|
||||
served[d.Line] = true
|
||||
}
|
||||
p, perr := r.batchPrincipalOf(ctx, b.KeyID)
|
||||
p, perr := r.principalOfKey(ctx, b.KeyID, r.batchBand())
|
||||
|
||||
var mu sync.Mutex
|
||||
stopWith := ""
|
||||
@@ -221,9 +221,10 @@ func (r *Router) serveLines(ctx context.Context, b *store.Batch, lines []batchLi
|
||||
return stopWith, nil
|
||||
}
|
||||
|
||||
// batchPrincipalOf is the key a batch was made with, as it stands now: a key
|
||||
// revoked since serves no more of it.
|
||||
func (r *Router) batchPrincipalOf(ctx context.Context, keyID string) (auth.Principal, error) {
|
||||
// principalOfKey is the key a batch or a job was made with, as it stands now:
|
||||
// a key revoked since serves no more of it. band is the admission band the
|
||||
// work goes in; empty is the key's own.
|
||||
func (r *Router) principalOfKey(ctx context.Context, keyID, band string) (auth.Principal, error) {
|
||||
k, err := r.store.KeyByID(ctx, keyID)
|
||||
if err != nil {
|
||||
return auth.Principal{}, err
|
||||
@@ -238,10 +239,13 @@ func (r *Router) batchPrincipalOf(ctx context.Context, keyID string) (auth.Princ
|
||||
if err != nil {
|
||||
return auth.Principal{}, err
|
||||
}
|
||||
if band == "" {
|
||||
band = eff.Band
|
||||
}
|
||||
return auth.Principal{
|
||||
KeyID: k.ID, KeyPrefix: k.Prefix, UserID: eff.UserID, ProjectID: eff.ProjectID,
|
||||
TeamID: eff.TeamID, OrgID: eff.OrgID,
|
||||
Band: r.batchBand(), RoutingProfile: eff.RoutingProfile, Limits: eff,
|
||||
Band: band, RoutingProfile: eff.RoutingProfile, Limits: eff,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -123,6 +123,12 @@ func (r *Router) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /v1/batches", r.handleListBatches)
|
||||
mux.HandleFunc("GET /v1/batches/{id}", r.handleGetBatch)
|
||||
mux.HandleFunc("POST /v1/batches/{id}/cancel", r.handleCancelBatch)
|
||||
// Jobs: 3D models, which take minutes and are asked after rather than
|
||||
// waited for. The result is a file.
|
||||
mux.HandleFunc("POST "+meshesPath, r.handleCreateJob)
|
||||
mux.HandleFunc("GET "+meshesPath, r.handleListJobs)
|
||||
mux.HandleFunc("GET "+meshesPath+"/{id}", r.handleGetJob)
|
||||
mux.HandleFunc("POST "+meshesPath+"/{id}/cancel", r.handleCancelJob)
|
||||
// Knowledge bases: documents a person's llmgw-kb read and embedded.
|
||||
r.registerKB(mux)
|
||||
// Liveness: this process is up. It says nothing about whether it can
|
||||
|
||||
@@ -111,6 +111,15 @@ func parseImageRequest(body []byte) (imageRequest, error) {
|
||||
// endpoint may serve as it likes.
|
||||
func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
|
||||
m, known := r.modelMap()[model]
|
||||
if path == meshesPath {
|
||||
// Until a model makes them (BACKLOG G3), every model the catalog
|
||||
// knows is one that does not; one it does not know is a
|
||||
// hand-configured endpoint's to answer.
|
||||
if known {
|
||||
return fmt.Sprintf("%s does not make 3D models", m.Name)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if path == imagesPath {
|
||||
if known && !m.MakesImages() {
|
||||
return fmt.Sprintf("%s does not make pictures", m.Name)
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Jobs: a request that takes minutes, answered with an id and asked after.
|
||||
//
|
||||
// A 3D model from a picture takes minutes on a card that first has to be
|
||||
// emptied of whatever else was on it. That is not a request to hold a socket
|
||||
// open for (activate.go says why), so it is a job: queued with 202, its
|
||||
// status and progress read back, its result downloaded as a file. The
|
||||
// request is kept as the client sent it and replayed, by whichever gateway
|
||||
// holds the job's lease, through the same path a client's request takes —
|
||||
// key, allowlist, limits, admission, guardrails, accounting — so a job is
|
||||
// charged and judged like any request, tagged job=<id>.
|
||||
|
||||
// meshesPath is where 3D models are asked for. The synchronous path behind
|
||||
// it is served to the job worker only; clients get a job.
|
||||
const meshesPath = "/v1/3d/generations"
|
||||
|
||||
const (
|
||||
purposeJobOutput = "job_output"
|
||||
// jobWindow is how long a job may take, queue included, before it is
|
||||
// given up on.
|
||||
jobWindow = 24 * time.Hour
|
||||
)
|
||||
|
||||
func (r *Router) jobJSON(ctx context.Context, j store.Job) map[string]any {
|
||||
out := map[string]any{
|
||||
"id": j.ID, "object": "job", "endpoint": j.Endpoint, "model": j.Model, "status": j.Status,
|
||||
"progress": nil, "output_file_id": strOrNil(j.OutputFileID), "content_type": strOrNil(j.ContentType),
|
||||
"error": nil, "created_at": j.CreatedAt.Unix(), "started_at": unixOrNil(j.StartedAt),
|
||||
"finished_at": unixOrNil(j.FinishedAt), "expires_at": unixOrNil(j.ExpiresAt), "usage": nil,
|
||||
}
|
||||
if j.Progress != "" && json.Valid([]byte(j.Progress)) {
|
||||
out["progress"] = json.RawMessage(j.Progress)
|
||||
}
|
||||
if j.Error != "" {
|
||||
out["error"] = map[string]string{"message": j.Error}
|
||||
}
|
||||
// What the work cost, from the usage record the replayed request left
|
||||
// under the job's id.
|
||||
if store.JobDone(j.Status) && r.store != nil {
|
||||
if u, ok, err := r.store.UsageByRequest(ctx, j.ID); err == nil && ok {
|
||||
out["usage"] = map[string]int64{"prompt_tokens": u.PromptTokens, "completion_tokens": u.CompletionTokens,
|
||||
"images": u.Images}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handleCreateJob queues a request for a 3D model. Refused now: no key, a
|
||||
// model that is not there or not allowed, a request the model cannot serve.
|
||||
// Refused when it is served, and said as the job's error: limits, the
|
||||
// guardrails, a model that will not come up.
|
||||
func (r *Router) handleCreateJob(w http.ResponseWriter, req *http.Request) {
|
||||
p, ok := r.batchPrincipal(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "could not read the request: "+err.Error())
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if in.Model == "" {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "model is required")
|
||||
return
|
||||
}
|
||||
if _, known := r.modelMap()[in.Model]; !known || r.hiddenModel(in.Model) {
|
||||
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found",
|
||||
fmt.Sprintf("the model %q does not exist", in.Model))
|
||||
return
|
||||
}
|
||||
if !r.permits(p, in.Model) {
|
||||
auth.WriteForbiddenModel(w, in.Model)
|
||||
return
|
||||
}
|
||||
if why := r.imageRouteRefusal(meshesPath, in.Model, body); why != "" {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
j := store.Job{ID: newID("job_"), Owner: ownerOf(p), KeyID: p.KeyID, Endpoint: meshesPath, Model: in.Model,
|
||||
Input: body, Status: store.JobQueued, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
|
||||
if err := r.store.PutJob(req.Context(), j); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
r.wakeJobs()
|
||||
writeJSON(w, http.StatusAccepted, r.jobJSON(req.Context(), j))
|
||||
}
|
||||
|
||||
// ownedJob is a job the caller owns, or a 404 said.
|
||||
func (r *Router) ownedJob(w http.ResponseWriter, req *http.Request, owner string) (store.Job, bool) {
|
||||
j, err := r.store.JobByID(req.Context(), req.PathValue("id"))
|
||||
if errors.Is(err, store.ErrNotFound) || err == nil && j.Owner != owner {
|
||||
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "job_not_found",
|
||||
fmt.Sprintf("no job %q", req.PathValue("id")))
|
||||
return store.Job{}, false
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return store.Job{}, false
|
||||
}
|
||||
return j, true
|
||||
}
|
||||
|
||||
func (r *Router) handleGetJob(w http.ResponseWriter, req *http.Request) {
|
||||
p, ok := r.batchPrincipal(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if j, ok := r.ownedJob(w, req, ownerOf(p)); ok {
|
||||
writeJSON(w, http.StatusOK, r.jobJSON(req.Context(), j))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Router) handleCancelJob(w http.ResponseWriter, req *http.Request) {
|
||||
p, ok := r.batchPrincipal(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
j, ok := r.ownedJob(w, req, ownerOf(p))
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
j, err := r.store.CancelJob(req.Context(), j.ID, time.Now())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
r.wakeJobs()
|
||||
writeJSON(w, http.StatusOK, r.jobJSON(req.Context(), j))
|
||||
}
|
||||
|
||||
func (r *Router) handleListJobs(w http.ResponseWriter, req *http.Request) {
|
||||
p, ok := r.batchPrincipal(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, _ := strconv.Atoi(req.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
js, err := r.store.Jobs(req.Context(), ownerOf(p), req.URL.Query().Get("after"), limit+1)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
more := len(js) > limit
|
||||
if more {
|
||||
js = js[:limit]
|
||||
}
|
||||
data := make([]map[string]any, 0, len(js))
|
||||
for _, j := range js {
|
||||
data = append(data, r.jobJSON(req.Context(), j))
|
||||
}
|
||||
out := map[string]any{"object": "list", "data": data, "has_more": more, "first_id": nil, "last_id": nil}
|
||||
if len(js) > 0 {
|
||||
out["first_id"], out["last_id"] = js[0].ID, js[len(js)-1].ID
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// errorMessage is what an answer said went wrong: the message of an OpenAI
|
||||
// error body, or the body itself, kept short.
|
||||
func errorMessage(body []byte) string {
|
||||
var e struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(body, &e) == nil && e.Error.Message != "" {
|
||||
return e.Error.Message
|
||||
}
|
||||
s := strings.TrimSpace(string(body))
|
||||
if len(s) > 300 {
|
||||
s = s[:300] + "…"
|
||||
}
|
||||
if s == "" {
|
||||
return "no reason given"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// jobComfy is a ComfyUI that makes one picture a workflow, slowly when
|
||||
// asked: stall holds every workflow until it is cleared, and the paths the
|
||||
// router took to stop one are counted.
|
||||
type jobComfy struct {
|
||||
srv *httptest.Server
|
||||
stall atomic.Bool
|
||||
prompts atomic.Int32
|
||||
cancelled atomic.Int32
|
||||
}
|
||||
|
||||
func newJobComfy(t *testing.T) *jobComfy {
|
||||
t.Helper()
|
||||
c := &jobComfy{}
|
||||
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/prompt":
|
||||
c.prompts.Add(1)
|
||||
fmt.Fprint(w, `{"prompt_id":"p1"}`)
|
||||
case r.URL.Path == "/history/p1":
|
||||
if c.stall.Load() {
|
||||
fmt.Fprint(w, `{}`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"p1":{"status":{"status_str":"success","completed":true},"outputs":{"7":{"images":[{"filename":"0.png","type":"temp"}]}}}}`)
|
||||
case r.URL.Path == "/view":
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
fmt.Fprint(w, "png-bytes")
|
||||
case r.URL.Path == "/queue" && r.Method == http.MethodPost, r.URL.Path == "/interrupt":
|
||||
c.cancelled.Add(1)
|
||||
fmt.Fprint(w, `{}`)
|
||||
case r.URL.Path == "/queue":
|
||||
fmt.Fprint(w, `{"queue_running":[[0,"p1"]],"queue_pending":[]}`)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(c.srv.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
// jobRouter is a gateway with the store, a key, a text model and a ComfyUI
|
||||
// model priced by the picture.
|
||||
func jobRouter(t *testing.T, c *jobComfy) (*Router, string) {
|
||||
t.Helper()
|
||||
cfg := baseConfig()
|
||||
cfg.Models = []catalog.Model{catModel("chat", nil), {Name: "flux", Source: "external",
|
||||
External: &catalog.External{URL: c.srv.URL, Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}}}
|
||||
cfg.Auth.Enabled = true
|
||||
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
||||
cfg.Auth.Pricing = map[string]config.Price{"flux": {PerImage: 5000}}
|
||||
cfg.Auth.FlushInterval = 20 * time.Millisecond
|
||||
cfg.Auth.BatchSize = 1
|
||||
rt := newTestRouter(t, cfg)
|
||||
rt.comfy = &comfy.Client{Poll: time.Millisecond}
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
|
||||
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
|
||||
})
|
||||
org, _ := rt.store.CreateOrg(t.Context(), "acme")
|
||||
team, _ := rt.store.CreateTeam(t.Context(), org, "platform")
|
||||
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := store.KeyPrefix + strings.Repeat("a", 43)
|
||||
sum := sha256.Sum256([]byte(secret))
|
||||
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-1", Hash: sum[:], Prefix: "aaaa", Name: "test",
|
||||
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rt, secret
|
||||
}
|
||||
|
||||
// queueJob puts a job in the store as the create endpoint would, for an
|
||||
// endpoint that serves today: until a model makes 3D models, the worker is
|
||||
// tried with pictures.
|
||||
func queueJob(t *testing.T, rt *Router, id, status string, body string) store.Job {
|
||||
t.Helper()
|
||||
now := time.Now()
|
||||
j := store.Job{ID: id, Owner: "project:" + mustProjectOf(t, rt, "key-1"), KeyID: "key-1", Endpoint: imagesPath,
|
||||
Model: "flux", Input: []byte(body), Status: status, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
|
||||
if err := rt.store.PutJob(t.Context(), j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
func waitJob(t *testing.T, h http.Handler, secret, id string, until func(map[string]any) bool) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
_, j, raw := call(t, h, secret, http.MethodGet, meshesPath+"/"+id, nil)
|
||||
if until(j) {
|
||||
return j
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the job did not get there: %s", raw)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func jobDone(j map[string]any) bool { return store.JobDone(fmt.Sprint(j["status"])) }
|
||||
|
||||
func TestAJobIsServedItsResultKeptAndItsWorkCharged(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
h := rt.Handler()
|
||||
j := queueJob(t, rt, "job_1", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
||||
rt.wakeJobs()
|
||||
|
||||
got := waitJob(t, h, secret, j.ID, jobDone)
|
||||
if got["status"] != "completed" || got["content_type"] != "application/json" || got["error"] != nil {
|
||||
t.Fatalf("%v", got)
|
||||
}
|
||||
if got["started_at"] == nil || got["finished_at"] == nil {
|
||||
t.Errorf("times: %v", got)
|
||||
}
|
||||
code, _, out := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
|
||||
if code != 200 || !strings.Contains(out, `"b64_json":"cG5nLWJ5dGVz"`) {
|
||||
t.Errorf("result %d: %s", code, out)
|
||||
}
|
||||
_, f, _ := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string), nil)
|
||||
if f["purpose"] != purposeJobOutput || f["filename"] != "job_1.json" || f["expires_at"] == nil {
|
||||
t.Errorf("file %v", f)
|
||||
}
|
||||
|
||||
// Charged like any request, by the picture, and findable by the job.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
u, ok, _ := rt.store.UsageByRequest(t.Context(), j.ID)
|
||||
if ok {
|
||||
if u.Tags["job"] != j.ID || u.Images != 1 || u.CostMicros != 5000 {
|
||||
t.Errorf("usage %+v", u)
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the job left no usage")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
got = waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["usage"] != nil })
|
||||
if u := got["usage"].(map[string]any); u["images"] != 1.0 {
|
||||
t.Errorf("usage shown %v", u)
|
||||
}
|
||||
_, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil)
|
||||
if d := list["data"].([]any); len(d) != 1 {
|
||||
t.Errorf("listed %v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJobIsRefusedBeforeItIsQueuedWhenItCannotBeServed(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
h := rt.Handler()
|
||||
code, _, raw := call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "chat", "prompt": "a cube"})
|
||||
if code != 400 || !strings.Contains(raw, "does not make 3D models") {
|
||||
t.Errorf("text model: %d %s", code, raw)
|
||||
}
|
||||
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "nope", "prompt": "a cube"})
|
||||
if code != 404 || !strings.Contains(raw, "model_not_found") {
|
||||
t.Errorf("unknown model: %d %s", code, raw)
|
||||
}
|
||||
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"prompt": "a cube"})
|
||||
if code != 400 || !strings.Contains(raw, "model is required") {
|
||||
t.Errorf("no model: %d %s", code, raw)
|
||||
}
|
||||
if code, _, _ := call(t, h, "", http.MethodPost, meshesPath, map[string]any{"model": "chat"}); code != 401 {
|
||||
t.Errorf("no key: %d", code)
|
||||
}
|
||||
if _, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil); len(list["data"].([]any)) != 0 {
|
||||
t.Errorf("a refused job was queued: %v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARunningJobIsCancelledAndTheEngineToldToStop(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
c.stall.Store(true)
|
||||
rt, secret := jobRouter(t, c)
|
||||
h := rt.Handler()
|
||||
j := queueJob(t, rt, "job_slow", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
||||
rt.wakeJobs()
|
||||
waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["status"] == "running" })
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for c.prompts.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
code, got, raw := call(t, h, secret, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil)
|
||||
if code != 200 || got["status"] != "cancelling" {
|
||||
t.Fatalf("cancel: %d %s", code, raw)
|
||||
}
|
||||
got = waitJob(t, h, secret, j.ID, jobDone)
|
||||
if got["status"] != "cancelled" || got["output_file_id"] != nil {
|
||||
t.Errorf("%v", got)
|
||||
}
|
||||
if c.cancelled.Load() == 0 {
|
||||
t.Error("ComfyUI was not told to stop")
|
||||
}
|
||||
|
||||
// One not yet taken ends at once, and is never sent.
|
||||
before := c.prompts.Load()
|
||||
j2 := queueJob(t, rt, "job_never", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
||||
_, got, _ = call(t, h, secret, http.MethodPost, meshesPath+"/"+j2.ID+"/cancel", nil)
|
||||
if got["status"] != "cancelled" {
|
||||
t.Errorf("%v", got)
|
||||
}
|
||||
rt.wakeJobs()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if c.prompts.Load() != before {
|
||||
t.Error("a cancelled job was sent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJobWhoseGatewayDiedIsTakenByAnother(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
h := rt.Handler()
|
||||
// As a gateway that died mid-job left it: running, its lease run out.
|
||||
j := queueJob(t, rt, "job_orphan", store.JobRunning, `{"model":"flux","prompt":"a cube"}`)
|
||||
j.StartedAt = time.Now().Add(-time.Minute)
|
||||
_ = rt.store.UpdateJob(t.Context(), j)
|
||||
rt.wakeJobs()
|
||||
if got := waitJob(t, h, secret, j.ID, jobDone); got["status"] != "completed" {
|
||||
t.Errorf("%v", got)
|
||||
}
|
||||
|
||||
// A refusal of the request is the job's end, said.
|
||||
bad := queueJob(t, rt, "job_bad", store.JobQueued, `{"model":"flux","prompt":"a cube","n":99}`)
|
||||
rt.wakeJobs()
|
||||
got := waitJob(t, h, secret, bad.ID, jobDone)
|
||||
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "n is between 1 and 4") {
|
||||
t.Errorf("%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobsAreTheirOwners(t *testing.T) {
|
||||
c := newJobComfy(t)
|
||||
rt, secret := jobRouter(t, c)
|
||||
h := rt.Handler()
|
||||
j := queueJob(t, rt, "job_mine", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
||||
rt.wakeJobs()
|
||||
waitJob(t, h, secret, j.ID, jobDone)
|
||||
|
||||
org, _ := rt.store.CreateOrg(t.Context(), "other")
|
||||
team, _ := rt.store.CreateTeam(t.Context(), org, "t")
|
||||
project, _ := rt.store.CreateProject(t.Context(), team, "p")
|
||||
other := store.KeyPrefix + strings.Repeat("b", 43)
|
||||
sum := sha256.Sum256([]byte(other))
|
||||
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
|
||||
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _, _ := call(t, h, other, http.MethodGet, meshesPath+"/"+j.ID, nil); code != 404 {
|
||||
t.Errorf("another project read a job: %d", code)
|
||||
}
|
||||
if code, _, _ := call(t, h, other, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil); code != 404 {
|
||||
t.Errorf("another project cancelled a job: %d", code)
|
||||
}
|
||||
_, mine, _ := call(t, h, secret, http.MethodGet, meshesPath+"/"+j.ID, nil)
|
||||
if code, _, _ := call(t, h, other, http.MethodGet, "/v1/files/"+mine["output_file_id"].(string)+"/content", nil); code != 404 {
|
||||
t.Errorf("another project read a job's result: %d", code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// The worker that serves jobs. Any gateway on the store may run one: a job
|
||||
// is worked on by whichever holds its lease, and one whose gateway died is
|
||||
// taken by another when the lease runs out.
|
||||
|
||||
const (
|
||||
jobLease = 2 * time.Minute
|
||||
jobPoll = 5 * time.Second
|
||||
// jobCancelCheck is how often a running job looks for a cancel.
|
||||
jobCancelCheck = 2 * time.Second
|
||||
// jobMaxResult bounds what one job may answer: a mesh, not a film.
|
||||
jobMaxResult = 256 << 20
|
||||
)
|
||||
|
||||
// wakeJobs has the worker look for work now rather than at its next poll.
|
||||
func (r *Router) wakeJobs() {
|
||||
select {
|
||||
case r.jobWake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Router) runJobs(ctx context.Context) {
|
||||
host, _ := os.Hostname()
|
||||
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
|
||||
t := time.NewTicker(jobPoll)
|
||||
defer t.Stop()
|
||||
for {
|
||||
for {
|
||||
now := time.Now()
|
||||
j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease))
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
r.log.Warn("could not look for jobs to serve", "err", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
r.workJob(ctx, owner, j)
|
||||
}
|
||||
// A job's record outlives its file by nothing: both go at the
|
||||
// retention.
|
||||
_, _, _, days := r.batchCfg()
|
||||
if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 {
|
||||
r.log.Info("jobs past their retention forgotten", "jobs", n)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
case <-r.jobWake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// workJob takes one job as far as it goes, holding its lease the while.
|
||||
func (r *Router) workJob(ctx context.Context, owner string, j store.Job) {
|
||||
jctx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }()
|
||||
// The lease kept while working; lost, the work stops and another
|
||||
// gateway has it.
|
||||
go func() {
|
||||
t := time.NewTicker(jobLease / 4)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-jctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held {
|
||||
stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil {
|
||||
r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// serveJob replays the job's request and keeps what came back. A failure
|
||||
// of the gateway's own (the store unreachable) is returned, and the job is
|
||||
// picked up again; a refusal of the request is the job's end.
|
||||
func (r *Router) serveJob(ctx context.Context, j *store.Job) error {
|
||||
now := time.Now()
|
||||
switch {
|
||||
case j.Status == store.JobCancelling:
|
||||
j.Status, j.FinishedAt = store.JobCancelled, now
|
||||
return r.store.UpdateJob(ctx, *j)
|
||||
case now.After(j.ExpiresAt):
|
||||
j.Status, j.FinishedAt = store.JobExpired, now
|
||||
j.Error = fmt.Sprintf("not finished within %s", jobWindow)
|
||||
return r.store.UpdateJob(ctx, *j)
|
||||
}
|
||||
if j.Status == store.JobQueued {
|
||||
j.Status, j.StartedAt = store.JobRunning, now
|
||||
if err := r.store.UpdateJob(ctx, *j); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A job found running was another gateway's, which died with it. It is
|
||||
// run again from the start: the engine's queue does not outlive the
|
||||
// request that was waiting on it.
|
||||
fail := func(msg string) error {
|
||||
j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now()
|
||||
return r.store.UpdateJob(ctx, *j)
|
||||
}
|
||||
p, err := r.principalOfKey(ctx, j.KeyID, "")
|
||||
if err != nil {
|
||||
return fail("the key this job was made with can no longer be used: " + err.Error())
|
||||
}
|
||||
|
||||
// A cancel asked for while it runs ends the request, which takes the
|
||||
// work out of the engine's queue.
|
||||
rctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() {
|
||||
t := time.NewTicker(jobCancelCheck)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-rctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
handler := r.handleInference(j.Endpoint)
|
||||
for attempt := 1; ; attempt++ {
|
||||
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input))
|
||||
if err != nil {
|
||||
return fail(err.Error())
|
||||
}
|
||||
hreq.Header.Set("Content-Type", "application/json")
|
||||
hreq.Header.Set("X-Request-Id", j.ID)
|
||||
hreq.Header.Set(tagsHeader, "job="+j.ID)
|
||||
w := &bufferWriter{header: http.Header{}, max: jobMaxResult}
|
||||
handler(w, hreq)
|
||||
if rctx.Err() != nil {
|
||||
if ctx.Err() != nil {
|
||||
// The lease was lost or the gateway is stopping: another
|
||||
// gateway's to pick up.
|
||||
return ctx.Err()
|
||||
}
|
||||
j.Status, j.FinishedAt = store.JobCancelled, time.Now()
|
||||
return r.store.UpdateJob(ctx, *j)
|
||||
}
|
||||
status := w.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
// Waited out, for as long as the job has: a model loading, a rate
|
||||
// limit, a queue full. Nobody is holding a connection, which is the
|
||||
// point of a job. Not a spent budget, which says no Retry-After
|
||||
// because waiting does not help.
|
||||
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
|
||||
time.Now().Before(j.ExpiresAt) {
|
||||
secs, _ := strconv.Atoi(wait)
|
||||
select {
|
||||
case <-rctx.Done():
|
||||
continue // said above
|
||||
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if status >= 300 {
|
||||
return fail(errorMessage(w.buf.Bytes()))
|
||||
}
|
||||
ct := w.header.Get("Content-Type")
|
||||
if ct == "" {
|
||||
ct = http.DetectContentType(w.buf.Bytes())
|
||||
}
|
||||
_, _, _, days := r.batchCfg()
|
||||
done := time.Now()
|
||||
f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput,
|
||||
Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)}
|
||||
if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done
|
||||
return r.store.UpdateJob(ctx, *j)
|
||||
}
|
||||
}
|
||||
|
||||
// extensionFor names a result's file by what it is.
|
||||
func extensionFor(contentType string) string {
|
||||
ct, _, _ := strings.Cut(contentType, ";")
|
||||
switch strings.TrimSpace(ct) {
|
||||
case "application/json":
|
||||
return ".json"
|
||||
case "model/gltf-binary":
|
||||
return ".glb"
|
||||
case "model/gltf+json":
|
||||
return ".gltf"
|
||||
case "model/obj":
|
||||
return ".obj"
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
}
|
||||
return ".bin"
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package router
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
@@ -60,6 +59,9 @@ type bufferWriter struct {
|
||||
header http.Header
|
||||
status int
|
||||
buf bytes.Buffer
|
||||
// max bounds what is kept; 1 MiB when unset, which is a judge's answer
|
||||
// many times over. A job's result is a file, and sets more.
|
||||
max int
|
||||
}
|
||||
|
||||
func (b *bufferWriter) Header() http.Header { return b.header }
|
||||
@@ -72,8 +74,12 @@ func (b *bufferWriter) Write(p []byte) (int, error) {
|
||||
if b.status == 0 {
|
||||
b.status = http.StatusOK
|
||||
}
|
||||
if b.buf.Len()+len(p) > 1<<20 {
|
||||
return 0, errors.New("a judge's answer over 1 MiB")
|
||||
limit := b.max
|
||||
if limit <= 0 {
|
||||
limit = 1 << 20
|
||||
}
|
||||
if b.buf.Len()+len(p) > limit {
|
||||
return 0, fmt.Errorf("an answer over %d bytes", limit)
|
||||
}
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
@@ -77,8 +77,10 @@ type Router struct {
|
||||
canaries canaryStats
|
||||
// slo keeps each model's recent token timings, for admission.
|
||||
slo sloTracker
|
||||
// batchWake has the batch worker look for work at once.
|
||||
// batchWake has the batch worker look for work at once; jobWake the
|
||||
// job worker.
|
||||
batchWake chan struct{}
|
||||
jobWake chan struct{}
|
||||
// reqLogs is each project's keeping of requests, as last read.
|
||||
reqLogs requestLogs
|
||||
// answers are the answers projects reuse; nil reuses none.
|
||||
@@ -304,6 +306,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
|
||||
r := &Router{
|
||||
batchWake: make(chan struct{}, 1),
|
||||
jobWake: make(chan struct{}, 1),
|
||||
cfg: cfg.Router,
|
||||
landing: cfg.Landing,
|
||||
log: log,
|
||||
@@ -528,6 +531,7 @@ func (r *Router) Start(ctx context.Context) {
|
||||
}
|
||||
if r.store != nil {
|
||||
go r.runBatches(ctx)
|
||||
go r.runJobs(ctx)
|
||||
go r.pruneRequestBodies(ctx)
|
||||
go r.refreshTenants(ctx)
|
||||
go r.purgeSessions(ctx)
|
||||
|
||||
Reference in new issue
Block a user