package router import ( "bytes" "context" "errors" "fmt" "net/http" "os" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // The worker that serves jobs. Any gateway on the store may run one: a job // is worked on by whichever holds its lease, and one whose gateway died is // taken by another when the lease runs out. const ( jobLease = 2 * time.Minute jobPoll = 5 * time.Second // jobCancelCheck is how often a running job looks for a cancel. jobCancelCheck = 2 * time.Second // jobMaxResult bounds what one job may answer: a mesh, not a film. jobMaxResult = 256 << 20 ) // wakeJobs has the worker look for work now rather than at its next poll. func (r *Router) wakeJobs() { select { case r.jobWake <- struct{}{}: default: } } func (r *Router) runJobs(ctx context.Context) { host, _ := os.Hostname() owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID("")) t := time.NewTicker(jobPoll) defer t.Stop() for { for { now := time.Now() j, ok, err := r.store.ClaimJob(ctx, owner, now, now.Add(jobLease)) if err != nil { if ctx.Err() == nil { r.log.Warn("could not look for jobs to serve", "err", err) } break } if !ok { break } r.workJob(ctx, owner, j) } // A job's record outlives its file by nothing: both go at the // retention. _, _, _, days := r.batchCfg() if n, err := r.store.DeleteJobsBefore(ctx, time.Now().AddDate(0, 0, -days)); err == nil && n > 0 { r.log.Info("jobs past their retention forgotten", "jobs", n) } select { case <-ctx.Done(): return case <-t.C: case <-r.jobWake: } } } // workJob takes one job as far as it goes, holding its lease the while. func (r *Router) workJob(ctx context.Context, owner string, j store.Job) { jctx, stop := context.WithCancel(ctx) defer stop() defer func() { _ = r.store.ReleaseJob(context.WithoutCancel(ctx), j.ID, owner) }() // The lease kept while working; lost, the work stops and another // gateway has it. go func() { t := time.NewTicker(jobLease / 4) defer t.Stop() for { select { case <-jctx.Done(): return case <-t.C: if held, err := r.store.RenewJobLease(jctx, j.ID, owner, time.Now().Add(jobLease)); err == nil && !held { stop() return } } } }() if err := r.serveJob(jctx, &j); err != nil && jctx.Err() == nil { r.log.Warn("a job stopped with an error; it is picked up again", "job", j.ID, "err", err) } } // serveJob replays the job's request and keeps what came back. A failure // of the gateway's own (the store unreachable) is returned, and the job is // picked up again; a refusal of the request is the job's end. func (r *Router) serveJob(ctx context.Context, j *store.Job) error { now := time.Now() switch { case j.Status == store.JobCancelling: j.Status, j.FinishedAt = store.JobCancelled, now return r.store.UpdateJob(ctx, *j) case now.After(j.ExpiresAt): j.Status, j.FinishedAt = store.JobExpired, now j.Error = fmt.Sprintf("not finished within %s", jobWindow) return r.store.UpdateJob(ctx, *j) } if j.Status == store.JobQueued { j.Status, j.StartedAt = store.JobRunning, now if err := r.store.UpdateJob(ctx, *j); err != nil { return err } } // A job found running was another gateway's, which died with it. It is // run again from the start: the engine's queue does not outlive the // request that was waiting on it. fail := func(msg string) error { j.Status, j.Error, j.FinishedAt = store.JobFailed, msg, time.Now() return r.store.UpdateJob(ctx, *j) } p, err := r.principalOfJob(ctx, *j) if err != nil { return fail("the key or account this job was made with can no longer be used: " + err.Error()) } // A cancel asked for while it runs ends the request, which takes the // work out of the engine's queue. rctx, cancel := context.WithCancel(ctx) defer cancel() go func() { t := time.NewTicker(jobCancelCheck) defer t.Stop() for { select { case <-rctx.Done(): return case <-t.C: if cur, err := r.store.JobByID(rctx, j.ID); err == nil && cur.Status == store.JobCancelling { cancel() return } } } }() handler := r.handleInference(j.Endpoint) for attempt := 1; ; attempt++ { hreq, err := http.NewRequestWithContext(auth.WithPrincipal(rctx, p), http.MethodPost, j.Endpoint, bytes.NewReader(j.Input)) if err != nil { return fail(err.Error()) } hreq.Header.Set("Content-Type", "application/json") hreq.Header.Set("X-Request-Id", j.ID) hreq.Header.Set(tagsHeader, "job="+j.ID) w := &bufferWriter{header: http.Header{}, max: jobMaxResult} handler(w, hreq) if rctx.Err() != nil { if ctx.Err() != nil { // The lease was lost or the gateway is stopping: another // gateway's to pick up. return ctx.Err() } j.Status, j.FinishedAt = store.JobCancelled, time.Now() return r.store.UpdateJob(ctx, *j) } status := w.status if status == 0 { status = http.StatusOK } // Waited out, for as long as the job has: a model loading, a rate // limit, a queue full. Nobody is holding a connection, which is the // point of a job. Not a spent budget, which says no Retry-After // because waiting does not help. if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) && time.Now().Before(j.ExpiresAt) { secs, _ := strconv.Atoi(wait) select { case <-rctx.Done(): continue // said above case <-time.After(time.Duration(max(secs, 1)) * time.Second): } continue } if status >= 300 { return fail(errorMessage(w.buf.Bytes())) } ct := w.header.Get("Content-Type") if ct == "" { ct = http.DetectContentType(w.buf.Bytes()) } _, _, _, days := r.batchCfg() done := time.Now() f := store.File{ID: newID("file-"), Owner: j.Owner, Purpose: purposeJobOutput, Filename: j.ID + extensionFor(ct), Bytes: int64(w.buf.Len()), CreatedAt: done, ExpiresAt: done.AddDate(0, 0, days)} if err := r.store.PutFile(ctx, f, w.buf.Bytes()); err != nil { return err } // Said done while it may still be said: the last word heard was the // node running, and a finished job saying it is running reads as // one that stopped halfway. _ = r.store.SetJobProgress(ctx, j.ID, `{"state":"done","percent":100}`, "") j.Status, j.OutputFileID, j.ContentType, j.FinishedAt = store.JobCompleted, f.ID, ct, done return r.store.UpdateJob(ctx, *j) } } // extensionFor names a result's file by what it is. func extensionFor(contentType string) string { ct, _, _ := strings.Cut(contentType, ";") switch strings.TrimSpace(ct) { case "application/json": return ".json" case "model/gltf-binary": return ".glb" case "model/gltf+json": return ".gltf" case "model/obj": return ".obj" case "image/png": return ".png" case "image/jpeg": return ".jpg" case "image/webp": return ".webp" } return ".bin" } // principalOfJob is who a job acts for, as they stand now: the key it was // made with, or — for one made from the portal — the person whose session // made it, while they may still use the gateway and the project the job is // charged to. func (r *Router) principalOfJob(ctx context.Context, j store.Job) (auth.Principal, error) { uid, session := strings.CutPrefix(j.KeyID, "session:") if !session { return r.principalOfKey(ctx, j.KeyID, "") } u, err := r.store.UserByID(ctx, uid) if err != nil { return auth.Principal{}, err } if u.State != store.UserActive { return auth.Principal{}, fmt.Errorf("the account that made it is %s", u.State) } project, _ := strings.CutPrefix(j.Owner, "project:") if !strings.HasPrefix(j.Owner, "project:") { project = "" } else if ok, err := r.store.CanUseProject(ctx, uid, project); err != nil || !ok { return auth.Principal{}, errors.New("its account is no longer a member of the project it is charged to") } eff, err := r.store.ResolveFor(ctx, j.KeyID, uid, project) if err != nil { return auth.Principal{}, err } return auth.Principal{KeyID: j.KeyID, KeyPrefix: "session", UserID: uid, ProjectID: eff.ProjectID, TeamID: eff.TeamID, OrgID: eff.OrgID, Band: eff.Band, RoutingProfile: eff.RoutingProfile, Limits: eff}, nil }