Files
llmgw/internal/store/jobs.go
T
Ilker ManapandClaude Opus 5.5 0460f52dd2
CI / test (push) Successful in 10m28s
Jobs run side by side, one per model at a time
One job at a time, a 3D job waiting for the card held up another model's
job that could have run (seen on the PoC, BACKLOG H7). The leader's worker
claims jobs while fewer than eight run, passing over the models that have
one running, and starts each in its own goroutine; one ending wakes it for
the next. Lease and leader-only rules are as they were.

BACKLOG: H5–H7 marked, H8 (presets) and H9 (what the PoC taught) written.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 10:06:20 +02:00

253 lines
8.1 KiB
Go

package store
import (
"context"
"database/sql"
"errors"
"strings"
"time"
)
// Job is one request that is answered with an id and asked after: work that
// takes minutes, which no client should hold a connection for. The request
// is kept as it was sent and replayed by whichever gateway holds the lease.
type Job struct {
ID string
Owner string
KeyID string
Endpoint string
Model string
// Input is the request's body, as the client sent it.
Input []byte
Status string
// Progress is what the engine said of the work so far, as JSON; empty
// until it said anything.
Progress string
// PromptID is the engine's own id for the work, so a gateway that takes
// over a running job asks after the same work rather than queueing it
// again.
PromptID string
OutputFileID string
ContentType string
Error string
CreatedAt time.Time
StartedAt time.Time
FinishedAt time.Time
// ExpiresAt is when a job not yet finished is given up on.
ExpiresAt time.Time
}
// Job states.
const (
JobQueued = "queued"
JobRunning = "running"
JobCompleted = "completed"
JobFailed = "failed"
JobCancelling = "cancelling"
JobCancelled = "cancelled"
JobExpired = "expired"
)
// JobDone is whether a state is an end.
func JobDone(status string) bool {
switch status {
case JobCompleted, JobFailed, JobCancelled, JobExpired:
return true
}
return false
}
const jobCols = `id, owner, key_id, endpoint, model, input, status, progress, prompt_id, output_file_id,
content_type, error, created_at, started_at, finished_at, expires_at`
func scanJob(sc interface{ Scan(...any) error }) (Job, error) {
var j Job
var created, started, finished, expires int64
err := sc.Scan(&j.ID, &j.Owner, &j.KeyID, &j.Endpoint, &j.Model, &j.Input, &j.Status, &j.Progress, &j.PromptID,
&j.OutputFileID, &j.ContentType, &j.Error, &created, &started, &finished, &expires)
if err != nil {
return Job{}, err
}
at := func(v int64) time.Time {
if v == 0 {
return time.Time{}
}
return time.UnixMilli(v)
}
j.CreatedAt, j.StartedAt, j.FinishedAt, j.ExpiresAt = at(created), at(started), at(finished), at(expires)
return j, nil
}
// PutJob records a new job.
func (s *DB) PutJob(ctx context.Context, j Job) error {
if j.Input == nil {
j.Input = []byte{}
}
_, err := s.db.ExecContext(ctx, `INSERT INTO jobs (id, owner, key_id, endpoint, model, input, status, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
j.ID, j.Owner, j.KeyID, j.Endpoint, j.Model, j.Input, j.Status, millis(j.CreatedAt), millis(j.ExpiresAt))
return err
}
// UpdateJob writes how a job stands: its state, result, error and times.
// Its progress and prompt id are written by SetJobProgress, which the work
// calls often and which must not undo an end written here.
func (s *DB) UpdateJob(ctx context.Context, j Job) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, output_file_id = ?, content_type = ?, error = ?,
started_at = ?, finished_at = ? WHERE id = ?`,
j.Status, j.OutputFileID, j.ContentType, j.Error, optMillis(j.StartedAt), optMillis(j.FinishedAt), j.ID)
return err
}
// SetJobProgress writes what the engine said of a running job, and its id
// for the work when it is known. A job that has ended keeps its end.
func (s *DB) SetJobProgress(ctx context.Context, id, progress, promptID string) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET progress = ?,
prompt_id = CASE WHEN ? = '' THEN prompt_id ELSE ? END
WHERE id = ? AND status IN (?, ?)`, progress, promptID, promptID, id, JobRunning, JobCancelling)
return err
}
// CancelJob asks for a job to stop: cancelling, unless it has ended already.
// It says the job as it stands.
func (s *DB) CancelJob(ctx context.Context, id string, now time.Time) (Job, error) {
// One not yet taken ends at once: there is nothing to stop.
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, finished_at = ?
WHERE id = ? AND status = ? AND lease_until < ?`, JobCancelled, millis(now), id, JobQueued, millis(now)); err != nil {
return Job{}, err
}
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ? AND status IN (?, ?)`,
JobCancelling, id, JobQueued, JobRunning); err != nil {
return Job{}, err
}
return s.JobByID(ctx, id)
}
// JobByID finds a job, ErrNotFound when there is none.
func (s *DB) JobByID(ctx context.Context, id string) (Job, error) {
j, err := scanJob(s.db.QueryRowContext(ctx, `SELECT `+jobCols+` FROM jobs WHERE id = ?`, id))
if errors.Is(err, sql.ErrNoRows) {
return Job{}, ErrNotFound
}
return j, err
}
// Jobs lists an owner's jobs, the newest first, those made before the job
// after when it is given.
func (s *DB) Jobs(ctx context.Context, owner, after string, limit int) ([]Job, error) {
q := `SELECT ` + jobCols + ` FROM jobs WHERE owner = ?`
args := []any{owner}
if after != "" {
q += ` AND created_at < (SELECT created_at FROM jobs WHERE id = ?)`
args = append(args, after)
}
if limit <= 0 || limit > 100 {
limit = 20
}
q += ` ORDER BY created_at DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Job
for rows.Next() {
j, err := scanJob(rows)
if err != nil {
return nil, err
}
out = append(out, j)
}
return out, rows.Err()
}
// AllJobs is every owner's jobs, newest first, those in one status when it
// is named: what an operator looks over.
func (s *DB) AllJobs(ctx context.Context, status string, limit int) ([]Job, error) {
q := `SELECT ` + jobCols + ` FROM jobs`
var args []any
if status != "" {
q += ` WHERE status = ?`
args = append(args, status)
}
if limit <= 0 || limit > 500 {
limit = 50
}
q += ` ORDER BY created_at DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Job
for rows.Next() {
j, err := scanJob(rows)
if err != nil {
return nil, err
}
out = append(out, j)
}
return out, rows.Err()
}
// ClaimJob takes the lease on the oldest job with work left that no gateway
// holds, for owner until the time given. ok is false when there is none.
//
// Jobs for the models in skip are passed over: a worker running as many of a
// model's jobs as it will at once takes the next job for another model
// rather than waiting behind it.
func (s *DB) ClaimJob(ctx context.Context, owner string, now, until time.Time, skip ...string) (Job, bool, error) {
var id string
args := []any{owner, millis(until), JobQueued, JobRunning, JobCancelling, millis(now)}
notIn := ""
if len(skip) > 0 {
notIn = " AND model NOT IN (?" + strings.Repeat(", ?", len(skip)-1) + ")"
for _, m := range skip {
args = append(args, m)
}
}
args = append(args, millis(now))
err := s.db.QueryRowContext(ctx, `UPDATE jobs SET lease_owner = ?, lease_until = ?
WHERE id = (SELECT id FROM jobs WHERE status IN (?, ?, ?) AND lease_until < ?`+notIn+`
ORDER BY created_at LIMIT 1)
AND lease_until < ?
RETURNING id`, args...).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return Job{}, false, nil
}
if err != nil {
return Job{}, false, err
}
j, err := s.JobByID(ctx, id)
return j, err == nil, err
}
// RenewJobLease keeps a lease held; false when it is held no longer.
func (s *DB) RenewJobLease(ctx context.Context, id, owner string, until time.Time) (bool, error) {
res, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = ? WHERE id = ? AND lease_owner = ?`,
millis(until), id, owner)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// ReleaseJob gives a lease up.
func (s *DB) ReleaseJob(ctx context.Context, id, owner string) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = 0, lease_owner = '' WHERE id = ? AND lease_owner = ?`, id, owner)
return err
}
// DeleteJobsBefore forgets the jobs that ended before the time given, whose
// result files are gone by then too.
func (s *DB) DeleteJobsBefore(ctx context.Context, before time.Time) (int64, error) {
res, err := s.db.ExecContext(ctx, `DELETE FROM jobs WHERE finished_at > 0 AND finished_at < ?`, millis(before))
if err != nil {
return 0, err
}
return res.RowsAffected()
}