Batches: OpenAI's Files and Batch APIs, served at the lowest priority under a store lease, each line a request of the key that made it

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 21:00:01 +02:00
1 parent 889be70b8f
commit 841ee21c12
19 files changed
+1775 -10

No files matched your search

+345
View File
@@ -0,0 +1,345 @@
package store
import (
"context"
"database/sql"
"errors"
"time"
)
// File is an uploaded or written file of the Files API, without its bytes.
type File struct {
ID string
Owner string
Purpose string
Filename string
Bytes int64
CreatedAt time.Time
// ExpiresAt is when it is deleted by itself; zero is never.
ExpiresAt time.Time
}
// PutFile keeps a file and its bytes.
func (s *DB) PutFile(ctx context.Context, f File, content []byte) error {
_, err := s.db.ExecContext(ctx, `INSERT INTO files (id, owner, purpose, filename, bytes, content, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, f.ID, f.Owner, f.Purpose, f.Filename, int64(len(content)), content,
millis(f.CreatedAt), optMillis(f.ExpiresAt))
return err
}
const fileCols = `id, owner, purpose, filename, bytes, created_at, expires_at`
func scanFile(sc interface{ Scan(...any) error }) (File, error) {
var f File
var created, expires int64
if err := sc.Scan(&f.ID, &f.Owner, &f.Purpose, &f.Filename, &f.Bytes, &created, &expires); err != nil {
return File{}, err
}
f.CreatedAt = time.UnixMilli(created)
if expires > 0 {
f.ExpiresAt = time.UnixMilli(expires)
}
return f, nil
}
// FileByID finds a file, ErrNotFound when there is none.
func (s *DB) FileByID(ctx context.Context, id string) (File, error) {
f, err := scanFile(s.db.QueryRowContext(ctx, `SELECT `+fileCols+` FROM files WHERE id = ?`, id))
if errors.Is(err, sql.ErrNoRows) {
return File{}, ErrNotFound
}
return f, err
}
// FileContent is a file's bytes.
func (s *DB) FileContent(ctx context.Context, id string) ([]byte, error) {
var b []byte
err := s.db.QueryRowContext(ctx, `SELECT content FROM files WHERE id = ?`, id).Scan(&b)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
return b, err
}
// Files lists an owner's files, the newest first; purpose narrows it.
func (s *DB) Files(ctx context.Context, owner, purpose string, limit int) ([]File, error) {
q := `SELECT ` + fileCols + ` FROM files WHERE owner = ?`
args := []any{owner}
if purpose != "" {
q += ` AND purpose = ?`
args = append(args, purpose)
}
if limit <= 0 || limit > 10000 {
limit = 10000
}
q += ` ORDER BY created_at DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []File
for rows.Next() {
f, err := scanFile(rows)
if err != nil {
return nil, err
}
out = append(out, f)
}
return out, rows.Err()
}
// DeleteFile removes a file; false when there was none.
func (s *DB) DeleteFile(ctx context.Context, id string) (bool, error) {
res, err := s.db.ExecContext(ctx, `DELETE FROM files WHERE id = ?`, id)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// DeleteExpiredFiles removes the files past their expiry.
func (s *DB) DeleteExpiredFiles(ctx context.Context, now time.Time) (int64, error) {
res, err := s.db.ExecContext(ctx, `DELETE FROM files WHERE expires_at > 0 AND expires_at < ?`, millis(now))
if err != nil {
return 0, err
}
return res.RowsAffected()
}
// Batch is one batch of the Batch API.
type Batch struct {
ID string
Owner string
KeyID string
Endpoint string
InputFileID string
CompletionWindow string
Status string
OutputFileID string
ErrorFileID string
Total int
Completed int
Failed int
CreatedAt time.Time
InProgressAt time.Time
FinalizingAt time.Time
CompletedAt time.Time
FailedAt time.Time
ExpiredAt time.Time
CancellingAt time.Time
CancelledAt time.Time
ExpiresAt time.Time
// Metadata is the client's, as JSON; Errors why validation failed, as
// JSON.
Metadata string
Errors string
}
// Batch states, as OpenAI names them.
const (
BatchValidating = "validating"
BatchFailed = "failed"
BatchInProgress = "in_progress"
BatchFinalizing = "finalizing"
BatchCompleted = "completed"
BatchExpired = "expired"
BatchCancelling = "cancelling"
BatchCancelled = "cancelled"
)
// BatchDone is whether a state is an end.
func BatchDone(status string) bool {
switch status {
case BatchFailed, BatchCompleted, BatchExpired, BatchCancelled:
return true
}
return false
}
const batchCols = `id, owner, key_id, endpoint, input_file_id, completion_window, status, output_file_id,
error_file_id, total, completed, failed, created_at, in_progress_at, finalizing_at, completed_at,
failed_at, expired_at, cancelling_at, cancelled_at, expires_at, metadata, errors`
func scanBatch(sc interface{ Scan(...any) error }) (Batch, error) {
var b Batch
var t [10]int64
err := sc.Scan(&b.ID, &b.Owner, &b.KeyID, &b.Endpoint, &b.InputFileID, &b.CompletionWindow, &b.Status,
&b.OutputFileID, &b.ErrorFileID, &b.Total, &b.Completed, &b.Failed,
&t[0], &t[1], &t[2], &t[3], &t[4], &t[5], &t[6], &t[7], &t[8], &b.Metadata, &b.Errors)
if err != nil {
return Batch{}, err
}
at := func(v int64) time.Time {
if v == 0 {
return time.Time{}
}
return time.UnixMilli(v)
}
b.CreatedAt, b.InProgressAt, b.FinalizingAt, b.CompletedAt = at(t[0]), at(t[1]), at(t[2]), at(t[3])
b.FailedAt, b.ExpiredAt, b.CancellingAt, b.CancelledAt, b.ExpiresAt = at(t[4]), at(t[5]), at(t[6]), at(t[7]), at(t[8])
return b, nil
}
// PutBatch records a new batch.
func (s *DB) PutBatch(ctx context.Context, b Batch) error {
_, err := s.db.ExecContext(ctx, `INSERT INTO batches (id, owner, key_id, endpoint, input_file_id,
completion_window, status, created_at, expires_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
b.ID, b.Owner, b.KeyID, b.Endpoint, b.InputFileID, b.CompletionWindow, b.Status,
millis(b.CreatedAt), millis(b.ExpiresAt), b.Metadata)
return err
}
// UpdateBatch writes a batch's progress: its state, counts, files, times
// and errors.
func (s *DB) UpdateBatch(ctx context.Context, b Batch) error {
_, err := s.db.ExecContext(ctx, `UPDATE batches SET status = ?, output_file_id = ?, error_file_id = ?,
total = ?, completed = ?, failed = ?, in_progress_at = ?, finalizing_at = ?, completed_at = ?,
failed_at = ?, expired_at = ?, cancelling_at = ?, cancelled_at = ?, errors = ? WHERE id = ?`,
b.Status, b.OutputFileID, b.ErrorFileID, b.Total, b.Completed, b.Failed,
optMillis(b.InProgressAt), optMillis(b.FinalizingAt), optMillis(b.CompletedAt), optMillis(b.FailedAt),
optMillis(b.ExpiredAt), optMillis(b.CancellingAt), optMillis(b.CancelledAt), b.Errors, b.ID)
return err
}
// CancelBatch asks for a batch to stop: cancelling, unless it has ended
// already. It says the batch as it stands.
func (s *DB) CancelBatch(ctx context.Context, id string, now time.Time) (Batch, error) {
if _, err := s.db.ExecContext(ctx, `UPDATE batches SET status = ?, cancelling_at = ?
WHERE id = ? AND status IN (?, ?, ?)`, BatchCancelling, millis(now), id,
BatchValidating, BatchInProgress, BatchFinalizing); err != nil {
return Batch{}, err
}
return s.BatchByID(ctx, id)
}
// BatchByID finds a batch, ErrNotFound when there is none.
func (s *DB) BatchByID(ctx context.Context, id string) (Batch, error) {
b, err := scanBatch(s.db.QueryRowContext(ctx, `SELECT `+batchCols+` FROM batches WHERE id = ?`, id))
if errors.Is(err, sql.ErrNoRows) {
return Batch{}, ErrNotFound
}
return b, err
}
// Batches lists an owner's batches, the newest first, those made before
// the batch after when it is given.
func (s *DB) Batches(ctx context.Context, owner, after string, limit int) ([]Batch, error) {
q := `SELECT ` + batchCols + ` FROM batches WHERE owner = ?`
args := []any{owner}
if after != "" {
q += ` AND created_at < (SELECT created_at FROM batches WHERE id = ?)`
args = append(args, after)
}
if limit <= 0 || limit > 100 {
limit = 20
}
q += ` ORDER BY created_at DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Batch
for rows.Next() {
b, err := scanBatch(rows)
if err != nil {
return nil, err
}
out = append(out, b)
}
return out, rows.Err()
}
// ClaimBatch takes the lease on one batch that has work left and that no
// gateway holds, for owner until the time given. ok is false when there is
// none.
func (s *DB) ClaimBatch(ctx context.Context, owner string, now, until time.Time) (Batch, bool, error) {
var id string
err := s.db.QueryRowContext(ctx, `UPDATE batches SET lease_owner = ?, lease_until = ?
WHERE id = (SELECT id FROM batches WHERE status IN (?, ?, ?, ?) AND lease_until < ?
ORDER BY created_at LIMIT 1)
AND lease_until < ?
RETURNING id`, owner, millis(until), BatchValidating, BatchInProgress, BatchFinalizing, BatchCancelling,
millis(now), millis(now)).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return Batch{}, false, nil
}
if err != nil {
return Batch{}, false, err
}
b, err := s.BatchByID(ctx, id)
return b, err == nil, err
}
// RenewBatchLease keeps a lease held; false when it is held no longer.
func (s *DB) RenewBatchLease(ctx context.Context, id, owner string, until time.Time) (bool, error) {
res, err := s.db.ExecContext(ctx, `UPDATE batches SET lease_until = ? WHERE id = ? AND lease_owner = ?`,
millis(until), id, owner)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// ReleaseBatch gives a lease up.
func (s *DB) ReleaseBatch(ctx context.Context, id, owner string) error {
_, err := s.db.ExecContext(ctx, `UPDATE batches SET lease_until = 0, lease_owner = '' WHERE id = ? AND lease_owner = ?`, id, owner)
return err
}
// BatchResult is one line of a batch, served.
type BatchResult struct {
Line int
OK bool
Data string
}
// PutBatchResult keeps one served line; a line kept already is left as it
// was.
func (s *DB) PutBatchResult(ctx context.Context, batchID string, r BatchResult) error {
ok := 0
if r.OK {
ok = 1
}
_, err := s.db.ExecContext(ctx, `INSERT INTO batch_results (batch_id, line, ok, data) VALUES (?, ?, ?, ?)
ON CONFLICT (batch_id, line) DO NOTHING`, batchID, r.Line, ok, r.Data)
return err
}
// BatchResults are a batch's served lines, in order.
func (s *DB) BatchResults(ctx context.Context, batchID string) ([]BatchResult, error) {
rows, err := s.db.QueryContext(ctx, `SELECT line, ok, data FROM batch_results WHERE batch_id = ? ORDER BY line`, batchID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []BatchResult
for rows.Next() {
var r BatchResult
var ok int
if err := rows.Scan(&r.Line, &ok, &r.Data); err != nil {
return nil, err
}
r.OK = ok == 1
out = append(out, r)
}
return out, rows.Err()
}
// DeleteBatchResults forgets a batch's lines, once its files are written.
func (s *DB) DeleteBatchResults(ctx context.Context, batchID string) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM batch_results WHERE batch_id = ?`, batchID)
return err
}
func optMillis(t time.Time) int64 {
if t.IsZero() {
return 0
}
return t.UnixMilli()
}
+80
View File
@@ -0,0 +1,80 @@
package store
import (
"errors"
"testing"
"time"
)
func TestFilesAndBatches(t *testing.T) {
checkFilesAndBatches(t, capacityStore(t))
}
func TestPostgresFilesAndBatches(t *testing.T) {
checkFilesAndBatches(t, openPG(t))
}
func checkFilesAndBatches(t *testing.T, db *DB) {
ctx := t.Context()
now := time.Now()
if err := db.PutFile(ctx, File{ID: "file-1", Owner: "p1", Purpose: "batch", Filename: "in.jsonl", CreatedAt: now},
[]byte("{\"a\":1}\n")); err != nil {
t.Fatal(err)
}
if err := db.PutFile(ctx, File{ID: "file-old", Owner: "p1", Purpose: "batch_output", Filename: "out.jsonl",
CreatedAt: now, ExpiresAt: now.Add(-time.Minute)}, []byte("x")); err != nil {
t.Fatal(err)
}
f, err := db.FileByID(ctx, "file-1")
if err != nil || f.Bytes != 8 || f.Owner != "p1" {
t.Fatalf("%+v %v", f, err)
}
if b, _ := db.FileContent(ctx, "file-1"); string(b) != "{\"a\":1}\n" {
t.Errorf("content %q", b)
}
if fs, _ := db.Files(ctx, "p1", "batch", 0); len(fs) != 1 {
t.Errorf("listed %v", fs)
}
if n, _ := db.DeleteExpiredFiles(ctx, now); n != 1 {
t.Errorf("expired %d", n)
}
b := Batch{ID: "batch-1", Owner: "p1", KeyID: "k", Endpoint: "/v1/chat/completions", InputFileID: "file-1",
CompletionWindow: "24h", Status: BatchValidating, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
if err := db.PutBatch(ctx, b); err != nil {
t.Fatal(err)
}
got, ok, err := db.ClaimBatch(ctx, "gw-a", now, now.Add(time.Minute))
if err != nil || !ok || got.ID != "batch-1" {
t.Fatalf("claim: %+v %v %v", got, ok, err)
}
if _, ok, _ := db.ClaimBatch(ctx, "gw-b", now, now.Add(time.Minute)); ok {
t.Error("a leased batch was claimed twice")
}
if held, _ := db.RenewBatchLease(ctx, "batch-1", "gw-b", now.Add(time.Hour)); held {
t.Error("another gateway renewed the lease")
}
got.Status, got.Total, got.InProgressAt = BatchInProgress, 2, now
if err := db.UpdateBatch(ctx, got); err != nil {
t.Fatal(err)
}
for _, r := range []BatchResult{{Line: 1, OK: false, Data: "e"}, {Line: 0, OK: true, Data: "a"}, {Line: 0, OK: false, Data: "dup"}} {
if err := db.PutBatchResult(ctx, "batch-1", r); err != nil {
t.Fatal(err)
}
}
rs, _ := db.BatchResults(ctx, "batch-1")
if len(rs) != 2 || rs[0].Data != "a" || !rs[0].OK || rs[1].OK {
t.Errorf("results %+v", rs)
}
c, err := db.CancelBatch(ctx, "batch-1", now)
if err != nil || c.Status != BatchCancelling || c.Total != 2 || c.CancellingAt.IsZero() {
t.Errorf("cancel %+v %v", c, err)
}
if bs, _ := db.Batches(ctx, "p1", "", 0); len(bs) != 1 {
t.Errorf("listed %d", len(bs))
}
if _, err := db.BatchByID(ctx, "nope"); !errors.Is(err, ErrNotFound) {
t.Errorf("missing: %v", err)
}
}
+3
View File
@@ -58,6 +58,9 @@ var copyOrder = []string{
// Who changed what is not lost by moving the store.
"audit_events",
"key_ttl",
"files",
"batches",
"batch_results",
}
// notCopied are tables deliberately left behind.
@@ -0,0 +1,56 @@
-- Ported from the SQLite migration of the same number.
-- The Files and Batch APIs. A file is what a client uploaded (a batch's
-- requests) or what a batch wrote (its answers, its errors), owned by the
-- project of the key that made it. A batch is a file of requests served one
-- by one at the lowest priority; each finished line is kept as it is
-- served, so a batch survives the control plane restarting under it, and
-- the lease says which gateway is working on it.
CREATE TABLE files (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
purpose TEXT NOT NULL,
filename TEXT NOT NULL,
bytes BIGINT NOT NULL,
content BYTEA NOT NULL,
created_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX files_owner ON files (owner, created_at);
CREATE TABLE batches (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
key_id TEXT NOT NULL,
endpoint TEXT NOT NULL,
input_file_id TEXT NOT NULL,
completion_window TEXT NOT NULL,
status TEXT NOT NULL,
output_file_id TEXT NOT NULL DEFAULT '',
error_file_id TEXT NOT NULL DEFAULT '',
total BIGINT NOT NULL DEFAULT 0,
completed BIGINT NOT NULL DEFAULT 0,
failed BIGINT NOT NULL DEFAULT 0,
created_at BIGINT NOT NULL,
in_progress_at BIGINT NOT NULL DEFAULT 0,
finalizing_at BIGINT NOT NULL DEFAULT 0,
completed_at BIGINT NOT NULL DEFAULT 0,
failed_at BIGINT NOT NULL DEFAULT 0,
expired_at BIGINT NOT NULL DEFAULT 0,
cancelling_at BIGINT NOT NULL DEFAULT 0,
cancelled_at BIGINT NOT NULL DEFAULT 0,
expires_at BIGINT NOT NULL,
metadata TEXT NOT NULL DEFAULT '',
errors TEXT NOT NULL DEFAULT '',
lease_owner TEXT NOT NULL DEFAULT '',
lease_until BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX batches_owner ON batches (owner, created_at);
CREATE INDEX batches_status ON batches (status, lease_until);
CREATE TABLE batch_results (
batch_id TEXT NOT NULL,
line BIGINT NOT NULL,
ok BIGINT NOT NULL,
data TEXT NOT NULL,
PRIMARY KEY (batch_id, line)
);
@@ -0,0 +1,55 @@
-- The Files and Batch APIs. A file is what a client uploaded (a batch's
-- requests) or what a batch wrote (its answers, its errors), owned by the
-- project of the key that made it. A batch is a file of requests served one
-- by one at the lowest priority; each finished line is kept as it is
-- served, so a batch survives the control plane restarting under it, and
-- the lease says which gateway is working on it.
CREATE TABLE files (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
purpose TEXT NOT NULL,
filename TEXT NOT NULL,
bytes INTEGER NOT NULL,
content BLOB NOT NULL,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX files_owner ON files (owner, created_at);
CREATE TABLE batches (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
key_id TEXT NOT NULL,
endpoint TEXT NOT NULL,
input_file_id TEXT NOT NULL,
completion_window TEXT NOT NULL,
status TEXT NOT NULL,
output_file_id TEXT NOT NULL DEFAULT '',
error_file_id TEXT NOT NULL DEFAULT '',
total INTEGER NOT NULL DEFAULT 0,
completed INTEGER NOT NULL DEFAULT 0,
failed INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
in_progress_at INTEGER NOT NULL DEFAULT 0,
finalizing_at INTEGER NOT NULL DEFAULT 0,
completed_at INTEGER NOT NULL DEFAULT 0,
failed_at INTEGER NOT NULL DEFAULT 0,
expired_at INTEGER NOT NULL DEFAULT 0,
cancelling_at INTEGER NOT NULL DEFAULT 0,
cancelled_at INTEGER NOT NULL DEFAULT 0,
expires_at INTEGER NOT NULL,
metadata TEXT NOT NULL DEFAULT '',
errors TEXT NOT NULL DEFAULT '',
lease_owner TEXT NOT NULL DEFAULT '',
lease_until INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX batches_owner ON batches (owner, created_at);
CREATE INDEX batches_status ON batches (status, lease_until);
CREATE TABLE batch_results (
batch_id TEXT NOT NULL,
line INTEGER NOT NULL,
ok INTEGER NOT NULL,
data TEXT NOT NULL,
PRIMARY KEY (batch_id, line)
);
+1 -1
View File
@@ -43,7 +43,7 @@ func openPG(t *testing.T) *DB {
// every subtest gets a fresh file, and failed only here.
`TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships,
key_claims, limits, models, sessions, users, projects, teams,
organizations, leadership, control_planes RESTART IDENTITY CASCADE`)
organizations, leadership, control_planes, files, batches, batch_results RESTART IDENTITY CASCADE`)
if err != nil {
t.Fatalf("clearing the test database: %v", err)
}