Files
llmgw/internal/router/mcp_media.go
T
Ilker ManapandClaude Opus 5.5 edeeb9f8e1 MCP takes pictures of real size, by id too, and waits for a cold model
The game's bridge found three things on the PoC. /mcp held its body to
64 KiB — the edit meant for it had raised the memory notes' limit instead —
so a 1024 px concept sent as base64 failed as "expected one JSON-RPC
request object". The notes are back to 64 KiB, /mcp takes 64 MB, and a
call past it is a 413 saying so.

/v1/files took batch files only, so a picture could not be sent once and
named after. purpose vision and user_data are kept too, as long as job
results, and generate_mesh hands a workflow model the id as it is.

generate_image and review_images failed at once while their model was
cold. They wait out an answer that says when to come back — loading,
another model holding the GPU, a full queue — for as long as the call may,
saying who holds the GPU and about how long; a refusal that says no
Retry-After, a spent budget, is said at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 07:24:59 +02:00

981 lines
33 KiB
Go

package router
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"image"
"image/color"
_ "image/jpeg" // a picture sent to look at may be a JPEG
"image/png"
"net/http"
"slices"
"strconv"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// --- MCP: pictures, 3D models, looking at pictures, workflows ---
//
// The gateway's own models offered as tools, so an agent — Claude Code
// driving Blender, say — makes a concept, a mesh from it and has a render
// looked at without a tool server of its own. Each tool makes its request
// of the gateway as the caller would, under the caller's key: the same
// allowlist, limits, licences left out, guardrails and accounting. What it
// makes is kept as a file of the caller's, said by id and by the address
// it is fetched from with the same key, since a tool server over HTTP
// cannot write to the caller's disk.
const (
// mcpBody bounds a tool call: up to four pictures to look at, as base64.
mcpBody = 64 << 20
// previewEdge is the long side of the picture shown to the agent beside
// the file: enough to see it, not a megapixel of tokens.
previewEdge = 512
// lookEdge is the long side of a picture sent to a vision model:
// 1280x720 is about 940 tokens.
lookEdge = 1280
// mcpWaitStreamed is how long a call answered as a stream waits for its
// job, saying how far along it is meanwhile; mcpWaitPlain how long one
// answered in one piece does, under the minute a proxy allows.
mcpWaitStreamed = 10 * time.Minute
mcpWaitPlain = 50 * time.Second
mcpWaitMax = 15 * time.Minute
mcpPoll = 2 * time.Second
)
// The prompt endings of generate_image's styles: what a picture a mesh is
// made from wants, and a surface that tiles.
var imageStyles = map[string]string{
"none": "",
"concept": ", single object centered, orthographic three-quarter view, flat even studio lighting, no shadows, " +
"no background elements, plain flat grey background, clean silhouette, game asset concept, high detail, no text, no watermark",
"side": ", single object, exact side view (profile), orthographic, flat even studio lighting, no shadows, " +
"plain flat grey background, clean silhouette, game asset concept, high detail, no text, no watermark",
"texture": ", seamless tileable texture, top-down orthographic, flat even lighting, no shadows, no perspective, no text",
}
// mcpMediaInstructions is said to an agent when it connects and the gateway
// has something to make pictures or 3D models with.
const mcpMediaInstructions = "Pictures, 3D models and a model that looks at pictures, on this gateway's own GPUs. " +
"A typical loop: generate_image (style concept) -> generate_mesh from its file_id -> import the GLB -> render -> " +
"review_images against the concept -> fix. Results are files: fetch them from their url with your API key. " +
"AI meshes suit rocks, debris and background props; hard-surface hero assets are usually better built by hand " +
"from the concept. A GPU holds one of these models at a time and switching costs a cold start, so batch calls " +
"of the same kind; gateway_status says which is loaded."
// mediaModels are the gateway's models by what they do, by name, deduplicated
// from the aliases that also resolve to them.
type mediaModels struct{ images, meshes, vision, workflows []string }
func (r *Router) mediaModels() mediaModels {
var out mediaModels
seen := map[string]bool{}
for _, m := range r.modelMap() {
if seen[m.Name] || m.IsGuard() {
continue
}
seen[m.Name] = true
if m.MakesImages() {
out.images = append(out.images, m.Name)
}
if m.MakesMeshes() {
out.meshes = append(out.meshes, m.Name)
}
if m.Capabilities.Vision {
out.vision = append(out.vision, m.Name)
}
if len(m.ComfyInputs()) > 0 {
out.workflows = append(out.workflows, m.Name)
}
}
for _, l := range [][]string{out.images, out.meshes, out.vision, out.workflows} {
slices.Sort(l)
}
return out
}
func (mm mediaModels) any() bool {
return len(mm.images)+len(mm.meshes)+len(mm.vision)+len(mm.workflows) > 0
}
// defaultOf is the model a tool uses when the call names none: the one the
// tool is best known with when the gateway has it, else the first.
func defaultOf(models []string, preferred ...string) string {
for _, p := range preferred {
if slices.Contains(models, p) {
return p
}
}
if len(models) == 0 {
return ""
}
return models[0]
}
func enumOrNone(models []string) map[string]any {
s := map[string]any{"type": "string"}
if len(models) > 0 {
s["enum"] = models
}
return s
}
// mediaTools are the tools for what the gateway's models can do: none for
// what it has no model for.
func (r *Router) mediaTools() []map[string]any {
mm := r.mediaModels()
var tools []map[string]any
fileArg := map[string]any{"type": "string",
"description": "A picture: a /v1/files id (file-…), a data URL, or base64"}
wait := map[string]any{"type": "integer", "minimum": 0, "maximum": int(mcpWaitMax.Seconds()),
"description": "Most seconds to wait for it before answering with the job to ask after (job_status)"}
if len(mm.images) > 0 {
tools = append(tools, map[string]any{
"name": "generate_image",
"description": "Make a picture from a prompt (~15 s warm, longer when the GPU must switch models). " +
"Keeps the PNG as a file and answers its file_id, url and seed, with a small preview. " +
"style: 'concept' (one object, 3/4 view, grey background: good for generate_mesh), 'side' " +
"(exact side view: best meshes for vehicles), 'texture' (seamless tileable surface), 'none' (the prompt as is).",
"inputSchema": map[string]any{
"type": "object",
"required": []string{"prompt"},
"properties": map[string]any{
"prompt": map[string]any{"type": "string"},
"style": map[string]any{"type": "string", "enum": []string{"none", "concept", "side", "texture"}, "default": "none"},
"size": map[string]any{"type": "integer", "description": "Square side in px, a multiple of 16, 256-2048", "default": 1024},
"seed": map[string]any{"type": "integer"},
"negative_prompt": map[string]any{"type": "string"},
"model": enumOrNone(mm.images),
},
},
})
}
if len(mm.meshes) > 0 {
tools = append(tools, map[string]any{
"name": "generate_mesh",
"description": "Make a 3D model (GLB) from a picture of one object on a plain background — the gateway " +
"cuts the background out itself — or from a prompt, for which a concept picture is made first. " +
"Takes 30 s to a few minutes; waits for it, saying how far along it is when the client takes a stream. " +
"Answers the GLB's file_id and url, and its licence. The mesh is about 1 m across, its orientation arbitrary.",
"inputSchema": map[string]any{
"type": "object",
"properties": map[string]any{
"image": fileArg,
"prompt": map[string]any{"type": "string", "description": "Used when no image is given"},
"model": enumOrNone(mm.meshes),
"seed": map[string]any{"type": "integer"},
"resolution": map[string]any{"type": "integer", "description": "The model's own grid, a multiple of 32 (TripoSR: 256 is enough)"},
"texture_resolution": map[string]any{"type": "integer", "minimum": 512, "maximum": 2048,
"description": "For a model that textures its mesh (sf3d): the texture's side in px"},
"inputs": map[string]any{"type": "object",
"description": "Any other input the model's workflow declares (gateway_status lists them): remesh, vertex_count, foreground_ratio, albedo_match…"},
"wait_seconds": wait,
},
},
})
}
if len(mm.vision) > 0 {
tools = append(tools, map[string]any{
"name": "review_images",
"description": "Ask a vision model about 1-4 pictures — a render beside its concept, say. A cheap critic " +
"for render-and-fix loops. json_schema makes it answer strict JSON. A cold start can take minutes " +
"when the GPU was making pictures.",
"inputSchema": map[string]any{
"type": "object",
"required": []string{"images", "question"},
"properties": map[string]any{
"images": map[string]any{"type": "array", "items": fileArg, "minItems": 1, "maxItems": 4},
"question": map[string]any{"type": "string"},
"json_schema": map[string]any{"type": "object", "description": "JSON Schema the answer must follow"},
"model": enumOrNone(mm.vision),
},
},
})
}
if len(mm.workflows) > 0 {
tools = append(tools, map[string]any{
"name": "run_workflow",
"description": "Run a ComfyUI workflow the gateway serves, with the inputs its catalog entry declares " +
"(gateway_status lists them). A file input is a /v1/files id, a data URL or base64. Answers its result file.",
"inputSchema": map[string]any{
"type": "object",
"required": []string{"model", "inputs"},
"properties": map[string]any{
"model": enumOrNone(mm.workflows),
"inputs": map[string]any{"type": "object"},
"wait_seconds": wait,
},
},
})
}
if mm.any() {
tools = append(tools,
map[string]any{
"name": "job_status",
"description": "Where a 3D model or workflow run stands, and its result file when done. wait_seconds waits for it to finish.",
"inputSchema": map[string]any{
"type": "object",
"required": []string{"job_id"},
"properties": map[string]any{"job_id": map[string]any{"type": "string"}, "wait_seconds": wait},
},
},
map[string]any{
"name": "gateway_status",
"description": "The gateway's picture, 3D, vision and workflow models: whether each is loaded on a GPU now, " +
"and a workflow's inputs; and your last few jobs. Use it to batch calls and avoid cold starts.",
"inputSchema": map[string]any{"type": "object", "properties": map[string]any{}},
})
}
return tools
}
// mediaCall is one tool call: who made it, and how to tell them how far it
// has got.
type mediaCall struct {
req *http.Request
progress func(percent float64, message string)
// streamed is a call whose answer is a stream, which may wait longer.
streamed bool
}
func (c mediaCall) wait(asked *int) time.Duration {
d := mcpWaitPlain
if c.streamed {
d = mcpWaitStreamed
}
if asked != nil {
d = min(time.Duration(*asked)*time.Second, mcpWaitMax)
}
return d
}
// isMediaTool is whether a tool is one of these.
func isMediaTool(name string) bool {
switch name {
case "generate_image", "generate_mesh", "review_images", "run_workflow", "job_status", "gateway_status":
return true
}
return false
}
// callMediaTool runs one of the tools. As with the notes, a failure is a
// tool result with isError set, which the model gets to read.
func (r *Router) callMediaTool(c mediaCall, name string, args json.RawMessage) map[string]any {
var res map[string]any
var err error
switch name {
case "generate_image":
res, err = r.toolGenerateImage(c, args)
case "generate_mesh":
res, err = r.toolGenerateMesh(c, args)
case "review_images":
res, err = r.toolReviewImages(c, args)
case "run_workflow":
res, err = r.toolRunWorkflow(c, args)
case "job_status":
res, err = r.toolJobStatus(c, args)
case "gateway_status":
res, err = r.toolGatewayStatus(c)
default:
err = errors.New("unknown tool " + name)
}
if err != nil {
return toolText(err.Error(), true)
}
return res
}
// self makes a request of the gateway's client API as the caller: their key,
// their agent name, and the context the call was made in.
func (r *Router) self(c mediaCall, method, path string, body any) (int, http.Header, []byte, error) {
h := r.selfHandler.Load()
if h == nil {
return 0, nil, nil, errors.New("the gateway's API is not mounted here")
}
var rd *bytes.Reader
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return 0, nil, nil, err
}
rd = bytes.NewReader(b)
} else {
rd = bytes.NewReader(nil)
}
out, err := http.NewRequestWithContext(c.req.Context(), method, path, rd)
if err != nil {
return 0, nil, nil, err
}
for _, k := range []string{"Authorization", "X-Api-Key", agentHeader, "X-Forwarded-For", "X-Real-Ip"} {
if v := c.req.Header.Get(k); v != "" {
out.Header.Set(k, v)
}
}
out.RemoteAddr = c.req.RemoteAddr
if body != nil {
out.Header.Set("Content-Type", "application/json")
}
// A picture or a mesh comes back whole: past the 1 MiB a judge's
// answer is held to.
w := &bufferWriter{header: http.Header{}, max: mcpBody * 2}
(*h).ServeHTTP(w, out)
status := w.status
if status == 0 {
status = http.StatusOK
}
return status, w.header, w.buf.Bytes(), nil
}
// selfJSON is self for an answer in JSON, a failure said as the API said it.
func (r *Router) selfJSON(c mediaCall, method, path string, body, into any) (http.Header, error) {
status, h, b, err := r.self(c, method, path, body)
if err != nil {
return nil, err
}
if status >= 300 {
return h, fmt.Errorf("%s %s: %d %s", method, path, status, errorMessage(b))
}
if into == nil {
return h, nil
}
return h, json.Unmarshal(b, into)
}
// selfJSONWaiting is selfJSON for a model that may be cold: an answer that
// says to come back — a model loading, another holding the GPU, a full
// queue, all with Retry-After — is waited out for as long as the call may
// wait, saying so meanwhile. One that does not say when, a spent budget, is
// not: waiting would not help.
func (r *Router) selfJSONWaiting(c mediaCall, model, path string, body, into any) (http.Header, error) {
deadline := time.Now().Add(c.wait(nil))
for {
status, h, b, err := r.self(c, http.MethodPost, path, body)
if err != nil {
return nil, err
}
retry, _ := strconv.Atoi(h.Get("Retry-After"))
if status < 300 {
return h, json.Unmarshal(b, into)
}
if retry <= 0 || (status != http.StatusTooManyRequests && status < 500) || !time.Now().Before(deadline) {
msg := fmt.Sprintf("POST %s: %d %s", path, status, errorMessage(b))
if retry > 0 {
msg += "; still not ready after waiting — call again, or ask gateway_status which model holds the GPU"
}
return h, errors.New(msg)
}
say := model + " is loading"
if held := h.Get(waitingForHeader); held != "" {
say = model + " is waiting for " + held + " to give up the GPU"
if eta := h.Get(etaHeader); eta != "" {
say += ", about " + eta + " s"
}
}
c.progress(5, say)
select {
case <-c.req.Context().Done():
return nil, c.req.Context().Err()
case <-time.After(min(time.Duration(retry)*time.Second, 10*time.Second, time.Until(deadline)+time.Second)):
}
}
}
// fileURL is where a file is fetched from, with the caller's key, as the
// caller reached the gateway.
func fileURL(req *http.Request, id string) string {
scheme := "http"
if req.TLS != nil {
scheme = "https"
}
if p := req.Header.Get("X-Forwarded-Proto"); p != "" {
scheme = strings.TrimSpace(strings.Split(p, ",")[0])
}
host := req.Host
if h := req.Header.Get("X-Forwarded-Host"); h != "" {
host = strings.TrimSpace(strings.Split(h, ",")[0])
}
return scheme + "://" + host + "/v1/files/" + id + "/content"
}
// keepFile keeps what a tool made as the caller's file, for as long as a
// job's result is kept.
func (r *Router) keepFile(c mediaCall, name string, content []byte) (store.File, error) {
p, ok := auth.FromContext(c.req.Context())
if !ok || r.store == nil {
return store.File{}, errors.New("this gateway keeps no files (it runs without authentication or a store)")
}
if r.noStore(callerSubject(c.req)) {
return store.File{}, errors.New("the gateway's policy for this caller keeps nothing, so no file is kept")
}
_, _, _, days := r.batchCfg()
now := time.Now()
f := store.File{ID: newID("file-"), Owner: ownerOf(p), Purpose: purposeJobOutput, Filename: name,
Bytes: int64(len(content)), CreatedAt: now, ExpiresAt: now.AddDate(0, 0, days)}
return f, r.store.PutFile(c.req.Context(), f, content)
}
// picture reads a picture argument: a file of the caller's by id, a data URL,
// or base64.
func (r *Router) picture(c mediaCall, s string) ([]byte, error) {
s = strings.TrimSpace(s)
switch {
case s == "":
return nil, errors.New("no picture given")
case strings.HasPrefix(s, "file-"):
p, ok := auth.FromContext(c.req.Context())
if !ok || r.store == nil {
return nil, errors.New("this gateway keeps no files: send the picture as base64 or a data URL")
}
f, err := r.store.FileByID(c.req.Context(), s)
if errors.Is(err, store.ErrNotFound) || err == nil && f.Owner != ownerOf(p) {
return nil, fmt.Errorf("no file %s of yours", s)
}
if err != nil {
return nil, err
}
return r.store.FileContent(c.req.Context(), s)
case strings.HasPrefix(s, "data:"):
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return nil, errors.New("a data URL without base64 in it")
}
s = after
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return nil, errors.New("the picture is neither a file id, a data URL nor base64")
}
return b, nil
}
func dataURL(png []byte) string {
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(png)
}
// shrink returns a picture as a PNG whose long side is at most edge.
func shrink(pic []byte, edge int) ([]byte, error) {
src, format, err := image.Decode(bytes.NewReader(pic))
if err != nil {
return nil, fmt.Errorf("not a picture this gateway reads (PNG or JPEG): %w", err)
}
b := src.Bounds()
w, h := b.Dx(), b.Dy()
if max(w, h) <= edge {
if format == "png" {
return pic, nil
}
edge = max(w, h) // only re-encoded
}
scale := float64(edge) / float64(max(w, h))
dw, dh := max(1, int(float64(w)*scale+0.5)), max(1, int(float64(h)*scale+0.5))
dst := image.NewNRGBA(image.Rect(0, 0, dw, dh))
// Each pixel the average of the box of pixels it stands for.
for y := range dh {
y0, y1 := b.Min.Y+y*h/dh, b.Min.Y+max((y+1)*h/dh, y*h/dh+1)
for x := range dw {
x0, x1 := b.Min.X+x*w/dw, b.Min.X+max((x+1)*w/dw, x*w/dw+1)
var sr, sg, sb, sa, n uint64
for yy := y0; yy < y1; yy++ {
for xx := x0; xx < x1; xx++ {
cr, cg, cb, ca := src.At(xx, yy).RGBA()
sr, sg, sb, sa, n = sr+uint64(cr), sg+uint64(cg), sb+uint64(cb), sa+uint64(ca), n+1
}
}
// Premultiplied sums, back to straight colour.
if sa == 0 {
continue
}
dst.SetNRGBA(x, y, color.NRGBA{
R: uint8(sr * 0xff / sa), G: uint8(sg * 0xff / sa), B: uint8(sb * 0xff / sa), A: uint8(sa / n >> 8),
})
}
}
var out bytes.Buffer
if err := png.Encode(&out, dst); err != nil {
return nil, err
}
return out.Bytes(), nil
}
// --- the tools ---
func (r *Router) toolGenerateImage(c mediaCall, raw json.RawMessage) (map[string]any, error) {
var a struct {
Prompt string `json:"prompt"`
Style string `json:"style"`
NegativePrompt string `json:"negative_prompt"`
Model string `json:"model"`
Size int `json:"size"`
Seed *int64 `json:"seed"`
}
if err := decodeArgs(raw, &a); err != nil {
return nil, err
}
res, _, err := r.makeImage(c, a.Prompt, a.Style, a.NegativePrompt, a.Model, a.Size, a.Seed)
if err != nil {
return nil, err
}
return res, nil
}
// makeImage makes a picture and keeps it: the tool's answer, and the PNG.
func (r *Router) makeImage(c mediaCall, prompt, style, negative, model string, size int, seed *int64) (map[string]any, []byte, error) {
if strings.TrimSpace(prompt) == "" {
return nil, nil, errors.New("prompt is required")
}
if style == "" {
style = "none"
}
suffix, ok := imageStyles[style]
if !ok {
return nil, nil, fmt.Errorf("style %q: concept, side, texture or none", style)
}
if model == "" {
model = defaultOf(r.mediaModels().images, "flux-schnell")
}
if size == 0 {
size = 1024
}
body := map[string]any{"model": model, "prompt": prompt + suffix, "n": 1,
"size": fmt.Sprintf("%dx%d", size, size), "response_format": "b64_json"}
if seed != nil {
body["seed"] = *seed
}
if negative != "" {
body["negative_prompt"] = negative
}
c.progress(5, "making the picture with "+model)
var out struct {
Data []struct {
B64 string `json:"b64_json"`
} `json:"data"`
}
h, err := r.selfJSONWaiting(c, model, imagesPath, body, &out)
if err != nil {
return nil, nil, err
}
if len(out.Data) == 0 {
return nil, nil, errors.New(model + " made no picture")
}
pic, err := base64.StdEncoding.DecodeString(out.Data[0].B64)
if err != nil {
return nil, nil, err
}
info := map[string]any{"model": model, "width": size, "height": size}
if s, err := strconv.ParseInt(h.Get("x-llmgw-seed"), 10, 64); err == nil {
info["seed"] = s
}
if m, ok := r.modelMap()[model]; ok && m.License != nil {
info["license"] = m.License
}
content := []map[string]any{}
if f, err := r.keepFile(c, "image-"+strings.TrimPrefix(newID(""), "-")+".png", pic); err == nil {
info["file_id"], info["url"], info["bytes"] = f.ID, fileURL(c.req, f.ID), f.Bytes
} else {
// Nowhere to keep it: the picture itself, whole.
info["note"] = err.Error() + "; the picture is in this answer at full size"
content = append(content, map[string]any{"type": "image", "mimeType": "image/png", "data": out.Data[0].B64})
}
text, _ := json.Marshal(info)
content = append([]map[string]any{{"type": "text", "text": string(text)}}, content...)
if _, kept := info["file_id"]; kept {
if small, err := shrink(pic, previewEdge); err == nil {
content = append(content, map[string]any{"type": "image", "mimeType": "image/png",
"data": base64.StdEncoding.EncodeToString(small)})
}
}
return map[string]any{"content": content, "isError": false}, pic, nil
}
func (r *Router) toolGenerateMesh(c mediaCall, raw json.RawMessage) (map[string]any, error) {
var a struct {
Image, Prompt, Model string
Seed *int64
Resolution int
TextureResolution int `json:"texture_resolution"`
Inputs map[string]any
WaitSeconds *int `json:"wait_seconds"`
}
if err := decodeArgs(raw, &a); err != nil {
return nil, err
}
mm := r.mediaModels()
if a.Model == "" {
a.Model = defaultOf(mm.meshes, "sf3d", "triposr")
}
m, ok := r.modelMap()[a.Model]
if !ok || !m.MakesMeshes() {
return nil, fmt.Errorf("%q makes no 3D models here; these do: %s", a.Model, strings.Join(mm.meshes, ", "))
}
extra := map[string]any{}
var pic []byte
var err error
switch {
case a.Image != "":
if pic, err = r.picture(c, a.Image); err != nil {
return nil, err
}
case a.Prompt != "":
res, made, err := r.makeImage(c, a.Prompt, "concept", "", "", 0, nil)
if err != nil {
return nil, fmt.Errorf("the concept picture: %w", err)
}
pic = made
var info map[string]any
if json.Unmarshal([]byte(res["content"].([]map[string]any)[0]["text"].(string)), &info) == nil {
extra["concept_file_id"], extra["concept_seed"] = info["file_id"], info["seed"]
}
default:
return nil, errors.New("an image, or a prompt to make one from")
}
// A model whose workflow declares its inputs takes them at the
// workflows' path, every one it names; one that does not, the 3D path's
// seed and grid.
path, body := meshesPath, map[string]any{"model": a.Model, "image": dataURL(pic)}
if declared := m.ComfyInputs(); len(declared) > 0 {
names := map[string]string{}
var image string
for _, in := range declared {
names[in.Name] = in.Type
if image == "" && in.Type == "image" {
image = in.Name
}
}
if image == "" {
return nil, fmt.Errorf("%s's workflow takes no picture", a.Model)
}
inputs := map[string]any{image: dataURL(pic)}
// A file of the caller's goes by its id, which the API takes.
if id := strings.TrimSpace(a.Image); strings.HasPrefix(id, "file-") {
inputs[image] = id
}
for k, v := range a.Inputs {
inputs[k] = v
}
// A textured mesh's resolution is its texture's; a grid's, the grid's.
if _, ok := names["resolution"]; ok {
switch {
case a.TextureResolution > 0:
inputs["resolution"] = a.TextureResolution
case a.Resolution > 0:
inputs["resolution"] = a.Resolution
}
}
path, body = workflowsPath, map[string]any{"model": a.Model, "inputs": inputs}
if a.Seed != nil {
body["seed"] = *a.Seed
}
} else {
if a.Seed != nil {
body["seed"] = *a.Seed
}
if a.Resolution > 0 {
body["resolution"] = a.Resolution
}
}
var job struct {
ID string `json:"id"`
}
if _, err := r.selfJSON(c, http.MethodPost, path, body, &job); err != nil {
return nil, err
}
c.progress(10, "queued as "+job.ID)
return r.waitJob(c, job.ID, c.wait(a.WaitSeconds), extra)
}
func (r *Router) toolRunWorkflow(c mediaCall, raw json.RawMessage) (map[string]any, error) {
var a struct {
Model string
Inputs map[string]any
WaitSeconds *int `json:"wait_seconds"`
}
if err := decodeArgs(raw, &a); err != nil {
return nil, err
}
m, ok := r.modelMap()[a.Model]
if !ok || len(m.ComfyInputs()) == 0 {
return nil, fmt.Errorf("%q runs no workflow here; these do: %s", a.Model, strings.Join(r.mediaModels().workflows, ", "))
}
// A file input given as a file id goes as it is: the API takes those.
var job struct {
ID string `json:"id"`
}
if _, err := r.selfJSON(c, http.MethodPost, workflowsPath, map[string]any{"model": a.Model, "inputs": a.Inputs}, &job); err != nil {
return nil, err
}
c.progress(10, "queued as "+job.ID)
return r.waitJob(c, job.ID, c.wait(a.WaitSeconds), nil)
}
func (r *Router) toolJobStatus(c mediaCall, raw json.RawMessage) (map[string]any, error) {
var a struct {
JobID string `json:"job_id"`
WaitSeconds *int `json:"wait_seconds"`
}
if err := decodeArgs(raw, &a); err != nil {
return nil, err
}
if a.JobID == "" {
return nil, errors.New("job_id is required")
}
d := time.Duration(0)
if a.WaitSeconds != nil {
d = c.wait(a.WaitSeconds)
}
return r.waitJob(c, a.JobID, d, nil)
}
// jobView is the part of a job a tool says.
type jobView struct {
ID string `json:"id"`
Model string `json:"model"`
Status string `json:"status"`
Progress json.RawMessage `json:"progress"`
OutputFileID *string `json:"output_file_id"`
ContentType *string `json:"content_type"`
Error *struct {
Message string `json:"message"`
} `json:"error"`
License json.RawMessage `json:"license"`
}
// waitJob asks after a job until it is done or d has passed, saying how far
// along it is, and answers where it stands: its file when done.
func (r *Router) waitJob(c mediaCall, id string, d time.Duration, extra map[string]any) (map[string]any, error) {
deadline := time.Now().Add(d)
var j jobView
for {
if _, err := r.selfJSON(c, http.MethodGet, workflowsPath+"/"+id, nil, &j); err != nil {
return nil, err
}
if store.JobDone(j.Status) || !time.Now().Before(deadline) {
break
}
pct, msg := jobProgress(j)
c.progress(pct, msg)
select {
case <-c.req.Context().Done():
return nil, c.req.Context().Err()
case <-time.After(mcpPoll):
}
}
info := map[string]any{"job_id": j.ID, "model": j.Model, "status": j.Status}
for k, v := range extra {
if v != nil {
info[k] = v
}
}
if len(j.License) > 0 && string(j.License) != "null" {
info["license"] = j.License
}
switch {
case j.Status == store.JobCompleted && j.OutputFileID != nil:
info["file_id"], info["url"] = *j.OutputFileID, fileURL(c.req, *j.OutputFileID)
if j.ContentType != nil {
info["content_type"] = *j.ContentType
}
if r.store != nil {
if f, err := r.store.FileByID(c.req.Context(), *j.OutputFileID); err == nil {
info["bytes"] = f.Bytes
}
}
case store.JobDone(j.Status):
if j.Error != nil {
info["error"] = j.Error.Message
}
b, _ := json.Marshal(info)
return toolText(string(b), true), nil
default:
info["progress"] = j.Progress
info["next"] = "not done yet: call job_status with this job_id (and wait_seconds) to wait for it"
}
b, _ := json.Marshal(info)
return toolText(string(b), false), nil
}
// jobProgress reads a job's progress as a percent and a phrase.
func jobProgress(j jobView) (float64, string) {
var p struct {
State string `json:"state"`
Ahead int `json:"ahead"`
Percent float64 `json:"percent"`
Node string `json:"node_class"`
HeldBy string `json:"held_by"`
}
_ = json.Unmarshal(j.Progress, &p)
switch {
case p.State == "queued":
return 10, fmt.Sprintf("%s: queued, %d ahead", j.ID, p.Ahead)
case p.State == "waiting_gpu":
return 10, j.ID + ": waiting for the GPU, held by " + p.HeldBy
case p.State == "running":
return 10 + 0.9*p.Percent, fmt.Sprintf("%s: %s %.0f%%", j.ID, p.Node, p.Percent)
}
return 10, j.ID + ": " + j.Status
}
func (r *Router) toolReviewImages(c mediaCall, raw json.RawMessage) (map[string]any, error) {
var a struct {
Images []string
Question string
JSONSchema map[string]any `json:"json_schema"`
Model string
}
if err := decodeArgs(raw, &a); err != nil {
return nil, err
}
if len(a.Images) == 0 || len(a.Images) > 4 {
return nil, errors.New("one to four images")
}
if strings.TrimSpace(a.Question) == "" {
return nil, errors.New("question is required")
}
if a.Model == "" {
a.Model = defaultOf(r.mediaModels().vision, "vlm")
}
if a.Model == "" {
return nil, errors.New("no model here looks at pictures")
}
parts := []map[string]any{{"type": "text", "text": a.Question}}
for i, s := range a.Images {
pic, err := r.picture(c, s)
if err != nil {
return nil, fmt.Errorf("image %d: %w", i+1, err)
}
small, err := shrink(pic, lookEdge)
if err != nil {
return nil, fmt.Errorf("image %d: %w", i+1, err)
}
parts = append(parts, map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURL(small)}})
}
body := map[string]any{"model": a.Model, "messages": []map[string]any{{"role": "user", "content": parts}}}
if a.JSONSchema != nil {
body["response_format"] = map[string]any{"type": "json_schema",
"json_schema": map[string]any{"name": "answer", "schema": a.JSONSchema, "strict": true}}
}
c.progress(10, "asking "+a.Model)
var out struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
}
if _, err := r.selfJSONWaiting(c, a.Model, "/v1/chat/completions", body, &out); err != nil {
return nil, err
}
if len(out.Choices) == 0 {
return nil, errors.New(a.Model + " gave no answer")
}
return toolText(out.Choices[0].Message.Content, false), nil
}
func (r *Router) toolGatewayStatus(c mediaCall) (map[string]any, error) {
var models struct {
Data []map[string]any `json:"data"`
}
if _, err := r.selfJSON(c, http.MethodGet, "/v1/models", nil, &models); err != nil {
return nil, err
}
var listed []map[string]any
for _, m := range models.Data {
caps, _ := m["capabilities"].(map[string]any)
if caps["images"] != true && caps["meshes"] != true && caps["vision"] != true && m["workflow"] == nil {
continue
}
e := map[string]any{"id": m["id"], "loaded": m["loaded"], "capabilities": caps}
if w := m["workflow"]; w != nil {
e["workflow"] = w
}
if l := m["license"]; l != nil {
e["license"] = l
}
listed = append(listed, e)
}
out := map[string]any{"models": listed}
var jobs struct {
Data []jobView `json:"data"`
}
if _, err := r.selfJSON(c, http.MethodGet, workflowsPath+"?limit=5", nil, &jobs); err == nil {
recent := make([]map[string]any, 0, len(jobs.Data))
for _, j := range jobs.Data {
recent = append(recent, map[string]any{"id": j.ID, "model": j.Model, "status": j.Status})
}
out["recent_jobs"] = recent
}
return toolJSON(out), nil
}
// decodeArgs reads a tool's arguments, by their JSON names: a field's own
// name in lower case unless its tag says.
func decodeArgs(raw json.RawMessage, into any) error {
if len(raw) == 0 || string(raw) == "null" {
raw = []byte("{}")
}
if err := json.Unmarshal(raw, into); err != nil {
return fmt.Errorf("the arguments: %w", err)
}
return nil
}
// streamWriter answers a tool call as server-sent events: progress
// notifications while it runs, then the answer.
type streamWriter struct {
mu sync.Mutex
w http.ResponseWriter
token json.RawMessage
}
func (s *streamWriter) write(text string) {
s.mu.Lock()
defer s.mu.Unlock()
_, _ = fmt.Fprint(s.w, text)
if f, ok := s.w.(http.Flusher); ok {
f.Flush()
}
}
func (s *streamWriter) event(v any) {
b, _ := json.Marshal(v)
s.write("event: message\ndata: " + string(b) + "\n\n")
}
// keepAlive writes a comment every while until stop is closed: a picture
// whose model starts cold is minutes of nothing said, which a proxy reads as
// a connection gone idle.
func (s *streamWriter) keepAlive(stop <-chan struct{}, every time.Duration) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-stop:
return
case <-t.C:
s.write(": still working\n\n")
}
}
}
func (s *streamWriter) progress(percent float64, message string) {
if len(s.token) == 0 {
return
}
s.event(map[string]any{"jsonrpc": "2.0", "method": "notifications/progress", "params": map[string]any{
"progressToken": s.token, "progress": percent, "total": 100, "message": message}})
}
// acceptsStream is a client that takes a POST's answer as a stream, as the
// streamable HTTP transport lets it.
func acceptsStream(req *http.Request) bool {
return strings.Contains(req.Header.Get("Accept"), "text/event-stream")
}