package router import ( "bytes" "encoding/base64" "encoding/json" "errors" "fmt" "image" "image/color" _ "image/jpeg" // a picture sent to look at may be a JPEG "image/png" "net/http" "slices" "strconv" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // --- MCP: pictures, 3D models, looking at pictures, workflows --- // // The gateway's own models offered as tools, so an agent — Claude Code // driving Blender, say — makes a concept, a mesh from it and has a render // looked at without a tool server of its own. Each tool makes its request // of the gateway as the caller would, under the caller's key: the same // allowlist, limits, licences left out, guardrails and accounting. What it // makes is kept as a file of the caller's, said by id and by the address // it is fetched from with the same key, since a tool server over HTTP // cannot write to the caller's disk. const ( // mcpBody bounds a tool call: up to four pictures to look at, as base64. mcpBody = 64 << 20 // previewEdge is the long side of the picture shown to the agent beside // the file: enough to see it, not a megapixel of tokens. previewEdge = 512 // lookEdge is the long side of a picture sent to a vision model: // 1280x720 is about 940 tokens. lookEdge = 1280 // mcpWaitStreamed is how long a call answered as a stream waits for its // job, saying how far along it is meanwhile; mcpWaitPlain how long one // answered in one piece does, under the minute a proxy allows. mcpWaitStreamed = 10 * time.Minute mcpWaitPlain = 50 * time.Second mcpWaitMax = 15 * time.Minute mcpPoll = 2 * time.Second ) // The prompt endings of generate_image's styles: what a picture a mesh is // made from wants, and a surface that tiles. var imageStyles = map[string]string{ "none": "", "concept": ", single object centered, orthographic three-quarter view, flat even studio lighting, no shadows, " + "no background elements, plain flat grey background, clean silhouette, game asset concept, high detail, no text, no watermark", "side": ", single object, exact side view (profile), orthographic, flat even studio lighting, no shadows, " + "plain flat grey background, clean silhouette, game asset concept, high detail, no text, no watermark", "texture": ", seamless tileable texture, top-down orthographic, flat even lighting, no shadows, no perspective, no text", } // mcpMediaInstructions is said to an agent when it connects and the gateway // has something to make pictures or 3D models with. const mcpMediaInstructions = "Pictures, 3D models and a model that looks at pictures, on this gateway's own GPUs. " + "A typical loop: generate_image (style concept) -> generate_mesh from its file_id -> import the GLB -> render -> " + "review_images against the concept -> fix. Results are files: fetch them from their url with your API key. " + "AI meshes suit rocks, debris and background props; hard-surface hero assets are usually better built by hand " + "from the concept. A GPU holds one of these models at a time and switching costs a cold start, so batch calls " + "of the same kind; gateway_status says which is loaded." // mediaModels are the gateway's models by what they do, by name, deduplicated // from the aliases that also resolve to them. type mediaModels struct{ images, meshes, vision, workflows []string } func (r *Router) mediaModels() mediaModels { var out mediaModels seen := map[string]bool{} for _, m := range r.modelMap() { if seen[m.Name] || m.IsGuard() { continue } seen[m.Name] = true if m.MakesImages() { out.images = append(out.images, m.Name) } if m.MakesMeshes() { out.meshes = append(out.meshes, m.Name) } if m.Capabilities.Vision { out.vision = append(out.vision, m.Name) } if len(m.ComfyInputs()) > 0 { out.workflows = append(out.workflows, m.Name) } } for _, l := range [][]string{out.images, out.meshes, out.vision, out.workflows} { slices.Sort(l) } return out } func (mm mediaModels) any() bool { return len(mm.images)+len(mm.meshes)+len(mm.vision)+len(mm.workflows) > 0 } // defaultOf is the model a tool uses when the call names none: the one the // tool is best known with when the gateway has it, else the first. func defaultOf(models []string, preferred ...string) string { for _, p := range preferred { if slices.Contains(models, p) { return p } } if len(models) == 0 { return "" } return models[0] } func enumOrNone(models []string) map[string]any { s := map[string]any{"type": "string"} if len(models) > 0 { s["enum"] = models } return s } // mediaTools are the tools for what the gateway's models can do: none for // what it has no model for. func (r *Router) mediaTools() []map[string]any { mm := r.mediaModels() var tools []map[string]any fileArg := map[string]any{"type": "string", "description": "A picture: a /v1/files id (file-…), a data URL, or base64"} wait := map[string]any{"type": "integer", "minimum": 0, "maximum": int(mcpWaitMax.Seconds()), "description": "Most seconds to wait for it before answering with the job to ask after (job_status)"} if len(mm.images) > 0 { tools = append(tools, map[string]any{ "name": "generate_image", "description": "Make a picture from a prompt (~15 s warm, longer when the GPU must switch models). " + "Keeps the PNG as a file and answers its file_id, url and seed, with a small preview. " + "style: 'concept' (one object, 3/4 view, grey background: good for generate_mesh), 'side' " + "(exact side view: best meshes for vehicles), 'texture' (seamless tileable surface), 'none' (the prompt as is).", "inputSchema": map[string]any{ "type": "object", "required": []string{"prompt"}, "properties": map[string]any{ "prompt": map[string]any{"type": "string"}, "style": map[string]any{"type": "string", "enum": []string{"none", "concept", "side", "texture"}, "default": "none"}, "size": map[string]any{"type": "integer", "description": "Square side in px, a multiple of 16, 256-2048", "default": 1024}, "seed": map[string]any{"type": "integer"}, "negative_prompt": map[string]any{"type": "string"}, "model": enumOrNone(mm.images), }, }, }) } if len(mm.meshes) > 0 { tools = append(tools, map[string]any{ "name": "generate_mesh", "description": "Make a 3D model (GLB) from a picture of one object on a plain background — the gateway " + "cuts the background out itself — or from a prompt, for which a concept picture is made first. " + "Takes 30 s to a few minutes; waits for it, saying how far along it is when the client takes a stream. " + "Answers the GLB's file_id and url, and its licence. The mesh is about 1 m across, its orientation arbitrary.", "inputSchema": map[string]any{ "type": "object", "properties": map[string]any{ "image": fileArg, "prompt": map[string]any{"type": "string", "description": "Used when no image is given"}, "model": enumOrNone(mm.meshes), "seed": map[string]any{"type": "integer"}, "resolution": map[string]any{"type": "integer", "description": "The model's own grid, a multiple of 32 (TripoSR: 256 is enough)"}, "texture_resolution": map[string]any{"type": "integer", "minimum": 512, "maximum": 2048, "description": "For a model that textures its mesh (sf3d): the texture's side in px"}, "inputs": map[string]any{"type": "object", "description": "Any other input the model's workflow declares (gateway_status lists them): remesh, vertex_count, foreground_ratio, albedo_match…"}, "wait_seconds": wait, }, }, }) } if len(mm.vision) > 0 { tools = append(tools, map[string]any{ "name": "review_images", "description": "Ask a vision model about 1-4 pictures — a render beside its concept, say. A cheap critic " + "for render-and-fix loops. json_schema makes it answer strict JSON. A cold start can take minutes " + "when the GPU was making pictures.", "inputSchema": map[string]any{ "type": "object", "required": []string{"images", "question"}, "properties": map[string]any{ "images": map[string]any{"type": "array", "items": fileArg, "minItems": 1, "maxItems": 4}, "question": map[string]any{"type": "string"}, "json_schema": map[string]any{"type": "object", "description": "JSON Schema the answer must follow"}, "model": enumOrNone(mm.vision), }, }, }) } if len(mm.workflows) > 0 { tools = append(tools, map[string]any{ "name": "run_workflow", "description": "Run a ComfyUI workflow the gateway serves, with the inputs its catalog entry declares " + "(gateway_status lists them). A file input is a /v1/files id, a data URL or base64. Answers its result file.", "inputSchema": map[string]any{ "type": "object", "required": []string{"model", "inputs"}, "properties": map[string]any{ "model": enumOrNone(mm.workflows), "inputs": map[string]any{"type": "object"}, "wait_seconds": wait, }, }, }) } if mm.any() { tools = append(tools, map[string]any{ "name": "job_status", "description": "Where a 3D model or workflow run stands, and its result file when done. wait_seconds waits for it to finish.", "inputSchema": map[string]any{ "type": "object", "required": []string{"job_id"}, "properties": map[string]any{"job_id": map[string]any{"type": "string"}, "wait_seconds": wait}, }, }, map[string]any{ "name": "gateway_status", "description": "The gateway's picture, 3D, vision and workflow models: whether each is loaded on a GPU now, " + "and a workflow's inputs; and your last few jobs. Use it to batch calls and avoid cold starts.", "inputSchema": map[string]any{"type": "object", "properties": map[string]any{}}, }) } return tools } // mediaCall is one tool call: who made it, and how to tell them how far it // has got. type mediaCall struct { req *http.Request progress func(percent float64, message string) // streamed is a call whose answer is a stream, which may wait longer. streamed bool } func (c mediaCall) wait(asked *int) time.Duration { d := mcpWaitPlain if c.streamed { d = mcpWaitStreamed } if asked != nil { d = min(time.Duration(*asked)*time.Second, mcpWaitMax) } return d } // isMediaTool is whether a tool is one of these. func isMediaTool(name string) bool { switch name { case "generate_image", "generate_mesh", "review_images", "run_workflow", "job_status", "gateway_status": return true } return false } // callMediaTool runs one of the tools. As with the notes, a failure is a // tool result with isError set, which the model gets to read. func (r *Router) callMediaTool(c mediaCall, name string, args json.RawMessage) map[string]any { var res map[string]any var err error switch name { case "generate_image": res, err = r.toolGenerateImage(c, args) case "generate_mesh": res, err = r.toolGenerateMesh(c, args) case "review_images": res, err = r.toolReviewImages(c, args) case "run_workflow": res, err = r.toolRunWorkflow(c, args) case "job_status": res, err = r.toolJobStatus(c, args) case "gateway_status": res, err = r.toolGatewayStatus(c) default: err = errors.New("unknown tool " + name) } if err != nil { return toolText(err.Error(), true) } return res } // self makes a request of the gateway's client API as the caller: their key, // their agent name, and the context the call was made in. func (r *Router) self(c mediaCall, method, path string, body any) (int, http.Header, []byte, error) { h := r.selfHandler.Load() if h == nil { return 0, nil, nil, errors.New("the gateway's API is not mounted here") } var rd *bytes.Reader if body != nil { b, err := json.Marshal(body) if err != nil { return 0, nil, nil, err } rd = bytes.NewReader(b) } else { rd = bytes.NewReader(nil) } out, err := http.NewRequestWithContext(c.req.Context(), method, path, rd) if err != nil { return 0, nil, nil, err } for _, k := range []string{"Authorization", "X-Api-Key", agentHeader, "X-Forwarded-For", "X-Real-Ip"} { if v := c.req.Header.Get(k); v != "" { out.Header.Set(k, v) } } out.RemoteAddr = c.req.RemoteAddr if body != nil { out.Header.Set("Content-Type", "application/json") } // A picture or a mesh comes back whole: past the 1 MiB a judge's // answer is held to. w := &bufferWriter{header: http.Header{}, max: mcpBody * 2} (*h).ServeHTTP(w, out) status := w.status if status == 0 { status = http.StatusOK } return status, w.header, w.buf.Bytes(), nil } // selfJSON is self for an answer in JSON, a failure said as the API said it. func (r *Router) selfJSON(c mediaCall, method, path string, body, into any) (http.Header, error) { status, h, b, err := r.self(c, method, path, body) if err != nil { return nil, err } if status >= 300 { return h, fmt.Errorf("%s %s: %d %s", method, path, status, errorMessage(b)) } if into == nil { return h, nil } return h, json.Unmarshal(b, into) } // selfJSONWaiting is selfJSON for a model that may be cold: an answer that // says to come back — a model loading, another holding the GPU, a full // queue, all with Retry-After — is waited out for as long as the call may // wait, saying so meanwhile. One that does not say when, a spent budget, is // not: waiting would not help. func (r *Router) selfJSONWaiting(c mediaCall, model, path string, body, into any) (http.Header, error) { deadline := time.Now().Add(c.wait(nil)) for { status, h, b, err := r.self(c, http.MethodPost, path, body) if err != nil { return nil, err } retry, _ := strconv.Atoi(h.Get("Retry-After")) if status < 300 { return h, json.Unmarshal(b, into) } if retry <= 0 || (status != http.StatusTooManyRequests && status < 500) || !time.Now().Before(deadline) { msg := fmt.Sprintf("POST %s: %d %s", path, status, errorMessage(b)) if retry > 0 { msg += "; still not ready after waiting — call again, or ask gateway_status which model holds the GPU" } return h, errors.New(msg) } say := model + " is loading" if held := h.Get(waitingForHeader); held != "" { say = model + " is waiting for " + held + " to give up the GPU" if eta := h.Get(etaHeader); eta != "" { say += ", about " + eta + " s" } } c.progress(5, say) select { case <-c.req.Context().Done(): return nil, c.req.Context().Err() case <-time.After(min(time.Duration(retry)*time.Second, 10*time.Second, time.Until(deadline)+time.Second)): } } } // fileURL is where a file is fetched from, with the caller's key, as the // caller reached the gateway. func fileURL(req *http.Request, id string) string { scheme := "http" if req.TLS != nil { scheme = "https" } if p := req.Header.Get("X-Forwarded-Proto"); p != "" { scheme = strings.TrimSpace(strings.Split(p, ",")[0]) } host := req.Host if h := req.Header.Get("X-Forwarded-Host"); h != "" { host = strings.TrimSpace(strings.Split(h, ",")[0]) } return scheme + "://" + host + "/v1/files/" + id + "/content" } // keepFile keeps what a tool made as the caller's file, for as long as a // job's result is kept. func (r *Router) keepFile(c mediaCall, name string, content []byte) (store.File, error) { p, ok := auth.FromContext(c.req.Context()) if !ok || r.store == nil { return store.File{}, errors.New("this gateway keeps no files (it runs without authentication or a store)") } if r.noStore(callerSubject(c.req)) { return store.File{}, errors.New("the gateway's policy for this caller keeps nothing, so no file is kept") } _, _, _, days := r.batchCfg() now := time.Now() f := store.File{ID: newID("file-"), Owner: ownerOf(p), Purpose: purposeJobOutput, Filename: name, Bytes: int64(len(content)), CreatedAt: now, ExpiresAt: now.AddDate(0, 0, days)} return f, r.store.PutFile(c.req.Context(), f, content) } // picture reads a picture argument: a file of the caller's by id, a data URL, // or base64. func (r *Router) picture(c mediaCall, s string) ([]byte, error) { s = strings.TrimSpace(s) switch { case s == "": return nil, errors.New("no picture given") case strings.HasPrefix(s, "file-"): p, ok := auth.FromContext(c.req.Context()) if !ok || r.store == nil { return nil, errors.New("this gateway keeps no files: send the picture as base64 or a data URL") } f, err := r.store.FileByID(c.req.Context(), s) if errors.Is(err, store.ErrNotFound) || err == nil && f.Owner != ownerOf(p) { return nil, fmt.Errorf("no file %s of yours", s) } if err != nil { return nil, err } return r.store.FileContent(c.req.Context(), s) case strings.HasPrefix(s, "data:"): _, after, ok := strings.Cut(s, ";base64,") if !ok { return nil, errors.New("a data URL without base64 in it") } s = after } b, err := base64.StdEncoding.DecodeString(s) if err != nil { return nil, errors.New("the picture is neither a file id, a data URL nor base64") } return b, nil } func dataURL(png []byte) string { return "data:image/png;base64," + base64.StdEncoding.EncodeToString(png) } // shrink returns a picture as a PNG whose long side is at most edge. func shrink(pic []byte, edge int) ([]byte, error) { src, format, err := image.Decode(bytes.NewReader(pic)) if err != nil { return nil, fmt.Errorf("not a picture this gateway reads (PNG or JPEG): %w", err) } b := src.Bounds() w, h := b.Dx(), b.Dy() if max(w, h) <= edge { if format == "png" { return pic, nil } edge = max(w, h) // only re-encoded } scale := float64(edge) / float64(max(w, h)) dw, dh := max(1, int(float64(w)*scale+0.5)), max(1, int(float64(h)*scale+0.5)) dst := image.NewNRGBA(image.Rect(0, 0, dw, dh)) // Each pixel the average of the box of pixels it stands for. for y := range dh { y0, y1 := b.Min.Y+y*h/dh, b.Min.Y+max((y+1)*h/dh, y*h/dh+1) for x := range dw { x0, x1 := b.Min.X+x*w/dw, b.Min.X+max((x+1)*w/dw, x*w/dw+1) var sr, sg, sb, sa, n uint64 for yy := y0; yy < y1; yy++ { for xx := x0; xx < x1; xx++ { cr, cg, cb, ca := src.At(xx, yy).RGBA() sr, sg, sb, sa, n = sr+uint64(cr), sg+uint64(cg), sb+uint64(cb), sa+uint64(ca), n+1 } } // Premultiplied sums, back to straight colour. if sa == 0 { continue } dst.SetNRGBA(x, y, color.NRGBA{ R: uint8(sr * 0xff / sa), G: uint8(sg * 0xff / sa), B: uint8(sb * 0xff / sa), A: uint8(sa / n >> 8), }) } } var out bytes.Buffer if err := png.Encode(&out, dst); err != nil { return nil, err } return out.Bytes(), nil } // --- the tools --- func (r *Router) toolGenerateImage(c mediaCall, raw json.RawMessage) (map[string]any, error) { var a struct { Prompt string `json:"prompt"` Style string `json:"style"` NegativePrompt string `json:"negative_prompt"` Model string `json:"model"` Size int `json:"size"` Seed *int64 `json:"seed"` } if err := decodeArgs(raw, &a); err != nil { return nil, err } res, _, err := r.makeImage(c, a.Prompt, a.Style, a.NegativePrompt, a.Model, a.Size, a.Seed) if err != nil { return nil, err } return res, nil } // makeImage makes a picture and keeps it: the tool's answer, and the PNG. func (r *Router) makeImage(c mediaCall, prompt, style, negative, model string, size int, seed *int64) (map[string]any, []byte, error) { if strings.TrimSpace(prompt) == "" { return nil, nil, errors.New("prompt is required") } if style == "" { style = "none" } suffix, ok := imageStyles[style] if !ok { return nil, nil, fmt.Errorf("style %q: concept, side, texture or none", style) } if model == "" { model = defaultOf(r.mediaModels().images, "flux-schnell") } if size == 0 { size = 1024 } body := map[string]any{"model": model, "prompt": prompt + suffix, "n": 1, "size": fmt.Sprintf("%dx%d", size, size), "response_format": "b64_json"} if seed != nil { body["seed"] = *seed } if negative != "" { body["negative_prompt"] = negative } c.progress(5, "making the picture with "+model) var out struct { Data []struct { B64 string `json:"b64_json"` } `json:"data"` } h, err := r.selfJSONWaiting(c, model, imagesPath, body, &out) if err != nil { return nil, nil, err } if len(out.Data) == 0 { return nil, nil, errors.New(model + " made no picture") } pic, err := base64.StdEncoding.DecodeString(out.Data[0].B64) if err != nil { return nil, nil, err } info := map[string]any{"model": model, "width": size, "height": size} if s, err := strconv.ParseInt(h.Get("x-llmgw-seed"), 10, 64); err == nil { info["seed"] = s } if m, ok := r.modelMap()[model]; ok && m.License != nil { info["license"] = m.License } content := []map[string]any{} if f, err := r.keepFile(c, "image-"+strings.TrimPrefix(newID(""), "-")+".png", pic); err == nil { info["file_id"], info["url"], info["bytes"] = f.ID, fileURL(c.req, f.ID), f.Bytes } else { // Nowhere to keep it: the picture itself, whole. info["note"] = err.Error() + "; the picture is in this answer at full size" content = append(content, map[string]any{"type": "image", "mimeType": "image/png", "data": out.Data[0].B64}) } text, _ := json.Marshal(info) content = append([]map[string]any{{"type": "text", "text": string(text)}}, content...) if _, kept := info["file_id"]; kept { if small, err := shrink(pic, previewEdge); err == nil { content = append(content, map[string]any{"type": "image", "mimeType": "image/png", "data": base64.StdEncoding.EncodeToString(small)}) } } return map[string]any{"content": content, "isError": false}, pic, nil } func (r *Router) toolGenerateMesh(c mediaCall, raw json.RawMessage) (map[string]any, error) { var a struct { Image, Prompt, Model string Seed *int64 Resolution int TextureResolution int `json:"texture_resolution"` Inputs map[string]any WaitSeconds *int `json:"wait_seconds"` } if err := decodeArgs(raw, &a); err != nil { return nil, err } mm := r.mediaModels() if a.Model == "" { a.Model = defaultOf(mm.meshes, "sf3d", "triposr") } m, ok := r.modelMap()[a.Model] if !ok || !m.MakesMeshes() { return nil, fmt.Errorf("%q makes no 3D models here; these do: %s", a.Model, strings.Join(mm.meshes, ", ")) } extra := map[string]any{} var pic []byte var err error switch { case a.Image != "": if pic, err = r.picture(c, a.Image); err != nil { return nil, err } case a.Prompt != "": res, made, err := r.makeImage(c, a.Prompt, "concept", "", "", 0, nil) if err != nil { return nil, fmt.Errorf("the concept picture: %w", err) } pic = made var info map[string]any if json.Unmarshal([]byte(res["content"].([]map[string]any)[0]["text"].(string)), &info) == nil { extra["concept_file_id"], extra["concept_seed"] = info["file_id"], info["seed"] } default: return nil, errors.New("an image, or a prompt to make one from") } // A model whose workflow declares its inputs takes them at the // workflows' path, every one it names; one that does not, the 3D path's // seed and grid. path, body := meshesPath, map[string]any{"model": a.Model, "image": dataURL(pic)} if declared := m.ComfyInputs(); len(declared) > 0 { names := map[string]string{} var image string for _, in := range declared { names[in.Name] = in.Type if image == "" && in.Type == "image" { image = in.Name } } if image == "" { return nil, fmt.Errorf("%s's workflow takes no picture", a.Model) } inputs := map[string]any{image: dataURL(pic)} // A file of the caller's goes by its id, which the API takes. if id := strings.TrimSpace(a.Image); strings.HasPrefix(id, "file-") { inputs[image] = id } for k, v := range a.Inputs { inputs[k] = v } // A textured mesh's resolution is its texture's; a grid's, the grid's. if _, ok := names["resolution"]; ok { switch { case a.TextureResolution > 0: inputs["resolution"] = a.TextureResolution case a.Resolution > 0: inputs["resolution"] = a.Resolution } } path, body = workflowsPath, map[string]any{"model": a.Model, "inputs": inputs} if a.Seed != nil { body["seed"] = *a.Seed } } else { if a.Seed != nil { body["seed"] = *a.Seed } if a.Resolution > 0 { body["resolution"] = a.Resolution } } var job struct { ID string `json:"id"` } if _, err := r.selfJSON(c, http.MethodPost, path, body, &job); err != nil { return nil, err } c.progress(10, "queued as "+job.ID) return r.waitJob(c, job.ID, c.wait(a.WaitSeconds), extra) } func (r *Router) toolRunWorkflow(c mediaCall, raw json.RawMessage) (map[string]any, error) { var a struct { Model string Inputs map[string]any WaitSeconds *int `json:"wait_seconds"` } if err := decodeArgs(raw, &a); err != nil { return nil, err } m, ok := r.modelMap()[a.Model] if !ok || len(m.ComfyInputs()) == 0 { return nil, fmt.Errorf("%q runs no workflow here; these do: %s", a.Model, strings.Join(r.mediaModels().workflows, ", ")) } // A file input given as a file id goes as it is: the API takes those. var job struct { ID string `json:"id"` } if _, err := r.selfJSON(c, http.MethodPost, workflowsPath, map[string]any{"model": a.Model, "inputs": a.Inputs}, &job); err != nil { return nil, err } c.progress(10, "queued as "+job.ID) return r.waitJob(c, job.ID, c.wait(a.WaitSeconds), nil) } func (r *Router) toolJobStatus(c mediaCall, raw json.RawMessage) (map[string]any, error) { var a struct { JobID string `json:"job_id"` WaitSeconds *int `json:"wait_seconds"` } if err := decodeArgs(raw, &a); err != nil { return nil, err } if a.JobID == "" { return nil, errors.New("job_id is required") } d := time.Duration(0) if a.WaitSeconds != nil { d = c.wait(a.WaitSeconds) } return r.waitJob(c, a.JobID, d, nil) } // jobView is the part of a job a tool says. type jobView struct { ID string `json:"id"` Model string `json:"model"` Status string `json:"status"` Progress json.RawMessage `json:"progress"` OutputFileID *string `json:"output_file_id"` ContentType *string `json:"content_type"` Error *struct { Message string `json:"message"` } `json:"error"` License json.RawMessage `json:"license"` } // waitJob asks after a job until it is done or d has passed, saying how far // along it is, and answers where it stands: its file when done. func (r *Router) waitJob(c mediaCall, id string, d time.Duration, extra map[string]any) (map[string]any, error) { deadline := time.Now().Add(d) var j jobView for { if _, err := r.selfJSON(c, http.MethodGet, workflowsPath+"/"+id, nil, &j); err != nil { return nil, err } if store.JobDone(j.Status) || !time.Now().Before(deadline) { break } pct, msg := jobProgress(j) c.progress(pct, msg) select { case <-c.req.Context().Done(): return nil, c.req.Context().Err() case <-time.After(mcpPoll): } } info := map[string]any{"job_id": j.ID, "model": j.Model, "status": j.Status} for k, v := range extra { if v != nil { info[k] = v } } if len(j.License) > 0 && string(j.License) != "null" { info["license"] = j.License } switch { case j.Status == store.JobCompleted && j.OutputFileID != nil: info["file_id"], info["url"] = *j.OutputFileID, fileURL(c.req, *j.OutputFileID) if j.ContentType != nil { info["content_type"] = *j.ContentType } if r.store != nil { if f, err := r.store.FileByID(c.req.Context(), *j.OutputFileID); err == nil { info["bytes"] = f.Bytes } } case store.JobDone(j.Status): if j.Error != nil { info["error"] = j.Error.Message } b, _ := json.Marshal(info) return toolText(string(b), true), nil default: info["progress"] = j.Progress info["next"] = "not done yet: call job_status with this job_id (and wait_seconds) to wait for it" } b, _ := json.Marshal(info) return toolText(string(b), false), nil } // jobProgress reads a job's progress as a percent and a phrase. func jobProgress(j jobView) (float64, string) { var p struct { State string `json:"state"` Ahead int `json:"ahead"` Percent float64 `json:"percent"` Node string `json:"node_class"` HeldBy string `json:"held_by"` } _ = json.Unmarshal(j.Progress, &p) switch { case p.State == "queued": return 10, fmt.Sprintf("%s: queued, %d ahead", j.ID, p.Ahead) case p.State == "waiting_gpu": return 10, j.ID + ": waiting for the GPU, held by " + p.HeldBy case p.State == "running": return 10 + 0.9*p.Percent, fmt.Sprintf("%s: %s %.0f%%", j.ID, p.Node, p.Percent) } return 10, j.ID + ": " + j.Status } func (r *Router) toolReviewImages(c mediaCall, raw json.RawMessage) (map[string]any, error) { var a struct { Images []string Question string JSONSchema map[string]any `json:"json_schema"` Model string } if err := decodeArgs(raw, &a); err != nil { return nil, err } if len(a.Images) == 0 || len(a.Images) > 4 { return nil, errors.New("one to four images") } if strings.TrimSpace(a.Question) == "" { return nil, errors.New("question is required") } if a.Model == "" { a.Model = defaultOf(r.mediaModels().vision, "vlm") } if a.Model == "" { return nil, errors.New("no model here looks at pictures") } parts := []map[string]any{{"type": "text", "text": a.Question}} for i, s := range a.Images { pic, err := r.picture(c, s) if err != nil { return nil, fmt.Errorf("image %d: %w", i+1, err) } small, err := shrink(pic, lookEdge) if err != nil { return nil, fmt.Errorf("image %d: %w", i+1, err) } parts = append(parts, map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURL(small)}}) } body := map[string]any{"model": a.Model, "messages": []map[string]any{{"role": "user", "content": parts}}} if a.JSONSchema != nil { body["response_format"] = map[string]any{"type": "json_schema", "json_schema": map[string]any{"name": "answer", "schema": a.JSONSchema, "strict": true}} } c.progress(10, "asking "+a.Model) var out struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` } `json:"choices"` } if _, err := r.selfJSONWaiting(c, a.Model, "/v1/chat/completions", body, &out); err != nil { return nil, err } if len(out.Choices) == 0 { return nil, errors.New(a.Model + " gave no answer") } return toolText(out.Choices[0].Message.Content, false), nil } func (r *Router) toolGatewayStatus(c mediaCall) (map[string]any, error) { var models struct { Data []map[string]any `json:"data"` } if _, err := r.selfJSON(c, http.MethodGet, "/v1/models", nil, &models); err != nil { return nil, err } var listed []map[string]any for _, m := range models.Data { caps, _ := m["capabilities"].(map[string]any) if caps["images"] != true && caps["meshes"] != true && caps["vision"] != true && m["workflow"] == nil { continue } e := map[string]any{"id": m["id"], "loaded": m["loaded"], "capabilities": caps} if w := m["workflow"]; w != nil { e["workflow"] = w } if l := m["license"]; l != nil { e["license"] = l } listed = append(listed, e) } out := map[string]any{"models": listed} var jobs struct { Data []jobView `json:"data"` } if _, err := r.selfJSON(c, http.MethodGet, workflowsPath+"?limit=5", nil, &jobs); err == nil { recent := make([]map[string]any, 0, len(jobs.Data)) for _, j := range jobs.Data { recent = append(recent, map[string]any{"id": j.ID, "model": j.Model, "status": j.Status}) } out["recent_jobs"] = recent } return toolJSON(out), nil } // decodeArgs reads a tool's arguments, by their JSON names: a field's own // name in lower case unless its tag says. func decodeArgs(raw json.RawMessage, into any) error { if len(raw) == 0 || string(raw) == "null" { raw = []byte("{}") } if err := json.Unmarshal(raw, into); err != nil { return fmt.Errorf("the arguments: %w", err) } return nil } // streamWriter answers a tool call as server-sent events: progress // notifications while it runs, then the answer. type streamWriter struct { mu sync.Mutex w http.ResponseWriter token json.RawMessage } func (s *streamWriter) write(text string) { s.mu.Lock() defer s.mu.Unlock() _, _ = fmt.Fprint(s.w, text) if f, ok := s.w.(http.Flusher); ok { f.Flush() } } func (s *streamWriter) event(v any) { b, _ := json.Marshal(v) s.write("event: message\ndata: " + string(b) + "\n\n") } // keepAlive writes a comment every while until stop is closed: a picture // whose model starts cold is minutes of nothing said, which a proxy reads as // a connection gone idle. func (s *streamWriter) keepAlive(stop <-chan struct{}, every time.Duration) { t := time.NewTicker(every) defer t.Stop() for { select { case <-stop: return case <-t.C: s.write(": still working\n\n") } } } func (s *streamWriter) progress(percent float64, message string) { if len(s.token) == 0 { return } s.event(map[string]any{"jsonrpc": "2.0", "method": "notifications/progress", "params": map[string]any{ "progressToken": s.token, "progress": percent, "total": 100, "message": message}}) } // acceptsStream is a client that takes a POST's answer as a stream, as the // streamable HTTP transport lets it. func acceptsStream(req *http.Request) bool { return strings.Contains(req.Header.Get("Accept"), "text/event-stream") }