Files
llmgw/internal/router/mcp_media_test.go
T
Ilker ManapandClaude Opus 5.5 edeeb9f8e1 MCP takes pictures of real size, by id too, and waits for a cold model
The game's bridge found three things on the PoC. /mcp held its body to
64 KiB — the edit meant for it had raised the memory notes' limit instead —
so a 1024 px concept sent as base64 failed as "expected one JSON-RPC
request object". The notes are back to 64 KiB, /mcp takes 64 MB, and a
call past it is a 413 saying so.

/v1/files took batch files only, so a picture could not be sent once and
named after. purpose vision and user_data are kept too, as long as job
results, and generate_mesh hands a workflow model the id as it is.

generate_image and review_images failed at once while their model was
cold. They wait out an answer that says when to come back — loading,
another model holding the GPU, a full queue — for as long as the call may,
saying who holds the GPU and about how long; a refusal that says no
Retry-After, a spent budget, is said at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 07:24:59 +02:00

372 lines
14 KiB
Go

package router
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"image"
"image/png"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// mcp makes one JSON-RPC call at /mcp and answers its result; streamed, as a
// client that takes server-sent events, it answers the events' data too.
func mcp(t *testing.T, h http.Handler, secret, method string, params any, streamed bool) (map[string]any, []string) {
t.Helper()
b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params})
req := httptest.NewRequest(http.MethodPost, "/mcp", bytes.NewReader(b))
req.Header.Set("Content-Type", "application/json")
if secret != "" {
req.Header.Set("Authorization", "Bearer "+secret)
}
if streamed {
req.Header.Set("Accept", "application/json, text/event-stream")
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: %d %s", method, rec.Code, rec.Body.String())
}
body := rec.Body.String()
var events []string
if streamed {
if ct := rec.Header().Get("Content-Type"); ct != "text/event-stream" {
t.Fatalf("a streamed call answered %s: %s", ct, body)
}
for _, line := range strings.Split(body, "\n") {
if d, ok := strings.CutPrefix(line, "data: "); ok {
events = append(events, d)
}
}
if len(events) == 0 {
t.Fatalf("no events: %s", body)
}
body = events[len(events)-1]
}
var resp struct {
Result map[string]any `json:"result"`
Error any `json:"error"`
}
if err := json.Unmarshal([]byte(body), &resp); err != nil || resp.Error != nil {
t.Fatalf("%s: %v %s", method, err, body)
}
return resp.Result, events
}
func toolNames(res map[string]any) []string {
var names []string
for _, tl := range res["tools"].([]any) {
names = append(names, tl.(map[string]any)["name"].(string))
}
return names
}
// resultInfo is a tool result's first text, as the JSON it holds.
func resultInfo(t *testing.T, res map[string]any) map[string]any {
t.Helper()
if res["isError"] == true {
t.Fatalf("the tool failed: %v", res["content"])
}
var info map[string]any
text := res["content"].([]any)[0].(map[string]any)["text"].(string)
if err := json.Unmarshal([]byte(text), &info); err != nil {
t.Fatalf("not JSON: %s", text)
}
return info
}
func TestTheMCPToolsAreWhatTheGatewaysModelsCanDo(t *testing.T) {
c := newMeshComfy(t)
rt, secret := meshRouter(t, c)
h := rt.Handler()
init, _ := mcp(t, h, secret, "initialize", map[string]any{"protocolVersion": "2025-06-18"}, false)
if init["protocolVersion"] != "2025-06-18" || !strings.Contains(init["instructions"].(string), "batch calls") {
t.Errorf("initialize: %v", init)
}
list, _ := mcp(t, h, secret, "tools/list", map[string]any{}, false)
names := strings.Join(toolNames(list), ",")
for _, want := range []string{"remember", "generate_image", "generate_mesh", "job_status", "gateway_status"} {
if !strings.Contains(names, want) {
t.Errorf("no %s in %s", want, names)
}
}
// Nothing here looks at pictures or declares a workflow's inputs.
if strings.Contains(names, "review_images") || strings.Contains(names, "run_workflow") {
t.Errorf("tools for models the gateway has not: %s", names)
}
// A gateway with only text models offers the notes alone.
plain := newTestRouter(t, baseConfig()).Handler()
list, _ = mcp(t, plain, "", "tools/list", map[string]any{}, false)
if got := strings.Join(toolNames(list), ","); got != "remember,recall,forget" {
t.Errorf("a text-only gateway offers %s", got)
}
}
func TestAMeshIsMadeThroughMCPAndKeptAsTheCallersFile(t *testing.T) {
c := newMeshComfy(t)
rt, secret := meshRouter(t, c)
h := rt.Handler()
args := map[string]any{"model": "trellis2", "image": "data:image/png;base64," + testPicture(t), "seed": 3}
for _, streamed := range []bool{false, true} {
res, events := mcp(t, h, secret, "tools/call", map[string]any{"name": "generate_mesh", "arguments": args,
"_meta": map[string]any{"progressToken": "tok"}}, streamed)
info := resultInfo(t, res)
if info["status"] != "completed" || info["license"].(map[string]any)["name"] != "MIT" {
t.Fatalf("streamed %v: %v", streamed, info)
}
id, _ := info["file_id"].(string)
if !strings.HasSuffix(info["url"].(string), "/v1/files/"+id+"/content") {
t.Errorf("url %v", info["url"])
}
// Fetched with the same key, it is the mesh.
code, _, file := call(t, h, secret, http.MethodGet, "/v1/files/"+id+"/content", nil)
if code != http.StatusOK || !strings.HasPrefix(file, "glTF") {
t.Fatalf("the file is not the mesh: %d", code)
}
if streamed && !strings.Contains(strings.Join(events, "\n"), `"progressToken":"tok"`) {
t.Errorf("no progress said while it ran: %v", events)
}
}
// The job went through the client API as the caller: it is theirs.
_, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil)
if n := len(list["data"].([]any)); n != 2 {
t.Errorf("%d jobs of the caller's, want the two made", n)
}
// Under no key, nothing is made.
b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": map[string]any{"name": "generate_mesh", "arguments": args}})
req := httptest.NewRequest(http.MethodPost, "/mcp", bytes.NewReader(b))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("a call with no key: %d", rec.Code)
}
}
func TestAPictureMadeThroughMCPComesWholeWhereNoFileCanBeKept(t *testing.T) {
rt, queued, mu := imageRouter(t)
h := rt.Handler()
res, _ := mcp(t, h, "", "tools/call", map[string]any{"name": "generate_image",
"arguments": map[string]any{"prompt": "a rock", "style": "concept", "size": 512, "seed": 9}}, false)
info := resultInfo(t, res)
if info["seed"] != 9.0 || info["model"] != "flux" || !strings.Contains(info["note"].(string), "keeps no files") {
t.Errorf("info %v", info)
}
content := res["content"].([]any)
if len(content) != 2 || content[1].(map[string]any)["type"] != "image" {
t.Fatalf("the picture is not in the answer: %v", content)
}
if b, _ := base64.StdEncoding.DecodeString(content[1].(map[string]any)["data"].(string)); string(b) != "png-bytes" {
t.Errorf("picture %q", b)
}
mu.Lock()
wf, _ := json.Marshal(*queued)
mu.Unlock()
if !strings.Contains(string(wf), "a rock, single object centered") || !strings.Contains(string(wf), `"width":512`) {
t.Errorf("the style was not added to the prompt: %s", wf)
}
}
func TestAPictureIsShrunkToItsLongSide(t *testing.T) {
var b bytes.Buffer
_ = png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 2000, 1000)))
out, err := shrink(b.Bytes(), 512)
if err != nil {
t.Fatal(err)
}
im, err := png.Decode(bytes.NewReader(out))
if err != nil || im.Bounds().Dx() != 512 || im.Bounds().Dy() != 256 {
t.Fatalf("%v %v", im.Bounds(), err)
}
b.Reset()
_ = png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 100, 50)))
if out, _ := shrink(b.Bytes(), 512); !bytes.Equal(out, b.Bytes()) {
t.Error("a small PNG was re-encoded")
}
}
func TestAWorkflowRunsThroughMCPFromItsDeclaredInputs(t *testing.T) {
wav := testWAV()
c := newRunComfy(t, wav, "song_00001_.wav")
rt, secret, _ := runRouter(t, c, songWorkflow, catalog.OutputAudio)
h := rt.Handler()
list, _ := mcp(t, h, secret, "tools/list", map[string]any{}, false)
if !strings.Contains(strings.Join(toolNames(list), ","), "run_workflow") {
t.Fatalf("no run_workflow for a model that declares its inputs: %v", toolNames(list))
}
res, _ := mcp(t, h, secret, "tools/call", map[string]any{"name": "run_workflow", "arguments": map[string]any{
"model": "ace", "inputs": map[string]any{"tags": "synthwave", "cover": "data:image/png;base64," + testPicture(t)}}}, false)
info := resultInfo(t, res)
if info["status"] != "completed" || info["content_type"] != "audio/wav" {
t.Fatalf("%v", info)
}
if _, _, file := call(t, h, secret, http.MethodGet, "/v1/files/"+info["file_id"].(string)+"/content", nil); file != string(wav) {
t.Error("the file is not the song")
}
// A run refused says why, as the API did.
res, _ = mcp(t, h, secret, "tools/call", map[string]any{"name": "run_workflow", "arguments": map[string]any{
"model": "ace", "inputs": map[string]any{"cover": "data:image/png;base64," + testPicture(t)}}}, false)
if res["isError"] != true || !strings.Contains(fmt.Sprint(res["content"]), "tags") {
t.Errorf("a run missing a required input: %v", res)
}
}
func TestPicturesAreLookedAtByAVisionModelShrunkAndToASchema(t *testing.T) {
e := newFakeEngine(t, "vlm")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"vlm"}})
vlm := catModel("vlm", nil)
vlm.Capabilities.Vision = true
cfg.Models = []catalog.Model{vlm}
h := newTestRouter(t, cfg).Handler()
var big bytes.Buffer
_ = png.Encode(&big, image.NewRGBA(image.Rect(0, 0, 3000, 1500)))
res, _ := mcp(t, h, "", "tools/call", map[string]any{"name": "review_images", "arguments": map[string]any{
"images": []string{base64.StdEncoding.EncodeToString(big.Bytes())},
"question": "Does the render match the concept?",
"json_schema": map[string]any{"type": "object", "properties": map[string]any{"match": map[string]any{"type": "boolean"}}},
}}, false)
if res["isError"] == true || !strings.Contains(fmt.Sprint(res["content"]), "hi") {
t.Fatalf("%v", res)
}
var sent struct {
ResponseFormat struct {
Type string `json:"type"`
} `json:"response_format"`
Messages []struct {
Content []struct {
Type string `json:"type"`
ImageURL struct {
URL string `json:"url"`
} `json:"image_url"`
} `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal(e.last.Load().([]byte), &sent); err != nil {
t.Fatal(err)
}
if sent.ResponseFormat.Type != "json_schema" || len(sent.Messages) != 1 || len(sent.Messages[0].Content) != 2 {
t.Fatalf("sent %+v", sent)
}
b64, _ := strings.CutPrefix(sent.Messages[0].Content[1].ImageURL.URL, "data:image/png;base64,")
pic, _ := base64.StdEncoding.DecodeString(b64)
im, err := png.Decode(bytes.NewReader(pic))
if err != nil || im.Bounds().Dx() != lookEdge || im.Bounds().Dy() != lookEdge/2 {
t.Errorf("the picture sent is %v (%v), not shrunk to %d", im.Bounds(), err, lookEdge)
}
}
// noisePNG is a picture that does not compress: n by n, so its base64 is
// past any small limit.
func noisePNG(t *testing.T, n int) []byte {
t.Helper()
im := image.NewRGBA(image.Rect(0, 0, n, n))
seed := uint32(1)
for i := range im.Pix {
seed = seed*1664525 + 1013904223
im.Pix[i] = byte(seed >> 24)
}
var b bytes.Buffer
_ = png.Encode(&b, im)
return b.Bytes()
}
func TestAPictureOfRealSizeGoesThroughMCPAndByFileID(t *testing.T) {
c := newMeshComfy(t)
rt, secret := meshRouter(t, c)
h := rt.Handler()
pic := noisePNG(t, 400) // ~640 KB, as a 1024 px concept is
res, _ := mcp(t, h, secret, "tools/call", map[string]any{"name": "generate_mesh", "arguments": map[string]any{
"model": "trellis2", "image": "data:image/png;base64," + base64.StdEncoding.EncodeToString(pic)}}, false)
if info := resultInfo(t, res); info["status"] != "completed" {
t.Fatalf("a mesh from a picture of real size: %v", info)
}
// Uploaded once as a picture to work from, then named by id.
var form bytes.Buffer
mw := multipart.NewWriter(&form)
_ = mw.WriteField("purpose", "vision")
fw, _ := mw.CreateFormFile("file", "rock.png")
_, _ = fw.Write(pic)
_ = mw.Close()
req := httptest.NewRequest(http.MethodPost, "/v1/files", &form)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+secret)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
var f map[string]any
if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &f) != nil || f["purpose"] != "vision" ||
f["expires_at"] == nil {
t.Fatalf("a picture uploaded to work from: %d %s", rec.Code, rec.Body.String())
}
res, _ = mcp(t, h, secret, "tools/call", map[string]any{"name": "generate_mesh", "arguments": map[string]any{
"model": "trellis2", "image": f["id"]}}, false)
if info := resultInfo(t, res); info["status"] != "completed" {
t.Fatalf("a mesh from an uploaded picture: %v", info)
}
}
func TestACallTooLargeIsSaidSo(t *testing.T) {
h := newTestRouter(t, baseConfig()).Handler()
big := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"x","arguments":{"image":"` +
strings.Repeat("A", mcpBody) + `"}}}`
req := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(big))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge || !strings.Contains(rec.Body.String(), "larger than") {
t.Errorf("%d %s", rec.Code, rec.Body.String()[:min(200, rec.Body.Len())])
}
}
func TestACallToAColdModelWaitsItOutSayingSo(t *testing.T) {
rt := newTestRouter(t, baseConfig())
_ = rt.Handler()
calls := 0
var fake http.Handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls++
if calls < 3 {
w.Header().Set("Retry-After", "1")
w.Header().Set(waitingForHeader, "sf3d")
w.Header().Set(etaHeader, "40")
writeError(w, http.StatusServiceUnavailable, "server_error", "model loading: waiting")
return
}
w.Header().Set("x-llmgw-seed", "4")
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
})
rt.selfHandler.Store(&fake)
var said []string
c := mediaCall{req: httptest.NewRequest(http.MethodPost, "/mcp", nil), streamed: true,
progress: func(_ float64, m string) { said = append(said, m) }}
var out map[string]any
h, err := rt.selfJSONWaiting(c, "flux-schnell", imagesPath, map[string]any{}, &out)
if err != nil || out["ok"] != true || h.Get("x-llmgw-seed") != "4" || calls != 3 {
t.Fatalf("%v %v after %d calls", err, out, calls)
}
if len(said) != 2 || !strings.Contains(said[0], "waiting for sf3d to give up the GPU, about 40 s") {
t.Errorf("said %q", said)
}
// Refused for good — no Retry-After — is said at once.
calls = 0
fake = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls++
writeError(w, http.StatusPaymentRequired, "insufficient_quota", "the budget is spent")
})
rt.selfHandler.Store(&fake)
if _, err := rt.selfJSONWaiting(c, "flux-schnell", imagesPath, map[string]any{}, &out); err == nil ||
!strings.Contains(err.Error(), "budget") || calls != 1 {
t.Errorf("%v after %d calls", err, calls)
}
}