G3, the part that needs no model: a ComfyUI model's comfyui.output (or external.output) says pictures or meshes. A request for a 3D model — a job at /v1/3d/generations — carries the picture as base64; at the replica it is uploaded to ComfyUI's input directory (the gate lets /upload/image through), the workflow is filled in with its name, the seed and the resolution, and the mesh the workflow saves comes back with its type, which is the file the job keeps. comfy.Client reads every list of files an output names, not "images" only, and types a file by its extension where ComfyUI sends octet-stream. A model's license (name, url, notice) is said in /v1/models and in the job, and written into a GLB's asset.extras, since the file travels without the API; a mesh it cannot be written into is refused. per_mesh in auth.pricing, usage.meshes in the rows, the sums and the export (migration 0045). A picture asked of a mesh model, or a mesh of a picture model, is refused saying which it makes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
248 lines
8.9 KiB
Go
248 lines
8.9 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"fmt"
|
|
"image"
|
|
"image/png"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// testGLB is the smallest glTF binary: an asset, and a binary chunk after
|
|
// its JSON that must come through untouched.
|
|
func testGLB(t *testing.T) []byte {
|
|
t.Helper()
|
|
js := []byte(`{"asset":{"version":"2.0","generator":"test"},"buffers":[{"byteLength":4}]}`)
|
|
for len(js)%4 != 0 {
|
|
js = append(js, ' ')
|
|
}
|
|
bin := []byte{1, 2, 3, 4}
|
|
var b []byte
|
|
b = append(b, "glTF"...)
|
|
b = binary.LittleEndian.AppendUint32(b, 2)
|
|
b = binary.LittleEndian.AppendUint32(b, uint32(12+8+len(js)+8+len(bin)))
|
|
b = binary.LittleEndian.AppendUint32(b, uint32(len(js)))
|
|
b = append(b, "JSON"...)
|
|
b = append(b, js...)
|
|
b = binary.LittleEndian.AppendUint32(b, uint32(len(bin)))
|
|
b = append(b, "BIN\x00"...)
|
|
return append(b, bin...)
|
|
}
|
|
|
|
// meshComfy is a ComfyUI that takes a picture and saves a mesh, as a 3D
|
|
// node's SaveGLB does: under "3d", not "images".
|
|
type meshComfy struct {
|
|
srv *httptest.Server
|
|
mu sync.Mutex
|
|
uploaded []string
|
|
workflow map[string]any
|
|
}
|
|
|
|
func newMeshComfy(t *testing.T) *meshComfy {
|
|
glb := testGLB(t)
|
|
c := &meshComfy{}
|
|
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/upload/image":
|
|
f, h, err := r.FormFile("image")
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
b, _ := io.ReadAll(f)
|
|
c.mu.Lock()
|
|
c.uploaded = append(c.uploaded, h.Filename+":"+http.DetectContentType(b))
|
|
c.mu.Unlock()
|
|
fmt.Fprintf(w, `{"name":%q,"subfolder":"","type":"input"}`, h.Filename)
|
|
case "/prompt":
|
|
var b struct {
|
|
Prompt map[string]any `json:"prompt"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&b)
|
|
c.mu.Lock()
|
|
c.workflow = b.Prompt
|
|
c.mu.Unlock()
|
|
fmt.Fprint(w, `{"prompt_id":"m1"}`)
|
|
case "/history/m1":
|
|
fmt.Fprint(w, `{"m1":{"status":{"status_str":"success","completed":true},"outputs":{
|
|
"3":{"text":["a preview"]},"4":{"3d":[{"filename":"mesh_00001_.glb","subfolder":"3d","type":"output"}]}}}}`)
|
|
case "/view":
|
|
// Served as ComfyUI serves a file it has no type for.
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
_, _ = w.Write(glb)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(c.srv.Close)
|
|
return c
|
|
}
|
|
|
|
const meshWorkflow = `{
|
|
"1": {"class_type": "LoadImage", "inputs": {"image": "{{image}}"}},
|
|
"2": {"class_type": "Trellis2ImageToMesh", "inputs": {"image": ["1", 0], "seed": "{{seed}}", "resolution": "{{resolution|512}}"}},
|
|
"4": {"class_type": "SaveGLB", "inputs": {"mesh": ["2", 0], "filename_prefix": "3d/mesh"}}
|
|
}`
|
|
|
|
func testPicture(t *testing.T) string {
|
|
var b bytes.Buffer
|
|
_ = png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 8, 8)))
|
|
return base64.StdEncoding.EncodeToString(b.Bytes())
|
|
}
|
|
|
|
func meshRouter(t *testing.T, c *meshComfy) (*Router, string) {
|
|
t.Helper()
|
|
cfg := baseConfig()
|
|
mit := &catalog.License{Name: "MIT", URL: "https://github.com/microsoft/TRELLIS.2/blob/main/LICENSE",
|
|
Notice: "Made with TRELLIS.2 (MIT)."}
|
|
cfg.Models = []catalog.Model{
|
|
{Name: "trellis2", Source: "external", License: mit, External: &catalog.External{URL: c.srv.URL,
|
|
Model: "trellis2", API: catalog.APIComfyUI, Output: catalog.OutputMeshes, Workflow: "w.json"}},
|
|
{Name: "flux", Source: "external", External: &catalog.External{URL: c.srv.URL,
|
|
Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}},
|
|
}
|
|
cfg.Auth.Enabled = true
|
|
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
|
cfg.Auth.Pricing = map[string]config.Price{"trellis2": {PerMesh: 250000}}
|
|
cfg.Auth.FlushInterval = 20 * time.Millisecond
|
|
cfg.Auth.BatchSize = 1
|
|
rt := newTestRouter(t, cfg)
|
|
rt.comfy = &comfy.Client{Poll: time.Millisecond}
|
|
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
|
{ID: "external:trellis2", URL: c.srv.URL, Models: []string{"trellis2"}, Engine: endpoint.EngineComfyUI,
|
|
Pool: "external", Weight: 1, UpstreamModel: "trellis2", Workflow: meshWorkflow},
|
|
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
|
|
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
|
|
})
|
|
org, _ := rt.store.CreateOrg(t.Context(), "studio")
|
|
team, _ := rt.store.CreateTeam(t.Context(), org, "art")
|
|
project, err := rt.store.CreateProject(t.Context(), team, "props")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret := store.KeyPrefix + strings.Repeat("m", 43)
|
|
sum := sha256.Sum256([]byte(secret))
|
|
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-m", Hash: sum[:], Prefix: "mmmm", Name: "art",
|
|
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rt, secret
|
|
}
|
|
|
|
func TestA3DModelIsMadeFromAPictureWithItsLicenceInside(t *testing.T) {
|
|
c := newMeshComfy(t)
|
|
rt, secret := meshRouter(t, c)
|
|
h := rt.Handler()
|
|
|
|
code, j, raw := call(t, h, secret, http.MethodPost, meshesPath,
|
|
map[string]any{"model": "trellis2", "image": "data:image/png;base64," + testPicture(t), "seed": 11, "resolution": 1024})
|
|
if code != http.StatusAccepted {
|
|
t.Fatalf("%d %s", code, raw)
|
|
}
|
|
got := waitJob(t, h, secret, j["id"].(string), jobDone)
|
|
if got["status"] != "completed" || got["content_type"] != "model/gltf-binary" {
|
|
t.Fatalf("job %v", got)
|
|
}
|
|
if lic, _ := got["license"].(map[string]any); lic["name"] != "MIT" {
|
|
t.Errorf("the job does not say the licence: %v", got["license"])
|
|
}
|
|
code, _, file := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
|
|
if code != http.StatusOK || !strings.HasPrefix(file, "glTF") {
|
|
t.Fatalf("the result is not the mesh: %d %q", code, file[:min(len(file), 40)])
|
|
}
|
|
b := []byte(file)
|
|
n := binary.LittleEndian.Uint32(b[12:])
|
|
var doc struct {
|
|
Asset struct {
|
|
Generator string `json:"generator"`
|
|
Extras struct {
|
|
License catalog.License `json:"license"`
|
|
} `json:"extras"`
|
|
} `json:"asset"`
|
|
}
|
|
if err := json.Unmarshal(b[20:20+n], &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if doc.Asset.Extras.License.Notice != "Made with TRELLIS.2 (MIT)." || doc.Asset.Generator != "test" {
|
|
t.Errorf("asset %+v: the notice is not in the mesh, or what was there is lost", doc.Asset)
|
|
}
|
|
if int(binary.LittleEndian.Uint32(b[8:])) != len(b) || !bytes.HasSuffix(b, []byte{1, 2, 3, 4}) {
|
|
t.Error("the mesh's length or its binary chunk did not survive")
|
|
}
|
|
|
|
c.mu.Lock()
|
|
wf, _ := json.Marshal(c.workflow)
|
|
uploaded := strings.Join(c.uploaded, ",")
|
|
c.mu.Unlock()
|
|
if !strings.HasSuffix(uploaded, ".png:image/png") || !strings.HasPrefix(uploaded, "llmgw-") {
|
|
t.Errorf("uploaded %q", uploaded)
|
|
}
|
|
name := strings.TrimSuffix(uploaded, ":image/png")
|
|
for _, want := range []string{`"image":"` + name + `"`, `"seed":11`, `"resolution":1024`} {
|
|
if !strings.Contains(string(wf), want) {
|
|
t.Errorf("%s not in the workflow sent: %s", want, wf)
|
|
}
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
u, ok, _ := rt.store.UsageByRequest(t.Context(), j["id"].(string))
|
|
if ok {
|
|
if u.Meshes != 1 || u.CostMicros < 250000 {
|
|
t.Errorf("recorded %d meshes, %d micros; want one at its price", u.Meshes, u.CostMicros)
|
|
}
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("no usage recorded")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestA3DModelIsAskedOfAModelThatMakesThem(t *testing.T) {
|
|
c := newMeshComfy(t)
|
|
rt, secret := meshRouter(t, c)
|
|
h := rt.Handler()
|
|
for name, c := range map[string]struct {
|
|
path string
|
|
body map[string]any
|
|
want string
|
|
}{
|
|
"a mesh of flux": {meshesPath, map[string]any{"model": "flux", "image": testPicture(t)}, "makes pictures, not 3D models"},
|
|
"no picture": {meshesPath, map[string]any{"model": "trellis2"}, "image is required"},
|
|
"not a picture": {meshesPath, map[string]any{"model": "trellis2", "image": base64.StdEncoding.EncodeToString([]byte("hello"))}, "not a PNG"},
|
|
"an odd resolution": {meshesPath, map[string]any{"model": "trellis2", "image": testPicture(t), "resolution": 700}, "not 512, 1024 or 1536"},
|
|
"a picture of trellis2": {imagesPath, map[string]any{"model": "trellis2", "prompt": "a cube"}, "makes 3D models, not pictures"},
|
|
"a chat with trellis2": {"/v1/chat/completions", map[string]any{"model": "trellis2", "messages": []any{}}, "makes 3D models"},
|
|
} {
|
|
code, _, raw := call(t, h, secret, http.MethodPost, c.path, c.body)
|
|
if code != http.StatusBadRequest || !strings.Contains(raw, c.want) {
|
|
t.Errorf("%s: %d %s", name, code, raw)
|
|
}
|
|
}
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
req.Header.Set("Authorization", "Bearer "+secret)
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, req)
|
|
if !strings.Contains(rec.Body.String(), `"meshes":true`) || !strings.Contains(rec.Body.String(), `"license":{"name":"MIT"`) {
|
|
t.Errorf("/v1/models: %s", rec.Body.String())
|
|
}
|
|
}
|