package router import ( "bytes" "context" "crypto/rand" "encoding/base64" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/comfy" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // 3D models from a picture, made by ComfyUI. Clients ask for them as a job // (jobs.go); the job worker replays the request through the ordinary path, // and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's // input directory, the model's workflow is filled in with its name and run, // and the mesh it saves comes back as the answer — the file the job keeps. // meshRequest is what a request for a 3D model asks. The picture is the // whole of the prompt: base64, or a data URL. type meshRequest struct { Model string `json:"model"` Image string `json:"image"` Seed *int64 `json:"seed"` Steps int `json:"steps"` // Resolution is the model's own grid, where it has a choice: TripoSR's // marching cubes take 256 by default, TRELLIS.2 512 to 1536. Zero leaves // it to the workflow. Resolution int `json:"resolution"` picture []byte ext string } // maxPicture bounds the picture a mesh is made from. const maxPicture = 16 << 20 func parseMeshRequest(body []byte) (meshRequest, error) { var in meshRequest if err := json.Unmarshal(body, &in); err != nil { return in, err } s := strings.TrimSpace(in.Image) if s == "" { return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL") } if strings.HasPrefix(s, "data:") { _, after, ok := strings.Cut(s, ";base64,") if !ok { return in, errors.New("image is a data URL without base64 in it") } s = after } if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 { return in, fmt.Errorf("image is past %d MB", maxPicture>>20) } b, err := base64.StdEncoding.DecodeString(s) if err != nil { return in, errors.New("image is not base64: " + err.Error()) } switch http.DetectContentType(b) { case "image/png": in.ext = ".png" case "image/jpeg": in.ext = ".jpg" case "image/webp": in.ext = ".webp" default: return in, errors.New("image is not a PNG, JPEG or WebP picture") } in.picture = b if in.Steps < 0 || in.Steps > maxSteps { return in, fmt.Errorf("steps is between 1 and %d", maxSteps) } if r := in.Resolution; r != 0 && (r < 64 || r > 2048 || r%32 != 0) { return in, fmt.Errorf("resolution %d is not a multiple of 32 from 64 to 2048", r) } return in, nil } // generateMeshes answers a request for a 3D model at a ComfyUI replica. It // writes the mesh itself, with its type, which is what the job keeps. func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) { in, err := parseMeshRequest(body) if err != nil { return proxy.Outcome{}, err } wf, _ := ep.ComfyFor(req.Model) tpl, err := comfy.Load(wf) if err != nil { return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err) } if t := r.cfg.Timeouts.Total; t > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, t) defer cancel() } start := time.Now() fail := func(err error) (proxy.Outcome, error) { var ce *comfy.Error switch { case errors.As(err, &ce): return proxy.Outcome{Retryable: ce.Retryable}, err case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx): return proxy.Outcome{}, proxy.ErrClientGone case errors.Is(err, context.DeadlineExceeded): return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total) } return proxy.Outcome{}, err } // A name of its own: two requests' pictures must not overwrite each // other in the replica's input directory. rnd := make([]byte, 8) _, _ = rand.Read(rnd) name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture) if err != nil { return fail(err) } seed := comfy.RandomSeed() if in.Seed != nil { seed = *in.Seed } var upstream struct { Model string `json:"model"` } _ = json.Unmarshal(body, &upstream) p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}} if in.Resolution > 0 { p.Extra["resolution"] = in.Resolution } workflow, err := tpl.Fill(p) if err != nil { return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err) } files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req)) if err != nil { return fail(err) } var mesh *comfy.Image for i := range files { if isMeshType(files[i].ContentType) { mesh = &files[i] break } } if mesh == nil { return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+ "its last node should save a .glb", ep.ID, len(files)) } data := mesh.Data if mesh.ContentType == "model/obj" { glb, err := objToGLB(data) if err != nil { return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err) } data, mesh.ContentType = glb, "model/gltf-binary" } m := r.modelMap()[req.Model] if m.License != nil && mesh.ContentType == "model/gltf-binary" { if with, err := glbWithLicense(data, *m.License); err == nil { data = with } else { // Refused rather than sent without it: a mesh travels without // the API, and the notice is what the terms ask to travel too. return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err) } } w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10)) if m.License != nil { w.Header().Set("x-llmgw-license", m.License.Name) } w.Header().Set("Content-Type", mesh.ContentType) w.WriteHeader(http.StatusOK) _, _ = w.Write(data) took := time.Since(start) return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took, Meshes: 1}, nil } func isMeshType(ct string) bool { return strings.HasPrefix(ct, "model/") || ct == "application/ply" } // glbWithLicense is a GLB with the licence in asset.extras.license, where a // file's terms travel with it: glTF's own place for what an application // adds, kept by every tool that loads and saves the file. func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) { if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 { return nil, errors.New("not a glTF 2 binary") } n := int(binary.LittleEndian.Uint32(glb[12:])) if string(glb[16:20]) != "JSON" || 20+n > len(glb) { return nil, errors.New("its first chunk is not its JSON") } var doc map[string]any d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n])) d.UseNumber() // every number as it was written if err := d.Decode(&doc); err != nil { return nil, err } asset, _ := doc["asset"].(map[string]any) if asset == nil { asset = map[string]any{"version": "2.0"} } extras, _ := asset["extras"].(map[string]any) if extras == nil { extras = map[string]any{} } // The whole licence, as /v1/models gives it: its id, whether it allows // commercial use and where it grants nothing are what a pipeline that // sorts assets by licence reads, and the name alone is for people. var l map[string]any b, err := json.Marshal(lic) if err != nil { return nil, err } if err := json.Unmarshal(b, &l); err != nil { return nil, err } extras["license"] = l asset["extras"] = extras doc["asset"] = asset js, err := json.Marshal(doc) if err != nil { return nil, err } // A chunk is padded to four bytes, the JSON one with spaces. for len(js)%4 != 0 { js = append(js, ' ') } rest := glb[20+n:] out := make([]byte, 0, 20+len(js)+len(rest)) out = append(out, "glTF"...) out = binary.LittleEndian.AppendUint32(out, 2) out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read out = append(out, "JSON"...) out = append(out, js...) return append(out, rest...), nil } // jobProgress is where a job's request says how far along it is: into the // job, for whoever asks after it. Nil for a request that is no job's — and // for one that only says it is: the job must be the same key's, so nobody // writes into another's progress by naming it in a tag. func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) { id := req.Tags["job"] if id == "" || r.store == nil || id != req.RequestID { return nil } j, err := r.store.JobByID(ctx, id) if err != nil || j.KeyID != req.KeyID { return nil } return func(p comfy.Progress) { b, err := json.Marshal(p) if err != nil { return } if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil { r.log.Warn("a job's progress could not be kept", "job", id, "err", err) } } } // registerMeshesPage mounts the page that makes 3D models, beside the // pictures page and for the same reasons: a session, never a key, and the // request sent down the path an API client's takes — here, a job. func (r *Router) registerMeshesPage(mux *http.ServeMux, pages *web.AccountPages) { pages.SetMeshes(func() bool { for _, m := range r.modelMap() { if m.MakesMeshes() { return true } } return false }) mux.HandleFunc("GET /3d", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { contexts, _ := r.chatContexts(req.Context(), user) current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context")) if err != nil { current = "" } if current == "" && !r.personalOff { current = personalContext } web.Meshes(user, r.modelsOf(req, user, catalog.Model.MakesMeshes), contexts, current, version.Version)(w, req) })) mux.HandleFunc("POST /3d/generate", pages.Guard(r.meshesSend)) mux.HandleFunc("GET /3d/jobs/{id}", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { if next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")); ok { r.handleGetJob(w, next) } })) // The mesh itself, from where the page is, for its viewer to load. mux.HandleFunc("GET /3d/jobs/{id}/file", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")) if !ok { return } p, _ := auth.FromContext(next.Context()) j, ok := r.ownedJob(w, next, ownerOf(p)) if !ok { return } if j.OutputFileID == "" { http.Error(w, "the job has made nothing yet", http.StatusNotFound) return } b, err := r.store.FileContent(req.Context(), j.OutputFileID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", j.ContentType) w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", j.ID+".glb")) _, _ = w.Write(b) })) } // asPerson is a page's request as the signed-in person's, in the place they // chose: what the API handlers read a key's principal from. func (r *Router) asPerson(w http.ResponseWriter, req *http.Request, user store.User, context string) (*http.Request, bool) { p, err := r.principalFor(req, user, context) if err != nil { http.Error(w, err.Error(), http.StatusForbidden) return nil, false } return req.WithContext(auth.WithPrincipal(req.Context(), p)), true } // meshesSend asks for a 3D model on behalf of a signed-in person. func (r *Router) meshesSend(w http.ResponseWriter, req *http.Request, user store.User) { body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)) if err != nil { http.Error(w, "could not read the request", http.StatusBadRequest) return } var in struct { Model string `json:"model"` Image string `json:"image"` Seed *int64 `json:"seed"` Resolution int `json:"resolution"` Context string `json:"context"` } if err := json.Unmarshal(body, &in); err != nil || in.Model == "" { http.Error(w, "a model and a picture are required", http.StatusBadRequest) return } // Rebuilt from the fields the page has, so it can set nothing else. out := map[string]any{"model": in.Model, "image": in.Image} if in.Seed != nil { out["seed"] = *in.Seed } if in.Resolution > 0 { out["resolution"] = in.Resolution } upstream, err := json.Marshal(out) if err != nil { http.Error(w, "could not build the request", http.StatusInternalServerError) return } next, ok := r.asPerson(w, req, user, in.Context) if !ok { return } next = next.Clone(next.Context()) next.Body = io.NopCloser(bytes.NewReader(upstream)) next.ContentLength = int64(len(upstream)) next.Header.Set("Content-Type", "application/json") r.handleCreateJob(w, next) }