A job says how far along it is; a ComfyUI model brings its custom nodes
G2: the gate keeps a websocket open to its ComfyUI under a client id of its
own, puts that id on every workflow it lets through — ComfyUI tells only
the client that queued a workflow how it goes — and keeps the last word on
each: the node running, its step of how many. It answers at
/progress/{prompt_id}. While a job's request waits, the router reads the
queue (queued, n ahead) and the gate (running, node, step, percent) and
writes it into the job, which a client reads back. Only into a job of the
same key: a tag naming another's job writes nothing.
G3, the rest of the plumbing: comfyui.nodes names custom nodes by https
repository and full commit, with the PyPI packages they need. The gate puts
each in the replica's own custom_nodes at that commit and installs the
packages beside the replica, constrained to the versions ComfyUI's
environment already has, which it never changes. The catalog refuses a
branch for a commit and anything but package specs for requirements. An
OBJ a node saves leaves the gateway as a GLB, colours kept, so every mesh
can carry its licence.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c87e413360
commit
88781cfcfe
14 files changed
+1064
-6
No files matched your search
@@ -247,6 +247,17 @@ type Client struct {
|
||||
// workflow out of the queue, or interrupts it if it is running, so a client
|
||||
// that went away does not leave the card working for nobody.
|
||||
func (c *Client) Generate(ctx context.Context, base, apiKey string, workflow map[string]any) ([]Image, error) {
|
||||
return c.GenerateWatch(ctx, base, apiKey, workflow, nil)
|
||||
}
|
||||
|
||||
// progressEvery is how often a workflow's progress is asked after, beside
|
||||
// its history: a few requests a second is for the end, not for the news.
|
||||
const progressEvery = time.Second
|
||||
|
||||
// GenerateWatch is Generate, saying how far along the workflow is as it
|
||||
// changes: queued and how many ahead, then the node running and its step.
|
||||
func (c *Client) GenerateWatch(ctx context.Context, base, apiKey string, workflow map[string]any,
|
||||
onProgress func(Progress)) ([]Image, error) {
|
||||
base = strings.TrimRight(base, "/")
|
||||
client := make([]byte, 8)
|
||||
_, _ = rand.Read(client)
|
||||
@@ -277,6 +288,8 @@ func (c *Client) Generate(ctx context.Context, base, apiKey string, workflow map
|
||||
}
|
||||
t := time.NewTicker(poll)
|
||||
defer t.Stop()
|
||||
var last Progress
|
||||
var asked time.Time
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -284,6 +297,13 @@ func (c *Client) Generate(ctx context.Context, base, apiKey string, workflow map
|
||||
return nil, ctx.Err()
|
||||
case <-t.C:
|
||||
}
|
||||
if onProgress != nil && time.Since(asked) >= progressEvery {
|
||||
asked = time.Now()
|
||||
if p, ok := c.progress(ctx, base, apiKey, id, workflow); ok && p != last {
|
||||
last = p
|
||||
onProgress(p)
|
||||
}
|
||||
}
|
||||
var hist map[string]historyEntry
|
||||
if _, err := c.do(ctx, http.MethodGet, base+"/history/"+url.PathEscape(id), apiKey, nil, &hist); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
@@ -638,3 +658,60 @@ func (c *Client) Upload(ctx context.Context, base, apiKey, name string, data []b
|
||||
}
|
||||
return out.Name, nil
|
||||
}
|
||||
|
||||
// progress is a workflow's: its place in the queue, from ComfyUI, and the
|
||||
// node running and its step, from the gate when there is one — a ComfyUI
|
||||
// outside the cluster has no gate, and says only where it is queued.
|
||||
func (c *Client) progress(ctx context.Context, base, apiKey, id string, workflow map[string]any) (Progress, bool) {
|
||||
p := Progress{PromptID: id, State: "queued"}
|
||||
var q struct {
|
||||
Running [][]json.RawMessage `json:"queue_running"`
|
||||
Pending [][]json.RawMessage `json:"queue_pending"`
|
||||
}
|
||||
if _, err := c.do(ctx, http.MethodGet, base+"/queue", apiKey, nil, &q); err != nil {
|
||||
return p, false
|
||||
}
|
||||
quoted := strconv.Quote(id)
|
||||
queued := false
|
||||
for _, r := range q.Running {
|
||||
if len(r) > 1 && string(r[1]) == quoted {
|
||||
p.State = "running"
|
||||
}
|
||||
}
|
||||
if p.State != "running" {
|
||||
// Pending in the order ComfyUI will take them: by number.
|
||||
mine := -1.0
|
||||
var numbers []float64
|
||||
for _, r := range q.Pending {
|
||||
if len(r) < 2 {
|
||||
continue
|
||||
}
|
||||
var n float64
|
||||
_ = json.Unmarshal(r[0], &n)
|
||||
if string(r[1]) == quoted {
|
||||
mine, queued = n, true
|
||||
continue
|
||||
}
|
||||
numbers = append(numbers, n)
|
||||
}
|
||||
if queued {
|
||||
p.Ahead = len(q.Running)
|
||||
for _, n := range numbers {
|
||||
if n < mine {
|
||||
p.Ahead++
|
||||
}
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
var g Progress
|
||||
if st, err := c.do(ctx, http.MethodGet, base+"/progress/"+url.PathEscape(id), apiKey, nil, &g); err == nil &&
|
||||
st == http.StatusOK && g.State != "" {
|
||||
p = g
|
||||
p.PromptID = id
|
||||
if node, ok := workflow[p.Node].(map[string]any); ok {
|
||||
p.NodeClass, _ = node["class_type"].(string)
|
||||
}
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
+49
-4
@@ -2,7 +2,10 @@ package comfy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -92,6 +95,18 @@ func RunGate(ctx context.Context, cfg GateConfig) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
out := cfg.Output
|
||||
if out == nil {
|
||||
out = os.Stdout
|
||||
}
|
||||
nodes, err := parseNodes(os.Getenv(NodesEnv))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
site, err := installNodes(ctx, state, cfg.Python, nodes, findUV(), func(s string) { fmt.Fprintln(out, "[gate] "+s) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
models := filepath.Join(state, "models")
|
||||
if err := LinkWeights(cfg.Weights, models); err != nil {
|
||||
return err
|
||||
@@ -112,9 +127,12 @@ func RunGate(ctx context.Context, cfg GateConfig) error {
|
||||
child := exec.Command(cfg.Python, args...) //nolint:gosec // the agent's own configuration
|
||||
child.Dir = cfg.Dir
|
||||
child.Env = os.Environ()
|
||||
out := cfg.Output
|
||||
if out == nil {
|
||||
out = os.Stdout
|
||||
if site != "" {
|
||||
pp := site
|
||||
if old := os.Getenv("PYTHONPATH"); old != "" {
|
||||
pp += string(os.PathListSeparator) + old
|
||||
}
|
||||
child.Env = append(child.Env, "PYTHONPATH="+pp)
|
||||
}
|
||||
// ComfyUI writes to a pipe of the gate's, never to what the gate was
|
||||
// given. That is the agent's, and an agent that restarts and adopts the
|
||||
@@ -141,9 +159,16 @@ func RunGate(ctx context.Context, cfg GateConfig) error {
|
||||
}()
|
||||
|
||||
upstream := &url.URL{Scheme: "http", Host: net.JoinHostPort("127.0.0.1", strconv.Itoa(inner))}
|
||||
// The workflows' progress, heard on ComfyUI's websocket under an id of
|
||||
// the gate's own.
|
||||
book := newProgressBook()
|
||||
cid := make([]byte, 8)
|
||||
_, _ = rand.Read(cid)
|
||||
clientID := "llmgw-gate-" + hex.EncodeToString(cid)
|
||||
go book.watch(ctx, upstream, clientID, log)
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Listen,
|
||||
Handler: NewGate(upstream, cfg.Key, cfg.Model),
|
||||
Handler: newGate(upstream, cfg.Key, cfg.Model, book, clientID),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
served := make(chan error, 1)
|
||||
@@ -184,6 +209,12 @@ func shutdown(srv *http.Server) {
|
||||
|
||||
// NewGate is the gate's handler, in front of the ComfyUI at upstream.
|
||||
func NewGate(upstream *url.URL, key, model string) http.Handler {
|
||||
return newGate(upstream, key, model, newProgressBook(), "")
|
||||
}
|
||||
|
||||
// newGate is the gate with the book of its workflows' progress, kept under
|
||||
// clientID; an empty one leaves the workflows' own client ids alone.
|
||||
func newGate(upstream *url.URL, key, model string, book *progressBook, clientID string) http.Handler {
|
||||
proxy := httputil.NewSingleHostReverseProxy(upstream)
|
||||
base := proxy.Director
|
||||
proxy.Director = func(r *http.Request) {
|
||||
@@ -232,6 +263,20 @@ func NewGate(upstream *url.URL, key, model string) http.Handler {
|
||||
fmt.Fprintf(w, `{"object":"list","data":[{"id":%q,"object":"model","owned_by":"comfyui"}]}`, model)
|
||||
return
|
||||
}
|
||||
if id, ok := strings.CutPrefix(r.URL.Path, "/progress/"); ok && r.Method == http.MethodGet {
|
||||
p, found := book.get(id)
|
||||
if !found {
|
||||
http.Error(w, "no word of that workflow", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(p)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/prompt" && r.Method == http.MethodPost && clientID != "" {
|
||||
book.promptThroughGate(proxy, clientID)(w, r)
|
||||
return
|
||||
}
|
||||
for _, p := range gatePaths {
|
||||
if r.URL.Path == p || strings.HasSuffix(p, "/") && strings.HasPrefix(r.URL.Path, p) {
|
||||
proxy.ServeHTTP(w, r)
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package comfy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Custom nodes, put in a replica's own base directory at the commit the
|
||||
// catalog names, their packages installed beside the replica rather than
|
||||
// into ComfyUI's environment: one model's node cannot break another
|
||||
// model's replicas, or the pictures every replica makes.
|
||||
|
||||
// Node is a custom node as the catalog gives it.
|
||||
type Node struct {
|
||||
Repo string `json:"repo"`
|
||||
Commit string `json:"commit"`
|
||||
Requirements []string `json:"requirements,omitempty"`
|
||||
}
|
||||
|
||||
// NodesEnv is where the agent hands the gate a model's custom nodes.
|
||||
const NodesEnv = "LLMGW_COMFYUI_NODES"
|
||||
|
||||
var nodeDir = regexp.MustCompile(`[^A-Za-z0-9._-]`)
|
||||
|
||||
// installNodes puts each node in state/custom_nodes at its commit and its
|
||||
// packages in state/site, and returns the PYTHONPATH ComfyUI is started
|
||||
// with: empty when no node wants a package. Run as the engines' user, in
|
||||
// the replica's own directory, before ComfyUI starts.
|
||||
func installNodes(ctx context.Context, state, python string, nodes []Node, uv string, out func(string)) (string, error) {
|
||||
if len(nodes) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Minute)
|
||||
defer cancel()
|
||||
var reqs []string
|
||||
for _, n := range nodes {
|
||||
name := nodeDir.ReplaceAllString(strings.TrimSuffix(path.Base(n.Repo), ".git"), "_")
|
||||
dir := filepath.Join(state, "custom_nodes", name)
|
||||
if head, err := gitOutput(ctx, dir, "rev-parse", "HEAD"); err == nil && head == n.Commit {
|
||||
reqs = append(reqs, n.Requirements...)
|
||||
continue
|
||||
}
|
||||
out(fmt.Sprintf("custom node %s at %s", n.Repo, n.Commit[:12]))
|
||||
_ = os.RemoveAll(dir)
|
||||
for _, args := range [][]string{
|
||||
{"init", "-q", dir},
|
||||
{"-C", dir, "remote", "add", "origin", n.Repo},
|
||||
{"-C", dir, "fetch", "-q", "--depth", "1", "origin", n.Commit},
|
||||
{"-C", dir, "checkout", "-q", "FETCH_HEAD"},
|
||||
} {
|
||||
if b, err := exec.CommandContext(ctx, "git", args...).CombinedOutput(); err != nil { //nolint:gosec // the catalog's repository, checked as https
|
||||
return "", fmt.Errorf("custom node %s at %s: git %s: %v: %s", n.Repo, n.Commit, args[len(args)-2], err, lastLines(b))
|
||||
}
|
||||
}
|
||||
reqs = append(reqs, n.Requirements...)
|
||||
}
|
||||
if len(reqs) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
if uv == "" {
|
||||
return "", fmt.Errorf("custom nodes need %s, and no uv is installed to install them with", strings.Join(reqs, ", "))
|
||||
}
|
||||
site := filepath.Join(state, "site")
|
||||
marker := filepath.Join(site, ".llmgw-requirements")
|
||||
want := strings.Join(reqs, "\n")
|
||||
if b, err := os.ReadFile(marker); err == nil && string(b) == want {
|
||||
return site, nil
|
||||
}
|
||||
out("installing " + strings.Join(reqs, " ") + " for the custom nodes")
|
||||
// What ComfyUI's environment has, as constraints: a package both want is
|
||||
// installed at the version ComfyUI runs, so the replica's copy and
|
||||
// ComfyUI's never disagree.
|
||||
freeze, err := exec.CommandContext(ctx, uv, "pip", "freeze", "--python", python).Output() //nolint:gosec // the agent's own uv
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading ComfyUI's packages: %w", err)
|
||||
}
|
||||
constraints := filepath.Join(state, "constraints.txt")
|
||||
if err := os.WriteFile(constraints, freeze, 0o644); err != nil { //nolint:gosec // not secret
|
||||
return "", err
|
||||
}
|
||||
_ = os.RemoveAll(site)
|
||||
args := append([]string{"pip", "install", "--python", python, "--target", site, "-c", constraints, "--quiet"}, reqs...)
|
||||
cmd := exec.CommandContext(ctx, uv, args...) //nolint:gosec // requirements checked by the catalog as package specs
|
||||
cmd.Env = append(os.Environ(), "UV_NO_CONFIG=1", "UV_CACHE_DIR="+filepath.Join(state, ".uv-cache"))
|
||||
if b, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("installing the custom nodes' packages (%s): %v: %s", strings.Join(reqs, " "), err, lastLines(b))
|
||||
}
|
||||
// The replica's copies come first on ComfyUI's path, but of anything
|
||||
// both have they are the same version, so which is found is the same.
|
||||
if err := os.WriteFile(marker, []byte(want), 0o644); err != nil { //nolint:gosec // not secret
|
||||
return "", err
|
||||
}
|
||||
return site, nil
|
||||
}
|
||||
|
||||
// parseNodes reads the nodes the agent handed over.
|
||||
func parseNodes(s string) ([]Node, error) {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var nodes []Node
|
||||
if err := json.Unmarshal([]byte(s), &nodes); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", NodesEnv, err)
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// findUV is the uv the agent installed engines with.
|
||||
func findUV() string {
|
||||
if m, _ := filepath.Glob("/opt/llmgw/engines/uv/*/*/uv"); len(m) > 0 {
|
||||
return m[len(m)-1]
|
||||
}
|
||||
if p, err := exec.LookPath("uv"); err == nil {
|
||||
return p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
|
||||
b, err := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).Output() //nolint:gosec // fixed arguments
|
||||
return strings.TrimSpace(string(b)), err
|
||||
}
|
||||
|
||||
func lastLines(b []byte) string {
|
||||
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
||||
if len(lines) > 4 {
|
||||
lines = lines[len(lines)-4:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package comfy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
|
||||
// How far along a workflow is, which ComfyUI says only on its websocket and
|
||||
// only to the client that queued the workflow. The gate keeps one websocket
|
||||
// open to its ComfyUI, under a client id of its own, puts that id on every
|
||||
// workflow it lets through, and keeps the last word on each: the node
|
||||
// running and its step out of how many. The router asks at
|
||||
// /progress/{prompt_id} while it waits, and a job says it.
|
||||
|
||||
// Progress is a workflow's state as the gate last heard it.
|
||||
type Progress struct {
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
// State is queued, running, done or failed.
|
||||
State string `json:"state"`
|
||||
// Ahead is how many workflows wait before it, while it is queued.
|
||||
Ahead int `json:"ahead,omitempty"`
|
||||
// Node is the workflow's node running, by its id; NodeClass its kind.
|
||||
Node string `json:"node,omitempty"`
|
||||
NodeClass string `json:"node_class,omitempty"`
|
||||
// Step of Steps is the running node's own count: a sampler's steps.
|
||||
Step int `json:"step,omitempty"`
|
||||
Steps int `json:"steps,omitempty"`
|
||||
// Percent is Step of Steps, where a node counts.
|
||||
Percent int `json:"percent,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// progressBook is the last word on each of the most recent workflows.
|
||||
type progressBook struct {
|
||||
mu sync.Mutex
|
||||
by map[string]*Progress
|
||||
order []string
|
||||
}
|
||||
|
||||
// keptPrompts bounds the book: a gate serves one replica, which runs one
|
||||
// workflow at a time and queues a few.
|
||||
const keptPrompts = 256
|
||||
|
||||
func newProgressBook() *progressBook { return &progressBook{by: map[string]*Progress{}} }
|
||||
|
||||
func (b *progressBook) update(id string, f func(*Progress)) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
p, ok := b.by[id]
|
||||
if !ok {
|
||||
p = &Progress{PromptID: id, State: "queued"}
|
||||
b.by[id] = p
|
||||
b.order = append(b.order, id)
|
||||
for len(b.order) > keptPrompts {
|
||||
delete(b.by, b.order[0])
|
||||
b.order = b.order[1:]
|
||||
}
|
||||
}
|
||||
f(p)
|
||||
}
|
||||
|
||||
func (b *progressBook) get(id string) (Progress, bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
p, ok := b.by[id]
|
||||
if !ok {
|
||||
return Progress{}, false
|
||||
}
|
||||
return *p, true
|
||||
}
|
||||
|
||||
// read takes one message off ComfyUI's websocket into the book.
|
||||
func (b *progressBook) read(msg []byte) {
|
||||
var m struct {
|
||||
Type string `json:"type"`
|
||||
Data struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
Node json.RawMessage `json:"node"`
|
||||
Value int `json:"value"`
|
||||
Max int `json:"max"`
|
||||
Message string `json:"exception_message"`
|
||||
NodeType string `json:"node_type"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(msg, &m) != nil {
|
||||
return
|
||||
}
|
||||
d := m.Data
|
||||
var node string
|
||||
_ = json.Unmarshal(d.Node, &node)
|
||||
switch m.Type {
|
||||
case "execution_start":
|
||||
b.update(d.PromptID, func(p *Progress) { p.State, p.Ahead = "running", 0 })
|
||||
case "executing":
|
||||
b.update(d.PromptID, func(p *Progress) {
|
||||
if node == "" {
|
||||
// No node: the workflow is through.
|
||||
p.State, p.Node, p.Step, p.Steps, p.Percent = "done", "", 0, 0, 100
|
||||
return
|
||||
}
|
||||
p.State, p.Ahead, p.Node, p.Step, p.Steps, p.Percent = "running", 0, node, 0, 0, 0
|
||||
})
|
||||
case "progress":
|
||||
b.update(d.PromptID, func(p *Progress) {
|
||||
p.State, p.Step, p.Steps = "running", d.Value, d.Max
|
||||
if node != "" {
|
||||
p.Node = node
|
||||
}
|
||||
if d.Max > 0 {
|
||||
p.Percent = min(100, d.Value*100/d.Max)
|
||||
}
|
||||
})
|
||||
case "execution_success":
|
||||
b.update(d.PromptID, func(p *Progress) { p.State, p.Percent = "done", 100 })
|
||||
case "execution_error", "execution_interrupted":
|
||||
b.update(d.PromptID, func(p *Progress) {
|
||||
p.State = "failed"
|
||||
p.Error = strings.TrimSpace(d.NodeType + ": " + d.Message)
|
||||
if m.Type == "execution_interrupted" {
|
||||
p.Error = "interrupted"
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// watch keeps a websocket open to the ComfyUI at upstream under clientID,
|
||||
// reading into the book, until ctx ends: opened again when it closes, as it
|
||||
// does whenever ComfyUI restarts.
|
||||
func (b *progressBook) watch(ctx context.Context, upstream *url.URL, clientID string, log *slog.Logger) {
|
||||
ws := "ws://" + upstream.Host + "/ws?clientId=" + url.QueryEscape(clientID)
|
||||
wait := time.Second
|
||||
for ctx.Err() == nil {
|
||||
conn, err := websocket.Dial(ws, "", "http://"+upstream.Host)
|
||||
if err == nil {
|
||||
wait = time.Second
|
||||
go func() { <-ctx.Done(); conn.Close() }()
|
||||
for {
|
||||
var msg []byte
|
||||
if err := websocket.Message.Receive(conn, &msg); err != nil {
|
||||
break
|
||||
}
|
||||
// Binary frames are previews of pictures; only text is news.
|
||||
if len(msg) > 0 && msg[0] == '{' {
|
||||
b.read(msg)
|
||||
}
|
||||
}
|
||||
conn.Close()
|
||||
} else if log != nil {
|
||||
log.Debug("ComfyUI's websocket is not open yet", "err", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(wait):
|
||||
}
|
||||
wait = min(wait*2, 30*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// withClient is a /prompt body with the gate's client id in it, so the
|
||||
// progress of the workflow it queues comes to the gate's websocket.
|
||||
func withClient(body []byte, clientID string) []byte {
|
||||
var m map[string]json.RawMessage
|
||||
if json.Unmarshal(body, &m) != nil {
|
||||
return body
|
||||
}
|
||||
id, _ := json.Marshal(clientID)
|
||||
m["client_id"] = id
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// promptThroughGate forwards a /prompt with the gate's client id, and books
|
||||
// the workflow it queued as queued.
|
||||
func (b *progressBook) promptThroughGate(proxy http.Handler, clientID string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body = withClient(body, clientID)
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
r.ContentLength = int64(len(body))
|
||||
rec := &captured{ResponseWriter: w}
|
||||
proxy.ServeHTTP(rec, r)
|
||||
var out struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
}
|
||||
if json.Unmarshal(rec.body.Bytes(), &out) == nil && out.PromptID != "" {
|
||||
b.update(out.PromptID, func(*Progress) {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// captured passes a response on and keeps a copy of the first of it.
|
||||
type captured struct {
|
||||
http.ResponseWriter
|
||||
body bytes.Buffer
|
||||
}
|
||||
|
||||
func (c *captured) Write(p []byte) (int, error) {
|
||||
if c.body.Len() < 64<<10 {
|
||||
c.body.Write(p)
|
||||
}
|
||||
return c.ResponseWriter.Write(p)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package comfy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTheBookReadsComfyUIsWebsocket(t *testing.T) {
|
||||
b := newProgressBook()
|
||||
for _, m := range []string{
|
||||
`{"type":"status","data":{"status":{"exec_info":{"queue_remaining":1}}}}`,
|
||||
`{"type":"execution_start","data":{"prompt_id":"p1"}}`,
|
||||
`{"type":"executing","data":{"node":"5","prompt_id":"p1"}}`,
|
||||
`{"type":"progress","data":{"value":12,"max":30,"prompt_id":"p1","node":"5"}}`,
|
||||
} {
|
||||
b.read([]byte(m))
|
||||
}
|
||||
p, _ := b.get("p1")
|
||||
if p.State != "running" || p.Node != "5" || p.Step != 12 || p.Steps != 30 || p.Percent != 40 {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
b.read([]byte(`{"type":"executing","data":{"node":null,"prompt_id":"p1"}}`))
|
||||
if p, _ := b.get("p1"); p.State != "done" || p.Percent != 100 {
|
||||
t.Errorf("finished: %+v", p)
|
||||
}
|
||||
b.read([]byte(`{"type":"execution_error","data":{"prompt_id":"p2","node_type":"KSampler","exception_message":"CUDA out of memory"}}`))
|
||||
if p, _ := b.get("p2"); p.State != "failed" || p.Error != "KSampler: CUDA out of memory" {
|
||||
t.Errorf("failed: %+v", p)
|
||||
}
|
||||
for i := range keptPrompts + 10 {
|
||||
b.update(fmt.Sprint("x", i), func(*Progress) {})
|
||||
}
|
||||
if _, ok := b.get("p1"); ok || len(b.by) != keptPrompts {
|
||||
t.Errorf("the book kept %d", len(b.by))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGateHearsTheWorkflowsItLetsThrough(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var sent map[string]any
|
||||
comfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/prompt" {
|
||||
mu.Lock()
|
||||
_ = json.NewDecoder(r.Body).Decode(&sent)
|
||||
mu.Unlock()
|
||||
fmt.Fprint(w, `{"prompt_id":"p9"}`)
|
||||
}
|
||||
}))
|
||||
defer comfy.Close()
|
||||
u, _ := url.Parse(comfy.URL)
|
||||
book := newProgressBook()
|
||||
gate := httptest.NewServer(newGate(u, "", "m", book, "llmgw-gate-1"))
|
||||
defer gate.Close()
|
||||
resp, err := http.Post(gate.URL+"/prompt", "application/json", strings.NewReader(`{"prompt":{},"client_id":"router"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.Contains(string(b), "p9") {
|
||||
t.Fatalf("the answer did not come through: %s", b)
|
||||
}
|
||||
mu.Lock()
|
||||
if sent["client_id"] != "llmgw-gate-1" {
|
||||
t.Errorf("ComfyUI was sent client %v: its progress would go to nobody listening", sent["client_id"])
|
||||
}
|
||||
mu.Unlock()
|
||||
book.read([]byte(`{"type":"progress","data":{"value":3,"max":4,"prompt_id":"p9","node":"2"}}`))
|
||||
resp, _ = http.Get(gate.URL + "/progress/p9")
|
||||
var p Progress
|
||||
_ = json.NewDecoder(resp.Body).Decode(&p)
|
||||
resp.Body.Close()
|
||||
if p.State != "running" || p.Percent != 75 {
|
||||
t.Errorf("/progress: %+v", p)
|
||||
}
|
||||
if resp, _ := http.Get(gate.URL + "/progress/unknown"); resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("an unknown workflow: %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingWorkflowSaysHowManyAreAheadAndThenHowFarAlong(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
stage := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
s := stage
|
||||
mu.Unlock()
|
||||
switch r.URL.Path {
|
||||
case "/prompt":
|
||||
fmt.Fprint(w, `{"prompt_id":"me"}`)
|
||||
case "/queue":
|
||||
if s == 0 {
|
||||
fmt.Fprint(w, `{"queue_running":[[1,"a"]],"queue_pending":[[3,"me"],[2,"b"],[4,"c"]]}`)
|
||||
} else {
|
||||
fmt.Fprint(w, `{"queue_running":[[3,"me"]],"queue_pending":[]}`)
|
||||
}
|
||||
case "/progress/me":
|
||||
fmt.Fprint(w, `{"state":"running","node":"5","step":6,"steps":12,"percent":50}`)
|
||||
case "/history/me":
|
||||
if s < 2 {
|
||||
fmt.Fprint(w, `{}`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"me":{"status":{"status_str":"success","completed":true},"outputs":{"9":{"images":[{"filename":"a.png","type":"temp"}]}}}}`)
|
||||
case "/view":
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
fmt.Fprint(w, "png")
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
var seen []Progress
|
||||
c := &Client{Poll: time.Millisecond}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.GenerateWatch(t.Context(), srv.URL, "", map[string]any{"5": map[string]any{"class_type": "KSampler"}},
|
||||
func(p Progress) {
|
||||
mu.Lock()
|
||||
seen = append(seen, p)
|
||||
stage++
|
||||
mu.Unlock()
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(seen) < 2 || seen[0].State != "queued" || seen[0].Ahead != 2 {
|
||||
t.Fatalf("%+v: queued behind the one running and the one numbered before it, wanted first", seen)
|
||||
}
|
||||
if seen[1].State != "running" || seen[1].Percent != 50 || seen[1].NodeClass != "KSampler" {
|
||||
t.Errorf("then %+v", seen[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeIsPutAtItsCommit(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("no git")
|
||||
}
|
||||
repo := t.TempDir()
|
||||
run := func(args ...string) string {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t")
|
||||
b, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v %s", args, err, b)
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
run("init", "-q")
|
||||
run("config", "uploadpack.allowReachableSHA1InWant", "true")
|
||||
_ = os.WriteFile(filepath.Join(repo, "__init__.py"), []byte("v1\n"), 0o644)
|
||||
run("add", ".")
|
||||
run("commit", "-qm", "one")
|
||||
first := run("rev-parse", "HEAD")
|
||||
_ = os.WriteFile(filepath.Join(repo, "__init__.py"), []byte("v2\n"), 0o644)
|
||||
run("commit", "-qam", "two")
|
||||
|
||||
state := t.TempDir()
|
||||
site, err := installNodes(t.Context(), state, "python3", []Node{{Repo: "file://" + repo, Commit: first}}, "", func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := os.ReadFile(filepath.Join(state, "custom_nodes", filepath.Base(repo), "__init__.py"))
|
||||
if string(b) != "v1\n" || site != "" {
|
||||
t.Errorf("the node is %q, site %q: wanted the commit named, not the branch's head", b, site)
|
||||
}
|
||||
if _, err := installNodes(t.Context(), state, "python3", []Node{{Repo: "file://" + repo, Commit: first,
|
||||
Requirements: []string{"trimesh"}}}, "", func(string) {}); err == nil || !strings.Contains(err.Error(), "no uv") {
|
||||
t.Errorf("packages without a uv: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user