diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index ff84ff9..4dc30fe 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -55,6 +55,13 @@ jobs: (cd dist && sha256sum "${name}" >> checksums.txt) done + # The reverse proxy configurations, so a release page carries + # everything install.md asks for and the repository is not needed. + for f in deploy/proxy/Caddyfile deploy/proxy/nginx.conf; do + cp "$f" dist/ + (cd dist && sha256sum "$(basename "$f")" >> checksums.txt) + done + # An NVML build for nodes with NVLink, which the portable binary # cannot serve. # diff --git a/README.md b/README.md index d8cd243..ed829e0 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ on a machine that has them. does not exist or miss one that does - [deploy/mirror/](deploy/mirror/) — a model mirror, which is the cheapest and most useful thing to build around llmgw +- [deploy/proxy/](deploy/proxy/) — Caddy and nginx in front of the client + port, with streaming and the forwarded headers right - [deploy/ha/](deploy/ha/) — running two control planes, and the one health check that has to be right - [docs/node-onboarding.md](docs/node-onboarding.md), diff --git a/deploy/proxy/Caddyfile b/deploy/proxy/Caddyfile new file mode 100644 index 0000000..a27034e --- /dev/null +++ b/deploy/proxy/Caddyfile @@ -0,0 +1,41 @@ +# llmgw behind Caddy: the client-facing port (8080) on a public name, with a +# certificate Caddy obtains and renews by itself. +# +# Replace llm.example.com, point its DNS at this machine, open 80 and 443, and +# run: caddy run --config /etc/caddy/Caddyfile +# +# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's +# own authority and must reach llmgw directly; 9090 and 9091 stay private. + +llm.example.com { + # The router reads at most 32 MiB of body (router.max_body_bytes). Refusing + # more here gives the client a 413 before the upload, not after it. + request_body { + max_size 32MB + } + + reverse_proxy 127.0.0.1:8080 { + # Tokens are written the moment they are generated. Without this a + # streamed reply can arrive in bursts, and a client measuring time to + # first token measures the proxy instead. + flush_interval -1 + + # Caddy sets X-Forwarded-Proto and X-Forwarded-Host itself and, with + # no trusted_proxies, discards whatever the client sent. llmgw relies + # on that: the proto decides whether session cookies are Secure, and + # the host is the base URL the landing page tells people to use. + # Behind a load balancer of your own, name it in trusted_proxies. + + transport http { + # A long generation can be silent for a while before the first + # token. llmgw has its own limits (timeouts.*); this must not be + # the shorter one. + response_header_timeout 15m + } + } + + # No `encode` on purpose: compressing a stream buffers it. + + # Access log to stderr, which under systemd is the journal. + log +} diff --git a/deploy/proxy/README.md b/deploy/proxy/README.md new file mode 100644 index 0000000..e82424c --- /dev/null +++ b/deploy/proxy/README.md @@ -0,0 +1,36 @@ +# A reverse proxy in front of llmgw + +The client-facing port (8080) is plain HTTP. On anything but a trusted network, +put it behind a proxy that holds a publicly trusted certificate. + +| File | For | +| --- | --- | +| [Caddyfile](Caddyfile) | Caddy — obtains and renews its own certificate; the least work | +| [nginx.conf](nginx.conf) | nginx, with a certificate from certbot | + +Both are also attached to every release. + +Four things both files get right, and any other proxy has to as well: + +- **Streaming is not buffered.** Replies are server-sent events; a buffering + proxy delivers them in one burst at the end. +- **The read timeout is longer than a model's silence.** nginx's default of 60 + seconds turns a slow first token into a 504. +- **`X-Forwarded-Proto` and `X-Forwarded-Host` are set by the proxy**, replacing + whatever the client sent. llmgw uses the first to mark session cookies + `Secure` and the second for the base URL on its landing page. +- **The body limit matches llmgw's** (32 MiB, `router.max_body_bytes`), so an + oversized request is refused before it is uploaded. + +Only port 8080 is proxied. Agents reach 7443 directly — it is mutual TLS with +the cluster's own authority, and a proxy terminating it would take away the +node identity it carries. 9090 (admin, pprof) stays on loopback; 9091 +(metrics) goes to your monitoring host only. + +With a proxy on the same machine, bind llmgw to loopback so nothing reaches it +around the proxy: + +```yaml +# /etc/llmgw/server.yaml +listen: "127.0.0.1:8080" +``` diff --git a/deploy/proxy/nginx.conf b/deploy/proxy/nginx.conf new file mode 100644 index 0000000..29157aa --- /dev/null +++ b/deploy/proxy/nginx.conf @@ -0,0 +1,63 @@ +# llmgw behind nginx: the client-facing port (8080) on a public name. +# +# Put this in /etc/nginx/sites-available/llmgw (or conf.d/llmgw.conf), replace +# llm.example.com, and get a certificate — certbot --nginx -d llm.example.com +# fills in the ssl_ lines below. Then: nginx -t && systemctl reload nginx +# +# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's +# own authority and must reach llmgw directly; 9090 and 9091 stay private. + +upstream llmgw { + server 127.0.0.1:8080; + keepalive 32; +} + +server { + listen 80; + listen [::]:80; + server_name llm.example.com; + return 301 https://$host$request_uri; +} + +server { + # `listen ... http2` rather than `http2 on;`: the latter needs nginx + # 1.25.1, and Ubuntu 22.04 and Debian 12 ship older ones. Newer nginx + # accepts this form with a deprecation warning. + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name llm.example.com; + + ssl_certificate /etc/letsencrypt/live/llm.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/llm.example.com/privkey.pem; + + # The router reads at most 32 MiB of body (router.max_body_bytes). + client_max_body_size 32m; + + location / { + proxy_pass http://llmgw; + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Set, never appended: llmgw decides from X-Forwarded-Proto whether + # session cookies are Secure, and builds the base URL the landing page + # shows from X-Forwarded-Host. A value passed through from the client + # would be the client's choice. + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Streamed replies: tokens go out as they are generated. Buffering + # turns a stream into one late burst, and gzip buffers too. + proxy_buffering off; + proxy_cache off; + gzip off; + + # nginx's default is 60 seconds between two reads from upstream. A + # large model can think longer than that before its first token, and + # the client then sees a 504 for a request that was about to succeed. + # llmgw has its own limits (timeouts.*); this must not be the shorter. + proxy_read_timeout 15m; + proxy_send_timeout 15m; + } +} diff --git a/docs/install.md b/docs/install.md index ad0091e..b959aeb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -128,8 +128,10 @@ If this machine is exposed, move the directory elsewhere once the nodes are in and pass `--pki` to onboarding. This covers the fleet-facing port only. The client-facing API is public and -belongs behind an ordinary publicly trusted certificate, usually terminated by a -reverse proxy in front of port 8080. +belongs behind an ordinary publicly trusted certificate, terminated by a +reverse proxy in front of port 8080. [deploy/proxy/](../deploy/proxy/) has a +Caddyfile and an nginx configuration that get streaming, timeouts and the +forwarded headers right; both are attached to every release. ### Check it