Ilker ManapandClaude Opus 5.5
069d334652
install.md said the client port belongs behind a reverse proxy and gave nothing to put there. deploy/proxy/ has both, attached to every release. What they get right is what a first attempt gets wrong: streamed replies are not buffered, the read timeout outlasts a model's silence before its first token (nginx's 60s default is a 504), the body limit matches the router's, and X-Forwarded-Proto and -Host are set by the proxy rather than passed through — llmgw marks session cookies Secure from the first and builds the landing page's base URL from the second. The Caddyfile was validated and run against a streaming upstream: events arrive as sent, and forged forwarded headers are replaced. nginx uses `listen ... http2` because `http2 on;` needs 1.25.1 and Ubuntu 22.04 and Debian 12 ship older. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
64 lines
2.4 KiB
Nginx Configuration File
64 lines
2.4 KiB
Nginx Configuration File
# llmgw behind nginx: the client-facing port (8080) on a public name.
|
|
#
|
|
# Put this in /etc/nginx/sites-available/llmgw (or conf.d/llmgw.conf), replace
|
|
# llm.example.com, and get a certificate — certbot --nginx -d llm.example.com
|
|
# fills in the ssl_ lines below. Then: nginx -t && systemctl reload nginx
|
|
#
|
|
# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's
|
|
# own authority and must reach llmgw directly; 9090 and 9091 stay private.
|
|
|
|
upstream llmgw {
|
|
server 127.0.0.1:8080;
|
|
keepalive 32;
|
|
}
|
|
|
|
server {
|
|
listen 80;
|
|
listen [::]:80;
|
|
server_name llm.example.com;
|
|
return 301 https://$host$request_uri;
|
|
}
|
|
|
|
server {
|
|
# `listen ... http2` rather than `http2 on;`: the latter needs nginx
|
|
# 1.25.1, and Ubuntu 22.04 and Debian 12 ship older ones. Newer nginx
|
|
# accepts this form with a deprecation warning.
|
|
listen 443 ssl http2;
|
|
listen [::]:443 ssl http2;
|
|
server_name llm.example.com;
|
|
|
|
ssl_certificate /etc/letsencrypt/live/llm.example.com/fullchain.pem;
|
|
ssl_certificate_key /etc/letsencrypt/live/llm.example.com/privkey.pem;
|
|
|
|
# The router reads at most 32 MiB of body (router.max_body_bytes).
|
|
client_max_body_size 32m;
|
|
|
|
location / {
|
|
proxy_pass http://llmgw;
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Connection "";
|
|
|
|
# Set, never appended: llmgw decides from X-Forwarded-Proto whether
|
|
# session cookies are Secure, and builds the base URL the landing page
|
|
# shows from X-Forwarded-Host. A value passed through from the client
|
|
# would be the client's choice.
|
|
proxy_set_header Host $host;
|
|
proxy_set_header X-Forwarded-Host $host;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
|
|
# Streamed replies: tokens go out as they are generated. Buffering
|
|
# turns a stream into one late burst, and gzip buffers too.
|
|
proxy_buffering off;
|
|
proxy_cache off;
|
|
gzip off;
|
|
|
|
# nginx's default is 60 seconds between two reads from upstream. A
|
|
# large model can think longer than that before its first token, and
|
|
# the client then sees a 504 for a request that was about to succeed.
|
|
# llmgw has its own limits (timeouts.*); this must not be the shorter.
|
|
proxy_read_timeout 15m;
|
|
proxy_send_timeout 15m;
|
|
}
|
|
}
|