{ "about.meta.description": "llmgw serves models on your own GPUs behind one OpenAI-compatible API.", "about.home_label": "llmgw home", "about.nav.label": "Sections", "about.nav.what": "What it does", "about.nav.fabrics": "Fabrics", "about.nav.install": "Install", "about.nav.proxy": "Reverse proxy", "about.nav.download": "Download", "about.nav.source": "Source", "about.nav.gateway": "This gateway", "about.hero.eyebrow": "Self-hosted LLM gateway ยท %s", "about.hero.title": "One endpoint in front of your own GPUs.", "about.hero.lede": "llmgw runs vLLM, SGLang and llama.cpp across your servers and serves them behind one OpenAI-compatible API. It knows which GPUs share an NVLink domain or an InfiniBand fabric, and a model split across machines never straddles two.", "about.hero.download": "Download", "about.hero.guide": "Install guide", "about.hero.oneline_label": "Set up a control plane", "about.copy": "Copy", "about.copied": "Copied", "about.selected": "Selected", "about.schem.title": "Clients reach one llmgw address. llmgw places a model's shards on GPUs inside one fabric at a time.", "about.schem.editor": "editor agent", "about.schem.caption": "A replica spanning nodes goes into one NVLink domain if one has room, otherwise onto one InfiniBand fabric. Two fabrics that cannot see each other are never bridged: the all-reduce would hang.", "about.what.title": "What it does", "about.what.intro": "One static binary. It runs as the control plane, as the agent on every GPU node, and as the weights mirror.", "about.what.api.title": "OpenAI API, your hardware", "about.what.api.text": "Point any OpenAI client at it. Only the base URL changes. Streaming, tool calls, /v1/models and /v1/responses work as your client expects.", "about.what.placement.title": "Placement that knows the fabric", "about.what.placement.text": "Agents report GPU class, memory, NVLink peers and InfiniBand ports. Replicas go where they fit, including ones larger than one machine, on the GPUs with the most NVLink pairs.", "about.what.ondemand.title": "Models load when asked for", "about.what.ondemand.text": "Keep more models in the catalog than your GPUs hold. The first request starts one, the chat page shows it loading, and it stops again after the idle timeout.", "about.what.keys.title": "Keys, budgets, rate limits", "about.what.keys.text": "Organisations, teams and projects, each with limits. Priority bands decide who waits when the cluster is full. Managers see their organisation's spend; people see their own.", "about.what.editors.title": "Editors connect from the account page", "about.what.editors.text": "Pick Continue, Codex or Cline and a model. The page makes a key and hands over the config, filled in, or a one-line command that writes it.", "about.what.chat.title": "A chat page for quick questions", "about.what.chat.text": "Markdown and maths rendered, stop mid-reply, tokens per second and time to first token, a system prompt and a temperature. The gateway keeps no conversations.", "about.what.mirror.title": "A mirror you browse", "about.what.mirror.text": "Explore what is trending on Hugging Face, read a model card under its row, and download once for the whole cluster. Only what your nodes can run is shown by default.", "about.what.engines.title": "Nodes install their own engines", "about.what.engines.text": "llama.cpp on every node; vLLM on every GPU node that can run it, from PyPI with every file hash-checked. A node that cannot run it says why.", "about.what.upgrades.title": "Upgrades that keep models running", "about.what.upgrades.text": "Upgrade the control plane, then the agents. A restarted agent adopts the engines it was running instead of reloading them, so replicas stay up throughout.", "about.what.dashboard.title": "A dashboard with nothing to set up", "about.what.dashboard.text": "/admin shows capacity per GPU class, CPU and memory per node, who is using what, each node's page and logs. It fits a phone. Prometheus is optional.", "about.fabrics.title": "InfiniBand and NVLink, several of each", "about.fabrics.intro": "A site may have independent InfiniBand fabrics, eight-GPU HGX boards and NVL72 racks side by side. llmgw tells them apart by what the adapters report, not by what a config file calls them.", "about.fabrics.guide": "Fabrics and NVLink guide", "about.fabrics.order_label": "Where a replica spanning nodes is placed, in order", "about.fabrics.nvlink.title": "One NVLink domain", "about.fabrics.nvlink.text": "NVL72 or NVLink Switch. Every card joined, IMEX up. Shards get NCCL_MNNVL_ENABLE=1.", "about.fabrics.ib.title": "One InfiniBand or RoCE fabric", "about.fabrics.ib.text": "Nodes under one leaf first, nodes with a degraded cable last.", "about.fabrics.never.title": "Never across two", "about.fabrics.never.text": "Two fabrics given one name are placed as two, and the cluster page says so.", "about.fabrics.doctor": "Check a node in one command", "about.fabrics.doctor_label": "Example output of llmgw doctor", "about.fabrics.sm.title": "Subnet managers, watched", "about.fabrics.sm.text": "Master, priority, standbys and failovers per fabric. A fabric with no standby is flagged. llmgw does not run one.", "about.fabrics.map.title": "A map of each fabric", "about.fabrics.map.text": "/admin/fabrics draws switches in tiers, cables coloured by rate, degraded ones in red, and live traffic on your adapters.", "about.fabrics.racks.title": "NVLink domains as racks", "about.fabrics.racks.text": "/admin/nvlink shows which trays are joined, whose IMEX is down, which link runs degraded.", "about.fabrics.ports.title": "Ports re-read every heartbeat", "about.fabrics.ports.text": "A port that goes down takes its fabric off the node within seconds. Counters and rates are exported to Prometheus.", "about.install.title": "From nothing to a first request", "about.install.intro": "Debian, Ubuntu or RHEL 10. NVIDIA drivers and the InfiniBand stack are yours to install; llmgw does the rest.", "about.install.guide": "The full guide covers the mirror, GPUs and what to do when something does not work.", "about.install.binary.title": "Put the binary on every machine", "about.install.binary.text": "Static, no dependencies, the same file everywhere.", "about.install.server.title": "Set up the control plane", "about.install.server.text": "Creates the user, the certificate authority, the certificate, server.yaml and your administrator account, then starts the service. Safe to run again. Add --host for any name nodes use that the machine cannot see itself.", "about.install.node.title": "Add a node", "about.install.node.text": "From the control plane, which holds the authority. The node gets its certificate, the binary and the agent, and installs its engine by itself.", "about.install.node.output_label": "Example output of llmgw ctl nodes", "about.install.model.title": "Add a model and send a request", "about.install.model.text": "No restart. If it does not place, llmgw ctl explain-placement names every node and why it was turned down.", "about.proxy.title": "Behind a reverse proxy", "about.proxy.intro": "Port 8080 is plain HTTP. Put it behind Caddy or nginx with a public certificate. Only 8080 goes through the proxy: agents reach 7443 directly, because it is mutual TLS with the cluster's own authority.", "about.proxy.loopback": "Bind llmgw to loopback when the proxy is on the same machine: listen: \"127.0.0.1:8080\".", "about.proxy.caddy": "Obtains and renews its own certificate. Replace llm.example.com.", "about.proxy.nginx": "Certificate from certbot --nginx. Works on nginx 1.18 and later.", "about.proxy.stream.title": "Streams are not buffered", "about.proxy.stream.text": "Tokens reach the client as they are generated, not in one burst at the end.", "about.proxy.timeout.title": "The read timeout outlasts the model", "about.proxy.timeout.text": "nginx's 60-second default turns a slow first token into a 504.", "about.proxy.headers.title": "Forwarded headers are set, not passed on", "about.proxy.headers.text": "llmgw marks session cookies Secure from X-Forwarded-Proto and shows X-Forwarded-Host as the base URL.", "about.proxy.body.title": "Body limit matches the router", "about.proxy.body.text": "32 MiB, so an oversized request is refused before it is uploaded.", "about.download.title": "Download", "about.download.intro": "Built by CI from the tag and checked against checksums.txt. Every release is on the releases page with its changelog.", "about.download.released": "Released %s", "about.download.col.for": "For", "about.download.col.file": "File", "about.download.col.notes": "Notes", "about.download.static_any": "Static. Runs on any libc.", "about.download.static": "Static.", "about.download.nvml": "Sees NVLink topology through NVML. For PCIe boxes with NVLink bridges. Needs the driver.", "about.download.mac_arm": "macOS Apple silicon", "about.download.mac_note": "For llmgw ctl and trying it on a laptop.", "about.download.proxy_note": "Reverse proxy for port 8080.", "about.download.checksums": "Checksums", "about.download.checksums_note": "SHA-256 of every file above.", "about.download.dev": "This is a development build (%s). Take a release from the releases page, where every file is listed with its checksum.", "about.status.title": "Where it stands", "about.status.intro": "Pre-1.0: configuration formats may change between minor versions. This is what has run on real hardware, and what has not yet.", "about.status.proven": "Run on real clusters", "about.status.proven.single": "Single-node replicas on real GPUs and on CPU", "about.status.proven.ondemand": "On-demand loading and idle shutdown", "about.status.proven.keys": "Keys, budgets, rate limits and priority bands", "about.status.proven.upgrades": "Rolling upgrades with no dropped replicas", "about.status.proven.stores": "SQLite and PostgreSQL stores", "about.status.proven.chat": "Chat, editor setup and the Responses API against a six-node test cluster", "about.status.open": "Not yet proven", "about.status.open.fabrics": "InfiniBand and NVLink on a real site: written and tested against the real output of sminfo, ibnetdiscover and nvidia-smi, but not yet run on the hardware. Run llmgw doctor --json on a few nodes first.", "about.status.open.planes": "Two control planes over long periods", "about.status.open.weights": "Nodes do not share downloaded weights; the mirror keeps each download local", "about.footer.changelog": "Changelog", "about.footer.reference": "Configuration reference" }