The portal in English and Turkish: a catalogue per language, a switcher at the foot of every page, landing.language, Turkish numbers and dates
Every template and handler message in internal/web goes through a JSON
catalogue under internal/web/locales/<code>/, with English as the fallback.
The language is the person's choice (a cookie set by /_lang/{code}), else
Accept-Language, else landing.language. Tests refuse a key some language
lacks, a translation with other arguments than English, and template text
left in English.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
11231d8c2b
113 files changed
+8091
-1981
No files matched your search
+123
-162
@@ -1,7 +1,7 @@
|
||||
{{define "about"}}{{if not .Site}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<html lang="{{lang}}"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="description" content="llmgw serves models on your own GPUs behind one OpenAI-compatible API.">
|
||||
<meta name="description" content="{{t "about.meta.description"}}">
|
||||
{{end}}<title>llmgw</title>
|
||||
{{if .Site}}<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
@@ -176,52 +176,50 @@ footer a{color:var(--muted)}
|
||||
|
||||
<div class="wrap">
|
||||
<header class="bar">
|
||||
<a class="mark" href="#top" aria-label="llmgw home">
|
||||
<a class="mark" href="#top" aria-label="{{t "about.home_label"}}">
|
||||
<svg width="26" height="26" viewBox="0 0 26 26" aria-hidden="true">
|
||||
<rect x="1" y="1" width="24" height="24" rx="6" fill="none" stroke="currentColor" stroke-width="2"/>
|
||||
<path d="M7 9h12M7 13h12M7 17h12" stroke="var(--fiber)" stroke-width="2" stroke-linecap="round"/>
|
||||
</svg>
|
||||
llmgw
|
||||
</a>
|
||||
<nav aria-label="Sections">
|
||||
<a href="#what">What it does</a>
|
||||
<a href="#fabrics">Fabrics</a>
|
||||
<a href="#install">Install</a>
|
||||
<a href="#proxy">Reverse proxy</a>
|
||||
<a href="#download">Download</a>
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a>
|
||||
{{if not .Site}}<a href="/">This gateway</a>{{end}}
|
||||
<nav aria-label="{{t "about.nav.label"}}">
|
||||
<a href="#what">{{t "about.nav.what"}}</a>
|
||||
<a href="#fabrics">{{t "about.nav.fabrics"}}</a>
|
||||
<a href="#install">{{t "about.nav.install"}}</a>
|
||||
<a href="#proxy">{{t "about.nav.proxy"}}</a>
|
||||
<a href="#download">{{t "about.nav.download"}}</a>
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw">{{t "about.nav.source"}}</a>
|
||||
{{if not .Site}}<a href="/">{{t "about.nav.gateway"}}</a>{{end}}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="hero" id="top">
|
||||
<div>
|
||||
<p class="eyebrow">Self-hosted LLM gateway · {{.Version}}</p>
|
||||
<h1 style="margin-top:14px">One endpoint in front of <em>your own GPUs</em>.</h1>
|
||||
<p class="lede">llmgw runs vLLM, SGLang and llama.cpp across your servers and serves them
|
||||
behind one OpenAI-compatible API. It knows which GPUs share an NVLink domain or an InfiniBand
|
||||
fabric, and a model split across machines never straddles two.</p>
|
||||
<p class="eyebrow">{{t "about.hero.eyebrow" .Version}}</p>
|
||||
<h1 style="margin-top:14px">{{th "about.hero.title"}}</h1>
|
||||
<p class="lede">{{t "about.hero.lede"}}</p>
|
||||
<div class="cta">
|
||||
<a class="btn" href="#download">Download{{with .Release}} {{.}}{{end}}</a>
|
||||
<a class="btn ghost" href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/quick-install.md">Install guide</a>
|
||||
<a class="btn" href="#download">{{t "about.hero.download"}}{{with .Release}} {{.}}{{end}}</a>
|
||||
<a class="btn ghost" href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/quick-install.md">{{t "about.hero.guide"}}</a>
|
||||
</div>
|
||||
<div class="oneline" aria-label="Set up a control plane">
|
||||
<div class="oneline" aria-label="{{t "about.hero.oneline_label"}}">
|
||||
<code><span class="p">$ </span>sudo llmgw server init --admin you@example.com</code>
|
||||
<button class="copy" style="position:static" data-copy="sudo llmgw server init --admin you@example.com" type="button">Copy</button>
|
||||
<button class="copy" style="position:static" data-copy="sudo llmgw server init --admin you@example.com" type="button">{{t "about.copy"}}</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<figure class="schem">
|
||||
<svg viewBox="0 0 440 250" role="img" aria-labelledby="schem-t">
|
||||
<title id="schem-t">Clients reach one llmgw address. llmgw places a model's shards on GPUs inside one fabric at a time.</title>
|
||||
<title id="schem-t">{{t "about.schem.title"}}</title>
|
||||
<!-- clients -->
|
||||
<g>
|
||||
<rect class="s-box" x="4" y="58" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="77" text-anchor="middle">openai sdk</text>
|
||||
<text class="s-label" x="50" y="77" text-anchor="middle" translate="no">openai sdk</text>
|
||||
<rect class="s-box" x="4" y="110" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="129" text-anchor="middle">editor agent</text>
|
||||
<text class="s-label" x="50" y="129" text-anchor="middle">{{t "about.schem.editor"}}</text>
|
||||
<rect class="s-box" x="4" y="162" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="181" text-anchor="middle">curl</text>
|
||||
<text class="s-label" x="50" y="181" text-anchor="middle" translate="no">curl</text>
|
||||
</g>
|
||||
<path id="c1" class="s-link" d="M96 73 C130 73 130 118 160 118"/>
|
||||
<path id="c2" class="s-link" d="M96 125 H160"/>
|
||||
@@ -234,7 +232,7 @@ footer a{color:var(--muted)}
|
||||
<path id="fb" class="s-link" d="M244 136 C272 136 272 188 300 188"/>
|
||||
<!-- fabric a -->
|
||||
<rect class="s-fab" x="300" y="14" width="136" height="98" rx="10"/>
|
||||
<text class="s-label" x="312" y="32">nvl72 · gb200</text>
|
||||
<text class="s-label" x="312" y="32" translate="no">nvl72 · gb200</text>
|
||||
<g transform="translate(312 42)">
|
||||
<rect class="s-gpu-a" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-a" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="84" width="22" height="22" rx="3"/>
|
||||
@@ -243,7 +241,7 @@ footer a{color:var(--muted)}
|
||||
</g>
|
||||
<!-- fabric b -->
|
||||
<rect class="s-fab" x="300" y="138" width="136" height="98" rx="10"/>
|
||||
<text class="s-label" x="312" y="156">ib-b · h100-80g</text>
|
||||
<text class="s-label" x="312" y="156" translate="no">ib-b · h100-80g</text>
|
||||
<g transform="translate(312 166)">
|
||||
<rect class="s-gpu-b" width="22" height="22" rx="3"/><rect class="s-gpu-b" x="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-idle" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" width="22" height="22" rx="3"/>
|
||||
@@ -257,50 +255,37 @@ footer a{color:var(--muted)}
|
||||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="1.2s" repeatCount="indefinite"><mpath href="#fa"/></animateMotion></circle>
|
||||
<circle class="pulse b" r="4"><animateMotion dur="2.4s" begin="2s" repeatCount="indefinite"><mpath href="#fb"/></animateMotion></circle>
|
||||
</svg>
|
||||
<figcaption>A replica spanning nodes goes into one NVLink domain if one has room, otherwise onto
|
||||
one InfiniBand fabric. Two fabrics that cannot see each other are never bridged: the all-reduce
|
||||
would hang.</figcaption>
|
||||
<figcaption>{{t "about.schem.caption"}}</figcaption>
|
||||
</figure>
|
||||
</div>
|
||||
|
||||
<section id="what">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>What it does</h2>
|
||||
<p class="intro">One static binary. It runs as the control plane, as the agent on every GPU
|
||||
node, and as the weights mirror.</p>
|
||||
<h2>{{t "about.what.title"}}</h2>
|
||||
<p class="intro">{{t "about.what.intro"}}</p>
|
||||
</div>
|
||||
<div class="facts">
|
||||
<div class="fact"><h3>OpenAI API, your hardware</h3>
|
||||
<p>Point any OpenAI client at it. Only the base URL changes. Streaming, tool calls,
|
||||
<code>/v1/models</code> and <code>/v1/responses</code> work as your client expects.</p></div>
|
||||
<div class="fact"><h3>Placement that knows the fabric</h3>
|
||||
<p>Agents report GPU class, memory, NVLink peers and InfiniBand ports. Replicas go where they
|
||||
fit, including ones larger than one machine, on the GPUs with the most NVLink pairs.</p></div>
|
||||
<div class="fact"><h3>Models load when asked for</h3>
|
||||
<p>Keep more models in the catalog than your GPUs hold. The first request starts one, the
|
||||
chat page shows it loading, and it stops again after the idle timeout.</p></div>
|
||||
<div class="fact"><h3>Keys, budgets, rate limits</h3>
|
||||
<p>Organisations, teams and projects, each with limits. Priority bands decide who waits
|
||||
when the cluster is full. Managers see their organisation's spend; people see their own.</p></div>
|
||||
<div class="fact"><h3>Editors connect from the account page</h3>
|
||||
<p>Pick Continue, Codex or Cline and a model. The page makes a key and hands over the config,
|
||||
filled in, or a one-line command that writes it.</p></div>
|
||||
<div class="fact"><h3>A chat page for quick questions</h3>
|
||||
<p>Markdown and maths rendered, stop mid-reply, tokens per second and time to first token,
|
||||
a system prompt and a temperature. The gateway keeps no conversations.</p></div>
|
||||
<div class="fact"><h3>A mirror you browse</h3>
|
||||
<p>Explore what is trending on Hugging Face, read a model card under its row, and download
|
||||
once for the whole cluster. Only what your nodes can run is shown by default.</p></div>
|
||||
<div class="fact"><h3>Nodes install their own engines</h3>
|
||||
<p>llama.cpp on every node; vLLM on every GPU node that can run it, from PyPI with every
|
||||
file hash-checked. A node that cannot run it says why.</p></div>
|
||||
<div class="fact"><h3>Upgrades that keep models running</h3>
|
||||
<p>Upgrade the control plane, then the agents. A restarted agent adopts the engines it was
|
||||
running instead of reloading them, so replicas stay up throughout.</p></div>
|
||||
<div class="fact"><h3>A dashboard with nothing to set up</h3>
|
||||
<p><code>/admin</code> shows capacity per GPU class, CPU and memory per node, who is using
|
||||
what, each node's page and logs. It fits a phone. Prometheus is optional.</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.api.title"}}</h3>
|
||||
<p>{{th "about.what.api.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.placement.title"}}</h3>
|
||||
<p>{{t "about.what.placement.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.ondemand.title"}}</h3>
|
||||
<p>{{t "about.what.ondemand.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.keys.title"}}</h3>
|
||||
<p>{{t "about.what.keys.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.editors.title"}}</h3>
|
||||
<p>{{t "about.what.editors.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.chat.title"}}</h3>
|
||||
<p>{{t "about.what.chat.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.mirror.title"}}</h3>
|
||||
<p>{{t "about.what.mirror.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.engines.title"}}</h3>
|
||||
<p>{{t "about.what.engines.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.upgrades.title"}}</h3>
|
||||
<p>{{t "about.what.upgrades.text"}}</p></div>
|
||||
<div class="fact"><h3>{{t "about.what.dashboard.title"}}</h3>
|
||||
<p>{{th "about.what.dashboard.text"}}</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -308,24 +293,19 @@ footer a{color:var(--muted)}
|
||||
<section id="fabrics">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>InfiniBand and NVLink, several of each</h2>
|
||||
<p class="intro">A site may have independent InfiniBand fabrics, eight-GPU HGX boards and NVL72
|
||||
racks side by side. llmgw tells them apart by what the adapters report, not by what a config
|
||||
file calls them.</p>
|
||||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/fabrics.md">Fabrics and NVLink guide</a></p>
|
||||
<h2>{{t "about.fabrics.title"}}</h2>
|
||||
<p class="intro">{{t "about.fabrics.intro"}}</p>
|
||||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/fabrics.md">{{t "about.fabrics.guide"}}</a></p>
|
||||
</div>
|
||||
<div class="fab-grid">
|
||||
<div class="order" aria-label="Where a replica spanning nodes is placed, in order">
|
||||
<div><b>1</b><h3>One NVLink domain</h3><p>NVL72 or NVLink Switch. Every card joined, IMEX up.
|
||||
Shards get <code>NCCL_MNNVL_ENABLE=1</code>.</p></div>
|
||||
<div><b>2</b><h3>One InfiniBand or RoCE fabric</h3><p>Nodes under one leaf first, nodes with a
|
||||
degraded cable last.</p></div>
|
||||
<div><b class="x">×</b><h3>Never across two</h3><p>Two fabrics given one name are placed as
|
||||
two, and the cluster page says so.</p></div>
|
||||
<div class="order" aria-label="{{t "about.fabrics.order_label"}}">
|
||||
<div><b>1</b><h3>{{t "about.fabrics.nvlink.title"}}</h3><p>{{th "about.fabrics.nvlink.text"}}</p></div>
|
||||
<div><b>2</b><h3>{{t "about.fabrics.ib.title"}}</h3><p>{{t "about.fabrics.ib.text"}}</p></div>
|
||||
<div><b class="x">×</b><h3>{{t "about.fabrics.never.title"}}</h3><p>{{t "about.fabrics.never.text"}}</p></div>
|
||||
</div>
|
||||
<div>
|
||||
<h3>Check a node in one command</h3>
|
||||
<div class="term" aria-label="Example output of llmgw doctor"><pre>$ sudo llmgw doctor
|
||||
<h3>{{t "about.fabrics.doctor"}}</h3>
|
||||
<div class="term" aria-label="{{t "about.fabrics.doctor_label"}}"><pre>$ sudo llmgw doctor
|
||||
<span class="fail">FAIL</span> nvlink this machine has NVSwitch but nvidia-fabricmanager is inactive
|
||||
<span class="dim">→ systemctl enable --now nvidia-fabricmanager</span>
|
||||
<span class="warn">WARN</span> fabric nvidia_peermem is not loaded: GPUDirect RDMA is off
|
||||
@@ -337,14 +317,10 @@ footer a{color:var(--muted)}
|
||||
<span class="ok">OK</span> agent NCCL bootstraps over ens3 (10.1.0.21)</pre></div>
|
||||
</div>
|
||||
<ul class="watch">
|
||||
<li><strong>Subnet managers, watched</strong>Master, priority, standbys and failovers per fabric.
|
||||
A fabric with no standby is flagged. llmgw does not run one.</li>
|
||||
<li><strong>A map of each fabric</strong><code>/admin/fabrics</code> draws switches in tiers,
|
||||
cables coloured by rate, degraded ones in red, and live traffic on your adapters.</li>
|
||||
<li><strong>NVLink domains as racks</strong><code>/admin/nvlink</code> shows which trays are
|
||||
joined, whose IMEX is down, which link runs degraded.</li>
|
||||
<li><strong>Ports re-read every heartbeat</strong>A port that goes down takes its fabric off the
|
||||
node within seconds. Counters and rates are exported to Prometheus.</li>
|
||||
<li><strong>{{t "about.fabrics.sm.title"}}</strong>{{t "about.fabrics.sm.text"}}</li>
|
||||
<li><strong>{{t "about.fabrics.map.title"}}</strong>{{th "about.fabrics.map.text"}}</li>
|
||||
<li><strong>{{t "about.fabrics.racks.title"}}</strong>{{th "about.fabrics.racks.text"}}</li>
|
||||
<li><strong>{{t "about.fabrics.ports.title"}}</strong>{{t "about.fabrics.ports.text"}}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
@@ -353,44 +329,38 @@ footer a{color:var(--muted)}
|
||||
<section id="install">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>From nothing to a first request</h2>
|
||||
<p class="intro">Debian, Ubuntu or RHEL 10. NVIDIA drivers and the InfiniBand stack are
|
||||
yours to install; llmgw does the rest.</p>
|
||||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/install.md">The full guide</a>
|
||||
covers the mirror, GPUs and what to do when something does not work.</p>
|
||||
<h2>{{t "about.install.title"}}</h2>
|
||||
<p class="intro">{{t "about.install.intro"}}</p>
|
||||
<p class="intro">{{th "about.install.guide"}}</p>
|
||||
</div>
|
||||
<ol class="steps">
|
||||
<li>
|
||||
<h3>Put the binary on every machine</h3>
|
||||
<p>Static, no dependencies, the same file everywhere.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}} <span class="c"># from the releases page</span>{{end}}
|
||||
<h3>{{t "about.install.binary.title"}}</h3>
|
||||
<p>{{t "about.install.binary.text"}}</p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}} <span class="c"># from the releases page</span>{{end}}
|
||||
U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
|
||||
curl -fsSLO $U/llmgw-$VERSION-linux-amd64
|
||||
curl -fsSL $U/checksums.txt | sha256sum -c --ignore-missing
|
||||
sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw</code></pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Set up the control plane</h3>
|
||||
<p>Creates the user, the certificate authority, the certificate, <code>server.yaml</code>
|
||||
and your administrator account, then starts the service. Safe to run again.
|
||||
Add <code>--host</code> for any name nodes use that the machine cannot see itself.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>sudo llmgw server init --admin you@example.com</code></pre></div>
|
||||
<h3>{{t "about.install.server.title"}}</h3>
|
||||
<p>{{th "about.install.server.text"}}</p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>sudo llmgw server init --admin you@example.com</code></pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Add a node</h3>
|
||||
<p>From the control plane, which holds the authority. The node gets its certificate,
|
||||
the binary and the agent, and installs its engine by itself.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>deploy/bootstrap/onboard-node.sh --node gpu01 --addr 10.0.0.141 \
|
||||
<h3>{{t "about.install.node.title"}}</h3>
|
||||
<p>{{t "about.install.node.text"}}</p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>deploy/bootstrap/onboard-node.sh --node gpu01 --addr 10.0.0.141 \
|
||||
--server https://10.0.0.140:7443 --fabric ib-a:mlx5_0,mlx5_1</code></pre></div>
|
||||
<div class="term" aria-label="Example output of llmgw ctl nodes"><pre>$ llmgw ctl nodes
|
||||
<div class="term" aria-label="{{t "about.install.node.output_label"}}"><pre>$ llmgw ctl nodes
|
||||
NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
|
||||
gpu01 <span class="ok">ready</span> 8/8 free 503G h100-80g ib-a 0 2s</pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Add a model and send a request</h3>
|
||||
<p>No restart. If it does not place, <code>llmgw ctl explain-placement</code> names every
|
||||
node and why it was turned down.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>llmgw ctl model add -f qwen-coder-3b.yaml
|
||||
<h3>{{t "about.install.model.title"}}</h3>
|
||||
<p>{{th "about.install.model.text"}}</p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>llmgw ctl model add -f qwen-coder-3b.yaml
|
||||
|
||||
curl https://llm.example.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
@@ -404,35 +374,32 @@ curl https://llm.example.com/v1/chat/completions \
|
||||
<section id="proxy">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Behind a reverse proxy</h2>
|
||||
<p class="intro">Port 8080 is plain HTTP. Put it behind Caddy or nginx with a public
|
||||
certificate. Only 8080 goes through the proxy: agents reach 7443 directly, because it is
|
||||
mutual TLS with the cluster's own authority.</p>
|
||||
<p class="intro">Bind llmgw to loopback when the proxy is on the same machine:
|
||||
<code>listen: "127.0.0.1:8080"</code>.</p>
|
||||
<h2>{{t "about.proxy.title"}}</h2>
|
||||
<p class="intro">{{t "about.proxy.intro"}}</p>
|
||||
<p class="intro">{{th "about.proxy.loopback"}}</p>
|
||||
</div>
|
||||
<div class="tabs">
|
||||
<input type="radio" name="proxy" id="t-caddy" checked>
|
||||
<input type="radio" name="proxy" id="t-nginx">
|
||||
<div class="tablist" role="presentation">
|
||||
<div class="tablist" role="presentation" translate="no">
|
||||
<label for="t-caddy">Caddy</label>
|
||||
<label for="t-nginx">nginx</label>
|
||||
</div>
|
||||
<div class="panel p-caddy">
|
||||
<p class="meta"><span>Obtains and renews its own certificate. Replace <code>llm.example.com</code>.</span>
|
||||
<a href="{{.Download}}/Caddyfile">Caddyfile</a></p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Caddy}}</code></pre></div>
|
||||
<p class="meta"><span>{{th "about.proxy.caddy"}}</span>
|
||||
<a href="{{.Download}}/Caddyfile" translate="no">Caddyfile</a></p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>{{.Caddy}}</code></pre></div>
|
||||
</div>
|
||||
<div class="panel p-nginx">
|
||||
<p class="meta"><span>Certificate from <code>certbot --nginx</code>. Works on nginx 1.18 and later.</span>
|
||||
<a href="{{.Download}}/nginx.conf">nginx.conf</a></p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Nginx}}</code></pre></div>
|
||||
<p class="meta"><span>{{th "about.proxy.nginx"}}</span>
|
||||
<a href="{{.Download}}/nginx.conf" translate="no">nginx.conf</a></p>
|
||||
<div class="code"><button class="copy" type="button">{{t "about.copy"}}</button><pre><code>{{.Nginx}}</code></pre></div>
|
||||
</div>
|
||||
<ul class="rules">
|
||||
<li><strong>Streams are not buffered</strong>Tokens reach the client as they are generated, not in one burst at the end.</li>
|
||||
<li><strong>The read timeout outlasts the model</strong>nginx's 60-second default turns a slow first token into a 504.</li>
|
||||
<li><strong>Forwarded headers are set, not passed on</strong>llmgw marks session cookies Secure from <code>X-Forwarded-Proto</code> and shows <code>X-Forwarded-Host</code> as the base URL.</li>
|
||||
<li><strong>Body limit matches the router</strong>32 MiB, so an oversized request is refused before it is uploaded.</li>
|
||||
<li><strong>{{t "about.proxy.stream.title"}}</strong>{{t "about.proxy.stream.text"}}</li>
|
||||
<li><strong>{{t "about.proxy.timeout.title"}}</strong>{{t "about.proxy.timeout.text"}}</li>
|
||||
<li><strong>{{t "about.proxy.headers.title"}}</strong>{{th "about.proxy.headers.text"}}</li>
|
||||
<li><strong>{{t "about.proxy.body.title"}}</strong>{{t "about.proxy.body.text"}}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
@@ -441,33 +408,29 @@ curl https://llm.example.com/v1/chat/completions \
|
||||
<section id="download">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Download</h2>
|
||||
<p class="intro">Built by CI from the tag and checked against <code>checksums.txt</code>.
|
||||
Every release is on the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>
|
||||
with its changelog.</p>
|
||||
<h2>{{t "about.download.title"}}</h2>
|
||||
<p class="intro">{{th "about.download.intro"}}</p>
|
||||
</div>
|
||||
<div>
|
||||
{{if .Release}}
|
||||
<div class="ver"><span class="tag">{{.Release}}</span>{{with .Date}}<span class="eyebrow">Released {{.}}</span>{{end}}</div>
|
||||
<div class="ver"><span class="tag">{{.Release}}</span>{{with .Date}}<span class="eyebrow">{{t "about.download.released" .}}</span>{{end}}</div>
|
||||
<div class="dl-wrap">
|
||||
<table class="dl">
|
||||
<thead><tr><th>For</th><th>File</th><th>Notes</th></tr></thead>
|
||||
<thead><tr><th>{{t "about.download.col.for"}}</th><th>{{t "about.download.col.file"}}</th><th>{{t "about.download.col.notes"}}</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>Linux x86-64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64"><code>llmgw-{{$.Release}}-linux-amd64</code></a></td><td class="note">Static. Runs on any libc.</td></tr>
|
||||
<tr><td>Linux ARM64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-arm64"><code>llmgw-{{$.Release}}-linux-arm64</code></a></td><td class="note">Static.</td></tr>
|
||||
<tr><td>Linux x86-64, NVLink</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64-nvml"><code>llmgw-{{$.Release}}-linux-amd64-nvml</code></a></td><td class="note">Sees NVLink topology through NVML. For PCIe boxes with NVLink bridges. Needs the driver.</td></tr>
|
||||
<tr><td>macOS Apple silicon</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-arm64"><code>llmgw-{{$.Release}}-darwin-arm64</code></a></td><td class="note">For <code>llmgw ctl</code> and trying it on a laptop.</td></tr>
|
||||
<tr><td>macOS Intel</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-amd64"><code>llmgw-{{$.Release}}-darwin-amd64</code></a></td><td class="note"></td></tr>
|
||||
<tr><td>Caddy</td><td><a href="{{$.Download}}/Caddyfile"><code>Caddyfile</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||||
<tr><td>nginx</td><td><a href="{{$.Download}}/nginx.conf"><code>nginx.conf</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||||
<tr><td>Checksums</td><td><a href="{{$.Download}}/checksums.txt"><code>checksums.txt</code></a></td><td class="note">SHA-256 of every file above.</td></tr>
|
||||
<tr><td translate="no">Linux x86-64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64"><code>llmgw-{{$.Release}}-linux-amd64</code></a></td><td class="note">{{t "about.download.static_any"}}</td></tr>
|
||||
<tr><td translate="no">Linux ARM64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-arm64"><code>llmgw-{{$.Release}}-linux-arm64</code></a></td><td class="note">{{t "about.download.static"}}</td></tr>
|
||||
<tr><td translate="no">Linux x86-64, NVLink</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64-nvml"><code>llmgw-{{$.Release}}-linux-amd64-nvml</code></a></td><td class="note">{{t "about.download.nvml"}}</td></tr>
|
||||
<tr><td>{{t "about.download.mac_arm"}}</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-arm64"><code>llmgw-{{$.Release}}-darwin-arm64</code></a></td><td class="note">{{th "about.download.mac_note"}}</td></tr>
|
||||
<tr><td translate="no">macOS Intel</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-amd64"><code>llmgw-{{$.Release}}-darwin-amd64</code></a></td><td class="note"></td></tr>
|
||||
<tr><td translate="no">Caddy</td><td><a href="{{$.Download}}/Caddyfile"><code>Caddyfile</code></a></td><td class="note">{{t "about.download.proxy_note"}}</td></tr>
|
||||
<tr><td translate="no">nginx</td><td><a href="{{$.Download}}/nginx.conf"><code>nginx.conf</code></a></td><td class="note">{{t "about.download.proxy_note"}}</td></tr>
|
||||
<tr><td>{{t "about.download.checksums"}}</td><td><a href="{{$.Download}}/checksums.txt"><code>checksums.txt</code></a></td><td class="note">{{t "about.download.checksums_note"}}</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}
|
||||
<p class="intro" style="margin-top:0">This is a development build ({{.Version}}). Take a release
|
||||
from the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>, where every
|
||||
file is listed with its checksum.</p>
|
||||
<p class="intro" style="margin-top:0">{{th "about.download.dev" .Version}}</p>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
@@ -476,30 +439,27 @@ curl https://llm.example.com/v1/chat/completions \
|
||||
<section id="status">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Where it stands</h2>
|
||||
<p class="intro">Pre-1.0: configuration formats may change between minor versions.
|
||||
This is what has run on real hardware, and what has not yet.</p>
|
||||
<h2>{{t "about.status.title"}}</h2>
|
||||
<p class="intro">{{t "about.status.intro"}}</p>
|
||||
</div>
|
||||
<div class="status">
|
||||
<div class="proven">
|
||||
<h3>Run on real clusters</h3>
|
||||
<h3>{{t "about.status.proven"}}</h3>
|
||||
<ul>
|
||||
<li>Single-node replicas on real GPUs and on CPU</li>
|
||||
<li>On-demand loading and idle shutdown</li>
|
||||
<li>Keys, budgets, rate limits and priority bands</li>
|
||||
<li>Rolling upgrades with no dropped replicas</li>
|
||||
<li>SQLite and PostgreSQL stores</li>
|
||||
<li>Chat, editor setup and the Responses API against a six-node test cluster</li>
|
||||
<li>{{t "about.status.proven.single"}}</li>
|
||||
<li>{{t "about.status.proven.ondemand"}}</li>
|
||||
<li>{{t "about.status.proven.keys"}}</li>
|
||||
<li>{{t "about.status.proven.upgrades"}}</li>
|
||||
<li>{{t "about.status.proven.stores"}}</li>
|
||||
<li>{{t "about.status.proven.chat"}}</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="open">
|
||||
<h3>Not yet proven</h3>
|
||||
<h3>{{t "about.status.open"}}</h3>
|
||||
<ul>
|
||||
<li>InfiniBand and NVLink on a real site: written and tested against the real output of
|
||||
<code>sminfo</code>, <code>ibnetdiscover</code> and <code>nvidia-smi</code>, but not yet run
|
||||
on the hardware. Run <code>llmgw doctor --json</code> on a few nodes first.</li>
|
||||
<li>Two control planes over long periods</li>
|
||||
<li>Nodes do not share downloaded weights; the mirror keeps each download local</li>
|
||||
<li>{{th "about.status.open.fabrics"}}</li>
|
||||
<li>{{t "about.status.open.planes"}}</li>
|
||||
<li>{{t "about.status.open.weights"}}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
@@ -508,25 +468,26 @@ curl https://llm.example.com/v1/chat/completions \
|
||||
|
||||
<footer>
|
||||
<span>llmgw {{.Version}}</span>
|
||||
<span><a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/CHANGELOG.md">Changelog</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/reference.md">Configuration reference</a></span>
|
||||
<span><a href="https://gitea.mynodes.xyz/ilker/llmgw">{{t "about.nav.source"}}</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/CHANGELOG.md">{{t "about.footer.changelog"}}</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/reference.md">{{t "about.footer.reference"}}</a></span>{{if not .Site}}{{template "langs"}}{{end}}
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var say = { copy: {{t "about.copy"}}, copied: {{t "about.copied"}}, selected: {{t "about.selected"}} };
|
||||
document.querySelectorAll('.copy').forEach(function (b) {
|
||||
b.addEventListener('click', function () {
|
||||
var text = b.dataset.copy || b.parentElement.querySelector('pre').innerText;
|
||||
var done = function (label) { b.textContent = label; setTimeout(function () { b.textContent = 'Copy'; }, 1600); };
|
||||
var done = function (label) { b.textContent = label; setTimeout(function () { b.textContent = say.copy; }, 1600); };
|
||||
if (navigator.clipboard && navigator.clipboard.writeText) {
|
||||
navigator.clipboard.writeText(text).then(function () { done('Copied'); }, function () { select(); });
|
||||
navigator.clipboard.writeText(text).then(function () { done(say.copied); }, function () { select(); });
|
||||
} else { select(); }
|
||||
function select() {
|
||||
var el = b.dataset.copy ? b.parentElement.querySelector('code') : b.parentElement.querySelector('pre');
|
||||
var r = document.createRange(); r.selectNodeContents(el);
|
||||
var s = window.getSelection(); s.removeAllRanges(); s.addRange(r);
|
||||
done('Selected');
|
||||
done(say.selected);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user