install.md explains every choice and is long for someone who wants the commands. quick-install.md has only those, and what to check after each: the binary, server init, onboarding GPU nodes (with doctor first and the script fetched from the release tag, since it is not installed anywhere), a model from the mirror's explorer, keys and editors, and Caddy. Every button and path it names was checked against the pages. quickstart.md is now said to be the from-source, one-workstation path it is; README and the about page point at the new guide. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
537 lines
32 KiB
HTML
537 lines
32 KiB
HTML
{{define "about"}}{{if not .Site}}<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<meta name="description" content="llmgw serves models on your own GPUs behind one OpenAI-compatible API.">
|
||
{{end}}<title>llmgw</title>
|
||
{{if .Site}}<link rel="preconnect" href="https://fonts.googleapis.com">
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wdth,wght@112,600;125,800&family=IBM+Plex+Sans:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500&display=swap">
|
||
{{end}}<style>
|
||
{{if not .Site}}@font-face{font-family:"Archivo";font-weight:600 800;font-stretch:112% 125%;font-display:swap;src:url("{{asset "fonts/archivo-latin.woff2"}}") format("woff2")}
|
||
@font-face{font-family:"IBM Plex Sans";font-weight:100 700;font-display:swap;src:url("{{asset "fonts/plex-sans-latin.woff2"}}") format("woff2")}
|
||
@font-face{font-family:"IBM Plex Mono";font-weight:400;font-display:swap;src:url("{{asset "fonts/plex-mono-400-latin.woff2"}}") format("woff2")}
|
||
@font-face{font-family:"IBM Plex Mono";font-weight:500;font-display:swap;src:url("{{asset "fonts/plex-mono-500-latin.woff2"}}") format("woff2")}
|
||
{{end}}:root{
|
||
--bg:#F3F5F5; --surface:#FFFFFF; --ink:#142023; --muted:#56696C; --line:#D3DCDC;
|
||
--fiber:#0B7F7D; --fiber-soft:#DDEFEE; --om1:#C4621A; --om1-soft:#F6E6D9;
|
||
--code-bg:#15201F; --code-ink:#DCEAE8; --code-dim:#7F9A97;
|
||
--display:"Archivo","Arial Narrow",system-ui,sans-serif;
|
||
--body:"IBM Plex Sans",system-ui,-apple-system,"Segoe UI",sans-serif;
|
||
--mono:"IBM Plex Mono",ui-monospace,"SFMono-Regular",Menlo,monospace;
|
||
}
|
||
@media (prefers-color-scheme:dark){
|
||
:root:not([data-theme="light"]){
|
||
--bg:#0E1617; --surface:#152022; --ink:#E2ECEB; --muted:#8FA5A4; --line:#26383A;
|
||
--fiber:#3FC2BC; --fiber-soft:#133634; --om1:#EE9350; --om1-soft:#3A2717;
|
||
--code-bg:#0A1011; --code-ink:#D5E5E3; --code-dim:#6E8886;
|
||
color-scheme:dark;
|
||
}
|
||
}
|
||
:root[data-theme="dark"]{
|
||
--bg:#0E1617; --surface:#152022; --ink:#E2ECEB; --muted:#8FA5A4; --line:#26383A;
|
||
--fiber:#3FC2BC; --fiber-soft:#133634; --om1:#EE9350; --om1-soft:#3A2717;
|
||
--code-bg:#0A1011; --code-ink:#D5E5E3; --code-dim:#6E8886;
|
||
color-scheme:dark;
|
||
}
|
||
*{box-sizing:border-box}
|
||
body{background:var(--bg);color:var(--ink);font:400 16px/1.6 var(--body);margin:0;padding-inline:20px;padding-block:0}
|
||
.wrap{max-width:1080px;margin:0 auto}
|
||
a{color:var(--fiber);text-underline-offset:3px}
|
||
a:focus-visible,button:focus-visible,input:focus-visible+label{outline:2px solid var(--fiber);outline-offset:3px;border-radius:4px}
|
||
h1,h2,h3{font-family:var(--display);text-wrap:balance;margin:0}
|
||
h2{font-size:clamp(1.6rem,3.2vw,2.1rem);font-variation-settings:"wdth" 112;font-weight:600;letter-spacing:-.01em}
|
||
h3{font-size:1.05rem;font-variation-settings:"wdth" 112;font-weight:600}
|
||
p{margin:0}
|
||
code,pre{font-family:var(--mono);font-size:.875rem}
|
||
:not(pre)>code{background:var(--fiber-soft);padding:.1em .35em;border-radius:4px;font-size:.85em}
|
||
.eyebrow{font:500 .75rem/1 var(--mono);letter-spacing:.12em;text-transform:uppercase;color:var(--muted)}
|
||
|
||
/* top bar */
|
||
.bar{display:flex;align-items:center;justify-content:space-between;gap:16px;padding-block:22px;border-bottom:1px solid var(--line)}
|
||
.mark{font:800 1.25rem/1 var(--display);font-variation-settings:"wdth" 125;letter-spacing:-.02em;color:var(--ink);text-decoration:none;display:flex;align-items:center;gap:10px}
|
||
.mark svg{flex:none}
|
||
.bar nav{display:flex;gap:20px;flex-wrap:wrap;font-size:.9rem}
|
||
.bar nav a{color:var(--muted);text-decoration:none}
|
||
.bar nav a:hover{color:var(--ink)}
|
||
|
||
/* hero */
|
||
.hero{display:grid;grid-template-columns:1.05fr .95fr;gap:48px;align-items:center;padding-block:64px 56px}
|
||
.hero h1{font-size:clamp(2.5rem,6vw,4.4rem);line-height:.98;font-weight:800;font-variation-settings:"wdth" 125;letter-spacing:-.03em}
|
||
.hero h1 em{font-style:normal;color:var(--fiber)}
|
||
.lede{margin-top:22px;font-size:1.12rem;color:var(--muted);max-width:34em}
|
||
.cta{display:flex;flex-wrap:wrap;gap:12px;margin-top:30px}
|
||
.btn{display:inline-flex;align-items:center;gap:8px;font:600 .95rem/1 var(--body);padding:13px 18px;border-radius:8px;text-decoration:none;border:1px solid var(--fiber);background:var(--fiber);color:var(--bg)}
|
||
.btn.ghost{background:transparent;color:var(--ink);border-color:var(--line)}
|
||
.btn.ghost:hover{border-color:var(--ink)}
|
||
.oneline{margin-top:26px;display:flex;align-items:center;gap:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;padding:12px 12px 12px 16px;max-width:100%}
|
||
.oneline code{flex:1;min-width:0;overflow-x:auto;white-space:nowrap;background:none;padding:0;font-size:.9rem;color:var(--code-ink)}
|
||
.oneline .p{color:var(--code-dim);user-select:none}
|
||
|
||
/* schematic */
|
||
figure{margin:0}
|
||
.schem{background:var(--surface);border:1px solid var(--line);border-radius:12px;padding:18px}
|
||
.schem svg{display:block;width:100%;height:auto}
|
||
.schem figcaption{margin-top:12px;font-size:.85rem;color:var(--muted)}
|
||
.s-label{font:500 11px var(--mono);fill:var(--muted)}
|
||
.s-strong{font:600 13px var(--body);fill:var(--ink)}
|
||
.s-box{fill:var(--bg);stroke:var(--line)}
|
||
.s-gw{fill:var(--fiber-soft);stroke:var(--fiber);stroke-width:1.5}
|
||
.s-fab{fill:none;stroke:var(--line);stroke-dasharray:5 4}
|
||
.s-link{fill:none;stroke:var(--line);stroke-width:2}
|
||
.s-gpu-a{fill:var(--fiber)} .s-gpu-b{fill:var(--om1)}
|
||
.s-gpu-idle{fill:var(--line)}
|
||
.pulse{fill:var(--fiber)}
|
||
.pulse.b{fill:var(--om1)}
|
||
@media (prefers-reduced-motion:reduce){.pulse{display:none}}
|
||
|
||
/* sections */
|
||
section{padding-block:56px;border-top:1px solid var(--line)}
|
||
.sec-head{display:grid;grid-template-columns:260px 1fr;gap:40px}
|
||
.sec-head .intro{color:var(--muted);margin-top:12px;font-size:.95rem}
|
||
|
||
.facts{display:grid;grid-template-columns:1fr 1fr;gap:28px 40px}
|
||
.fact h3{display:flex;gap:10px;align-items:baseline}
|
||
.fact h3::before{content:"";width:10px;height:10px;border-radius:2px;background:var(--fiber);flex:none;transform:translateY(-1px)}
|
||
.fact p{margin-top:6px;color:var(--muted);font-size:.95rem;max-width:36em}
|
||
|
||
/* steps */
|
||
.steps{list-style:none;margin:0;padding:0;display:grid;gap:28px;counter-reset:s}
|
||
.steps li{display:grid;grid-template-columns:44px 1fr;gap:4px 16px;counter-increment:s}
|
||
.steps li>*{grid-column:2}
|
||
.steps li::before{content:counter(s);grid-column:1;grid-row:span 4;font:800 1.5rem/1.1 var(--display);font-variation-settings:"wdth" 125;color:var(--fiber)}
|
||
.steps p{color:var(--muted);font-size:.95rem;max-width:40em}
|
||
.code{position:relative;margin-top:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;min-width:0}
|
||
.code pre{margin:0;padding:16px 18px;overflow-x:auto;line-height:1.55}
|
||
.code .c{color:var(--code-dim)}
|
||
.copy{position:absolute;top:8px;right:8px;font:500 .75rem/1 var(--body);padding:7px 10px;border-radius:6px;border:1px solid #2d4442;background:#1c2b2a;color:var(--code-ink);cursor:pointer}
|
||
.copy:hover{border-color:var(--code-dim)}
|
||
|
||
/* nodes table in step */
|
||
.term{margin-top:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;overflow-x:auto}
|
||
.term pre{margin:0;padding:14px 18px;font-size:.8rem;line-height:1.6}
|
||
.term .ok{color:#6FD3A6}
|
||
|
||
/* proxy tabs */
|
||
.tabs{min-width:0}
|
||
.tabs input{position:absolute;opacity:0;pointer-events:none}
|
||
.tablist{display:flex;gap:4px;border-bottom:1px solid var(--line)}
|
||
.tablist label{cursor:pointer;padding:10px 16px;font:600 .9rem/1 var(--body);color:var(--muted);border-bottom:2px solid transparent;margin-bottom:-1px}
|
||
#t-caddy:checked~.tablist label[for="t-caddy"],#t-nginx:checked~.tablist label[for="t-nginx"]{color:var(--ink);border-color:var(--fiber)}
|
||
.panel{display:none;margin-top:16px}
|
||
#t-caddy:checked~.p-caddy,#t-nginx:checked~.p-nginx{display:block}
|
||
.panel .meta{display:flex;flex-wrap:wrap;justify-content:space-between;gap:8px 16px;font-size:.88rem;color:var(--muted)}
|
||
.panel .code pre{max-height:460px}
|
||
.rules{margin-top:28px;display:grid;grid-template-columns:1fr 1fr;gap:18px 32px;padding:0;list-style:none}
|
||
.rules li{font-size:.92rem;color:var(--muted);padding-left:16px;border-left:2px solid var(--om1)}
|
||
.rules strong{color:var(--ink);font-weight:600;display:block}
|
||
|
||
/* downloads */
|
||
.dl{width:100%;border-collapse:collapse;font-size:.92rem}
|
||
.dl-wrap{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--surface)}
|
||
.dl th{text-align:left;font:500 .72rem/1 var(--mono);letter-spacing:.1em;text-transform:uppercase;color:var(--muted);padding:14px 16px;border-bottom:1px solid var(--line)}
|
||
.dl td{padding:12px 16px;border-bottom:1px solid var(--line);vertical-align:top}
|
||
.dl tr:last-child td{border-bottom:0}
|
||
.dl td:first-child{white-space:nowrap;font-weight:500}
|
||
.dl code{background:none;padding:0;font-size:.85rem}
|
||
.dl .note{color:var(--muted)}
|
||
.ver{display:flex;flex-wrap:wrap;gap:10px 24px;align-items:baseline;margin-bottom:18px}
|
||
.tag{font:500 .8rem/1 var(--mono);padding:6px 9px;border-radius:5px;background:var(--fiber-soft);color:var(--fiber)}
|
||
|
||
/* status */
|
||
.status{display:grid;grid-template-columns:1fr 1fr;gap:32px}
|
||
.status h3{margin-bottom:10px}
|
||
.status ul{margin:0;padding:0;list-style:none;display:grid;gap:10px}
|
||
.status li{font-size:.93rem;color:var(--muted);display:grid;grid-template-columns:18px 1fr;gap:8px}
|
||
.status li::before{font:600 .9rem/1.6 var(--mono)}
|
||
.proven li::before{content:"✓";color:var(--fiber)}
|
||
.open li::before{content:"○";color:var(--om1)}
|
||
|
||
.next{font:500 .66rem/1 var(--mono);letter-spacing:.08em;text-transform:uppercase;padding:4px 6px;border-radius:4px;background:var(--om1-soft);color:var(--om1);white-space:nowrap;align-self:center}
|
||
.fab-grid{display:grid;grid-template-columns:1fr;gap:28px;min-width:0}
|
||
.term .warn{color:#E9B35A} .term .fail{color:#F07C6C} .term .dim{color:var(--code-dim)}
|
||
.watch{display:grid;grid-template-columns:1fr 1fr;gap:18px 32px;padding:0;margin:0;list-style:none}
|
||
.watch li{font-size:.92rem;color:var(--muted);padding-left:16px;border-left:2px solid var(--fiber)}
|
||
.watch strong{color:var(--ink);font-weight:600;display:block}
|
||
.watch code{font-size:.8em}
|
||
.order{display:grid;grid-template-columns:repeat(3,1fr);gap:0;border:1px solid var(--line);border-radius:10px;background:var(--surface);overflow:hidden}
|
||
.order div{padding:16px 18px;border-right:1px solid var(--line)}
|
||
.order div:last-child{border-right:0}
|
||
.order b{display:block;font:800 1.4rem/1 var(--display);font-variation-settings:"wdth" 125;color:var(--fiber);margin-bottom:8px}
|
||
.order b.x{color:var(--om1)}
|
||
.order h3{font-size:.95rem}
|
||
.order p{font-size:.88rem;color:var(--muted);margin-top:4px}
|
||
footer{border-top:1px solid var(--line);padding-block:28px 40px;display:flex;flex-wrap:wrap;justify-content:space-between;gap:12px;font-size:.85rem;color:var(--muted)}
|
||
footer a{color:var(--muted)}
|
||
|
||
@media (max-width:860px){
|
||
.hero{grid-template-columns:1fr;gap:36px;padding-block:44px 40px}
|
||
.sec-head{grid-template-columns:1fr;gap:20px}
|
||
.facts,.rules,.status,.watch,.order{grid-template-columns:1fr}
|
||
.order div{border-right:0;border-bottom:1px solid var(--line)}
|
||
.order div:last-child{border-bottom:0}
|
||
.bar nav{display:none}
|
||
}
|
||
</style>
|
||
{{if not .Site}}</head><body>{{end}}
|
||
|
||
<div class="wrap">
|
||
<header class="bar">
|
||
<a class="mark" href="#top" aria-label="llmgw home">
|
||
<svg width="26" height="26" viewBox="0 0 26 26" aria-hidden="true">
|
||
<rect x="1" y="1" width="24" height="24" rx="6" fill="none" stroke="currentColor" stroke-width="2"/>
|
||
<path d="M7 9h12M7 13h12M7 17h12" stroke="var(--fiber)" stroke-width="2" stroke-linecap="round"/>
|
||
</svg>
|
||
llmgw
|
||
</a>
|
||
<nav aria-label="Sections">
|
||
<a href="#what">What it does</a>
|
||
<a href="#fabrics">Fabrics</a>
|
||
<a href="#install">Install</a>
|
||
<a href="#proxy">Reverse proxy</a>
|
||
<a href="#download">Download</a>
|
||
<a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a>
|
||
{{if not .Site}}<a href="/">This gateway</a>{{end}}
|
||
</nav>
|
||
</header>
|
||
|
||
<div class="hero" id="top">
|
||
<div>
|
||
<p class="eyebrow">Self-hosted LLM gateway · {{.Version}}</p>
|
||
<h1 style="margin-top:14px">One endpoint in front of <em>your own GPUs</em>.</h1>
|
||
<p class="lede">llmgw runs vLLM, SGLang and llama.cpp across your servers and serves them
|
||
behind one OpenAI-compatible API. It knows which GPUs share an NVLink domain or an InfiniBand
|
||
fabric, and a model split across machines never straddles two.</p>
|
||
<div class="cta">
|
||
<a class="btn" href="#download">Download{{with .Release}} {{.}}{{end}}</a>
|
||
<a class="btn ghost" href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/quick-install.md">Install guide</a>
|
||
</div>
|
||
<div class="oneline" aria-label="Set up a control plane">
|
||
<code><span class="p">$ </span>sudo llmgw server init --admin you@example.com</code>
|
||
<button class="copy" style="position:static" data-copy="sudo llmgw server init --admin you@example.com" type="button">Copy</button>
|
||
</div>
|
||
</div>
|
||
|
||
<figure class="schem">
|
||
<svg viewBox="0 0 440 250" role="img" aria-labelledby="schem-t">
|
||
<title id="schem-t">Clients reach one llmgw address. llmgw places a model's shards on GPUs inside one fabric at a time.</title>
|
||
<!-- clients -->
|
||
<g>
|
||
<rect class="s-box" x="4" y="58" width="92" height="30" rx="6"/>
|
||
<text class="s-label" x="50" y="77" text-anchor="middle">openai sdk</text>
|
||
<rect class="s-box" x="4" y="110" width="92" height="30" rx="6"/>
|
||
<text class="s-label" x="50" y="129" text-anchor="middle">editor agent</text>
|
||
<rect class="s-box" x="4" y="162" width="92" height="30" rx="6"/>
|
||
<text class="s-label" x="50" y="181" text-anchor="middle">curl</text>
|
||
</g>
|
||
<path id="c1" class="s-link" d="M96 73 C130 73 130 118 160 118"/>
|
||
<path id="c2" class="s-link" d="M96 125 H160"/>
|
||
<path id="c3" class="s-link" d="M96 177 C130 177 130 132 160 132"/>
|
||
<!-- gateway -->
|
||
<rect class="s-gw" x="160" y="96" width="84" height="58" rx="8"/>
|
||
<text class="s-strong" x="202" y="122" text-anchor="middle">llmgw</text>
|
||
<text class="s-label" x="202" y="139" text-anchor="middle">:8080</text>
|
||
<path id="fa" class="s-link" d="M244 114 C272 114 272 62 300 62"/>
|
||
<path id="fb" class="s-link" d="M244 136 C272 136 272 188 300 188"/>
|
||
<!-- fabric a -->
|
||
<rect class="s-fab" x="300" y="14" width="136" height="98" rx="10"/>
|
||
<text class="s-label" x="312" y="32">nvl72 · gb200</text>
|
||
<g transform="translate(312 42)">
|
||
<rect class="s-gpu-a" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="28" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-a" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="84" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-a" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="28" y="28" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-idle" x="56" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" y="28" width="22" height="22" rx="3"/>
|
||
</g>
|
||
<!-- fabric b -->
|
||
<rect class="s-fab" x="300" y="138" width="136" height="98" rx="10"/>
|
||
<text class="s-label" x="312" y="156">ib-b · h100-80g</text>
|
||
<g transform="translate(312 166)">
|
||
<rect class="s-gpu-b" width="22" height="22" rx="3"/><rect class="s-gpu-b" x="28" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-idle" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-idle" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="28" y="28" width="22" height="22" rx="3"/>
|
||
<rect class="s-gpu-idle" x="56" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" y="28" width="22" height="22" rx="3"/>
|
||
</g>
|
||
<!-- requests in flight -->
|
||
<circle class="pulse" r="4"><animateMotion dur="2.4s" repeatCount="indefinite"><mpath href="#c1"/></animateMotion></circle>
|
||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="0.8s" repeatCount="indefinite"><mpath href="#c2"/></animateMotion></circle>
|
||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="1.6s" repeatCount="indefinite"><mpath href="#c3"/></animateMotion></circle>
|
||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="1.2s" repeatCount="indefinite"><mpath href="#fa"/></animateMotion></circle>
|
||
<circle class="pulse b" r="4"><animateMotion dur="2.4s" begin="2s" repeatCount="indefinite"><mpath href="#fb"/></animateMotion></circle>
|
||
</svg>
|
||
<figcaption>A replica spanning nodes goes into one NVLink domain if one has room, otherwise onto
|
||
one InfiniBand fabric. Two fabrics that cannot see each other are never bridged: the all-reduce
|
||
would hang.</figcaption>
|
||
</figure>
|
||
</div>
|
||
|
||
<section id="what">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>What it does</h2>
|
||
<p class="intro">One static binary. It runs as the control plane, as the agent on every GPU
|
||
node, and as the weights mirror.</p>
|
||
</div>
|
||
<div class="facts">
|
||
<div class="fact"><h3>OpenAI API, your hardware</h3>
|
||
<p>Point any OpenAI client at it. Only the base URL changes. Streaming, tool calls,
|
||
<code>/v1/models</code> and <code>/v1/responses</code> work as your client expects.</p></div>
|
||
<div class="fact"><h3>Placement that knows the fabric</h3>
|
||
<p>Agents report GPU class, memory, NVLink peers and InfiniBand ports. Replicas go where they
|
||
fit, including ones larger than one machine, on the GPUs with the most NVLink pairs.</p></div>
|
||
<div class="fact"><h3>Models load when asked for</h3>
|
||
<p>Keep more models in the catalog than your GPUs hold. The first request starts one, the
|
||
chat page shows it loading, and it stops again after the idle timeout.</p></div>
|
||
<div class="fact"><h3>Keys, budgets, rate limits</h3>
|
||
<p>Organisations, teams and projects, each with limits. Priority bands decide who waits
|
||
when the cluster is full. Managers see their organisation's spend; people see their own.</p></div>
|
||
<div class="fact"><h3>Editors connect from the account page {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||
<p>Pick Continue, Codex or Cline and a model. The page makes a key and hands over the config,
|
||
filled in, or a one-line command that writes it.</p></div>
|
||
<div class="fact"><h3>A chat page for quick questions {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||
<p>Markdown and maths rendered, stop mid-reply, tokens per second and time to first token,
|
||
a system prompt and a temperature. The gateway keeps no conversations.</p></div>
|
||
<div class="fact"><h3>A mirror you browse {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||
<p>Explore what is trending on Hugging Face, read a model card under its row, and download
|
||
once for the whole cluster. Only what your nodes can run is shown by default.</p></div>
|
||
<div class="fact"><h3>Nodes install their own engines{{if .Site}} <span class="next">next release</span>{{end}}</h3>
|
||
<p>llama.cpp on every node; vLLM on every GPU node that can run it, from PyPI with every
|
||
file hash-checked. A node that cannot run it says why.</p></div>
|
||
<div class="fact"><h3>Upgrades that keep models running</h3>
|
||
<p>Upgrade the control plane, then the agents. A restarted agent adopts the engines it was
|
||
running instead of reloading them, so replicas stay up throughout.</p></div>
|
||
<div class="fact"><h3>A dashboard with nothing to set up</h3>
|
||
<p><code>/admin</code> shows capacity per GPU class, CPU and memory per node, who is using
|
||
what, each node's page and logs. It fits a phone. Prometheus is optional.</p></div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section id="fabrics">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>InfiniBand and NVLink, several of each</h2>
|
||
<p class="intro">A site may have independent InfiniBand fabrics, eight-GPU HGX boards and NVL72
|
||
racks side by side. llmgw tells them apart by what the adapters report, not by what a config
|
||
file calls them.</p>
|
||
{{if .Site}}<p class="intro"><span class="next">next release</span> Everything in this section is on main
|
||
and ships in the next release.</p>{{end}}
|
||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/fabrics.md">Fabrics and NVLink guide</a></p>
|
||
</div>
|
||
<div class="fab-grid">
|
||
<div class="order" aria-label="Where a replica spanning nodes is placed, in order">
|
||
<div><b>1</b><h3>One NVLink domain</h3><p>NVL72 or NVLink Switch. Every card joined, IMEX up.
|
||
Shards get <code>NCCL_MNNVL_ENABLE=1</code>.</p></div>
|
||
<div><b>2</b><h3>One InfiniBand or RoCE fabric</h3><p>Nodes under one leaf first, nodes with a
|
||
degraded cable last.</p></div>
|
||
<div><b class="x">×</b><h3>Never across two</h3><p>Two fabrics given one name are placed as
|
||
two, and the cluster page says so.</p></div>
|
||
</div>
|
||
<div>
|
||
<h3>Check a node in one command</h3>
|
||
<div class="term" aria-label="Example output of llmgw doctor"><pre>$ sudo llmgw doctor
|
||
<span class="fail">FAIL</span> nvlink this machine has NVSwitch but nvidia-fabricmanager is inactive
|
||
<span class="dim">→ systemctl enable --now nvidia-fabricmanager</span>
|
||
<span class="warn">WARN</span> fabric nvidia_peermem is not loaded: GPUDirect RDMA is off
|
||
<span class="dim">→ modprobe nvidia_peermem</span>
|
||
<span class="ok">OK</span> gpu 8 GPUs: 8× NVIDIA H100 80GB HBM3
|
||
<span class="ok">OK</span> nvlink NVLink groups: [0 1 2 3 4 5 6 7]
|
||
<span class="ok">OK</span> fabric mlx5_0/1 ACTIVE at 400 Gb/s (InfiniBand), subnet manager 0x248a07030030f0c0
|
||
<span class="ok">OK</span> fabric subnet managers: 0x…f0c0 (master, priority 15), 0x…f1d4 (standby, priority 14)
|
||
<span class="ok">OK</span> agent NCCL bootstraps over ens3 (10.1.0.21)</pre></div>
|
||
</div>
|
||
<ul class="watch">
|
||
<li><strong>Subnet managers, watched</strong>Master, priority, standbys and failovers per fabric.
|
||
A fabric with no standby is flagged. llmgw does not run one.</li>
|
||
<li><strong>A map of each fabric</strong><code>/admin/fabrics</code> draws switches in tiers,
|
||
cables coloured by rate, degraded ones in red, and live traffic on your adapters.</li>
|
||
<li><strong>NVLink domains as racks</strong><code>/admin/nvlink</code> shows which trays are
|
||
joined, whose IMEX is down, which link runs degraded.</li>
|
||
<li><strong>Ports re-read every heartbeat</strong>A port that goes down takes its fabric off the
|
||
node within seconds. Counters and rates are exported to Prometheus.</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section id="install">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>From nothing to a first request</h2>
|
||
<p class="intro">Debian, Ubuntu or RHEL 10. NVIDIA drivers and the InfiniBand stack are
|
||
yours to install; llmgw does the rest.</p>
|
||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/install.md">The full guide</a>
|
||
covers the mirror, GPUs and what to do when something does not work.</p>
|
||
</div>
|
||
<ol class="steps">
|
||
<li>
|
||
<h3>Put the binary on every machine</h3>
|
||
<p>Static, no dependencies, the same file everywhere.</p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}} <span class="c"># from the releases page</span>{{end}}
|
||
U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
|
||
curl -fsSLO $U/llmgw-$VERSION-linux-amd64
|
||
curl -fsSL $U/checksums.txt | sha256sum -c --ignore-missing
|
||
sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw</code></pre></div>
|
||
</li>
|
||
<li>
|
||
<h3>Set up the control plane</h3>
|
||
<p>Creates the user, the certificate authority, the certificate, <code>server.yaml</code>
|
||
and your administrator account, then starts the service. Safe to run again.
|
||
Add <code>--host</code> for any name nodes use that the machine cannot see itself.</p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>sudo llmgw server init --admin you@example.com</code></pre></div>
|
||
</li>
|
||
<li>
|
||
<h3>Add a node</h3>
|
||
<p>From the control plane, which holds the authority. The node gets its certificate,
|
||
the binary and the agent, and installs its engine by itself.</p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>deploy/bootstrap/onboard-node.sh --node gpu01 --addr 10.0.0.141 \
|
||
--server https://10.0.0.140:7443 --fabric ib-a:mlx5_0,mlx5_1</code></pre></div>
|
||
<div class="term" aria-label="Example output of llmgw ctl nodes"><pre>$ llmgw ctl nodes
|
||
NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
|
||
gpu01 <span class="ok">ready</span> 8/8 free 503G h100-80g ib-a 0 2s</pre></div>
|
||
</li>
|
||
<li>
|
||
<h3>Add a model and send a request</h3>
|
||
<p>No restart. If it does not place, <code>llmgw ctl explain-placement</code> names every
|
||
node and why it was turned down.</p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>llmgw ctl model add -f qwen-coder-3b.yaml
|
||
|
||
curl https://llm.example.com/v1/chat/completions \
|
||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"model": "coder", "messages": [{"role": "user", "content": "hello"}]}'</code></pre></div>
|
||
</li>
|
||
</ol>
|
||
</div>
|
||
</section>
|
||
|
||
<section id="proxy">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>Behind a reverse proxy</h2>
|
||
<p class="intro">Port 8080 is plain HTTP. Put it behind Caddy or nginx with a public
|
||
certificate. Only 8080 goes through the proxy: agents reach 7443 directly, because it is
|
||
mutual TLS with the cluster's own authority.</p>
|
||
<p class="intro">Bind llmgw to loopback when the proxy is on the same machine:
|
||
<code>listen: "127.0.0.1:8080"</code>.</p>
|
||
</div>
|
||
<div class="tabs">
|
||
<input type="radio" name="proxy" id="t-caddy" checked>
|
||
<input type="radio" name="proxy" id="t-nginx">
|
||
<div class="tablist" role="presentation">
|
||
<label for="t-caddy">Caddy</label>
|
||
<label for="t-nginx">nginx</label>
|
||
</div>
|
||
<div class="panel p-caddy">
|
||
<p class="meta"><span>Obtains and renews its own certificate. Replace <code>llm.example.com</code>.</span>
|
||
<a href="{{.Download}}/Caddyfile">Caddyfile</a></p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Caddy}}</code></pre></div>
|
||
</div>
|
||
<div class="panel p-nginx">
|
||
<p class="meta"><span>Certificate from <code>certbot --nginx</code>. Works on nginx 1.18 and later.</span>
|
||
<a href="{{.Download}}/nginx.conf">nginx.conf</a></p>
|
||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Nginx}}</code></pre></div>
|
||
</div>
|
||
<ul class="rules">
|
||
<li><strong>Streams are not buffered</strong>Tokens reach the client as they are generated, not in one burst at the end.</li>
|
||
<li><strong>The read timeout outlasts the model</strong>nginx's 60-second default turns a slow first token into a 504.</li>
|
||
<li><strong>Forwarded headers are set, not passed on</strong>llmgw marks session cookies Secure from <code>X-Forwarded-Proto</code> and shows <code>X-Forwarded-Host</code> as the base URL.</li>
|
||
<li><strong>Body limit matches the router</strong>32 MiB, so an oversized request is refused before it is uploaded.</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section id="download">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>Download</h2>
|
||
<p class="intro">Built by CI from the tag and checked against <code>checksums.txt</code>.
|
||
Every release is on the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>
|
||
with its changelog.</p>
|
||
</div>
|
||
<div>
|
||
{{if .Release}}
|
||
<div class="ver"><span class="tag">{{.Release}}</span>{{with .Date}}<span class="eyebrow">Released {{.}}</span>{{end}}</div>
|
||
<div class="dl-wrap">
|
||
<table class="dl">
|
||
<thead><tr><th>For</th><th>File</th><th>Notes</th></tr></thead>
|
||
<tbody>
|
||
<tr><td>Linux x86-64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64"><code>llmgw-{{$.Release}}-linux-amd64</code></a></td><td class="note">Static. Runs on any libc.</td></tr>
|
||
<tr><td>Linux ARM64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-arm64"><code>llmgw-{{$.Release}}-linux-arm64</code></a></td><td class="note">Static.</td></tr>
|
||
<tr><td>Linux x86-64, NVLink</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64-nvml"><code>llmgw-{{$.Release}}-linux-amd64-nvml</code></a></td><td class="note">Sees NVLink topology through NVML. For PCIe boxes with NVLink bridges. Needs the driver.</td></tr>
|
||
<tr><td>macOS Apple silicon</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-arm64"><code>llmgw-{{$.Release}}-darwin-arm64</code></a></td><td class="note">For <code>llmgw ctl</code> and trying it on a laptop.</td></tr>
|
||
<tr><td>macOS Intel</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-amd64"><code>llmgw-{{$.Release}}-darwin-amd64</code></a></td><td class="note"></td></tr>
|
||
<tr><td>Caddy</td><td><a href="{{$.Download}}/Caddyfile"><code>Caddyfile</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||
<tr><td>nginx</td><td><a href="{{$.Download}}/nginx.conf"><code>nginx.conf</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||
<tr><td>Checksums</td><td><a href="{{$.Download}}/checksums.txt"><code>checksums.txt</code></a></td><td class="note">SHA-256 of every file above.</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
{{else}}
|
||
<p class="intro" style="margin-top:0">This is a development build ({{.Version}}). Take a release
|
||
from the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>, where every
|
||
file is listed with its checksum.</p>
|
||
{{end}}
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section id="status">
|
||
<div class="sec-head">
|
||
<div>
|
||
<h2>Where it stands</h2>
|
||
<p class="intro">Pre-1.0: configuration formats may change between minor versions.
|
||
This is what has run on real hardware, and what has not yet.</p>
|
||
</div>
|
||
<div class="status">
|
||
<div class="proven">
|
||
<h3>Run on real clusters</h3>
|
||
<ul>
|
||
<li>Single-node replicas on real GPUs and on CPU</li>
|
||
<li>On-demand loading and idle shutdown</li>
|
||
<li>Keys, budgets, rate limits and priority bands</li>
|
||
<li>Rolling upgrades with no dropped replicas</li>
|
||
<li>SQLite and PostgreSQL stores</li>
|
||
<li>Chat, editor setup and the Responses API against a six-node test cluster</li>
|
||
</ul>
|
||
</div>
|
||
<div class="open">
|
||
<h3>Not yet proven</h3>
|
||
<ul>
|
||
<li>InfiniBand and NVLink on a real site: written and tested against the real output of
|
||
<code>sminfo</code>, <code>ibnetdiscover</code> and <code>nvidia-smi</code>, but not yet run
|
||
on the hardware. Run <code>llmgw doctor --json</code> on a few nodes first.</li>
|
||
<li>Two control planes over long periods</li>
|
||
<li>Nodes do not share downloaded weights; the mirror keeps each download local</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<footer>
|
||
<span>llmgw {{.Version}}</span>
|
||
<span><a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a> ·
|
||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/CHANGELOG.md">Changelog</a> ·
|
||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/reference.md">Configuration reference</a></span>
|
||
</footer>
|
||
</div>
|
||
|
||
<script>
|
||
document.querySelectorAll('.copy').forEach(function (b) {
|
||
b.addEventListener('click', function () {
|
||
var text = b.dataset.copy || b.parentElement.querySelector('pre').innerText;
|
||
var done = function (label) { b.textContent = label; setTimeout(function () { b.textContent = 'Copy'; }, 1600); };
|
||
if (navigator.clipboard && navigator.clipboard.writeText) {
|
||
navigator.clipboard.writeText(text).then(function () { done('Copied'); }, function () { select(); });
|
||
} else { select(); }
|
||
function select() {
|
||
var el = b.dataset.copy ? b.parentElement.querySelector('code') : b.parentElement.querySelector('pre');
|
||
var r = document.createRange(); r.selectNodeContents(el);
|
||
var s = window.getSelection(); s.removeAllRanges(); s.addRange(r);
|
||
done('Selected');
|
||
}
|
||
});
|
||
});
|
||
</script>
|
||
{{if not .Site}}</body></html>{{end}}{{end}}
|