{{define "about"}}{{if not .Site}} {{end}}llmgw {{template "sitestyle" .}} {{if not .Site}}{{end}}
llmgw

{{t "about.hero.eyebrow" .Version}}

{{th "about.hero.title"}}

{{t "about.hero.lede"}}

{{t "about.hero.download"}}{{with .Release}} {{.}}{{end}} {{t "about.hero.guide"}}
$ sudo llmgw server init --admin you@example.com
{{t "about.schem.title"}} openai sdk {{t "about.schem.editor"}} curl llmgw :8080 nvl72 · gb200 ib-b · h100-80g
{{t "about.schem.caption"}}

{{t "about.what.title"}}

{{t "about.what.intro"}}

{{t "about.what.api.title"}}

{{th "about.what.api.text"}}

{{t "about.what.placement.title"}}

{{t "about.what.placement.text"}}

{{t "about.what.ondemand.title"}}

{{t "about.what.ondemand.text"}}

{{t "about.what.keys.title"}}

{{t "about.what.keys.text"}}

{{t "about.what.editors.title"}}

{{t "about.what.editors.text"}}

{{t "about.what.chat.title"}}

{{t "about.what.chat.text"}}

{{t "about.what.mirror.title"}}

{{t "about.what.mirror.text"}}

{{t "about.what.engines.title"}}

{{t "about.what.engines.text"}}

{{t "about.what.upgrades.title"}}

{{t "about.what.upgrades.text"}}

{{t "about.what.dashboard.title"}}

{{th "about.what.dashboard.text"}}

{{t "about.what.knowledge.title"}}

{{th "about.what.knowledge.text"}}

{{t "about.what.guard.title"}}

{{t "about.what.guard.text"}}

{{t "about.what.signin.title"}}

{{t "about.what.signin.text"}}

{{t "about.what.ha.title"}}

{{t "about.what.ha.text"}}

{{t "about.what.media.title"}}

{{t "about.what.media.text"}}

{{t "about.fabrics.title"}}

{{t "about.fabrics.intro"}}

{{t "about.fabrics.guide"}}

1

{{t "about.fabrics.nvlink.title"}}

{{th "about.fabrics.nvlink.text"}}

2

{{t "about.fabrics.ib.title"}}

{{t "about.fabrics.ib.text"}}

×

{{t "about.fabrics.never.title"}}

{{t "about.fabrics.never.text"}}

{{t "about.fabrics.doctor"}}

$ sudo llmgw doctor
FAIL  nvlink  this machine has NVSwitch but nvidia-fabricmanager is inactive
              → systemctl enable --now nvidia-fabricmanager
WARN  fabric  nvidia_peermem is not loaded: GPUDirect RDMA is off
              → modprobe nvidia_peermem
OK    gpu     8 GPUs: 8× NVIDIA H100 80GB HBM3
OK    nvlink  NVLink groups: [0 1 2 3 4 5 6 7]
OK    fabric  mlx5_0/1 ACTIVE at 400 Gb/s (InfiniBand), subnet manager 0x248a07030030f0c0
OK    fabric  subnet managers: 0x…f0c0 (master, priority 15), 0x…f1d4 (standby, priority 14)
OK    agent   NCCL bootstraps over ens3 (10.1.0.21)
  • {{t "about.fabrics.sm.title"}}{{t "about.fabrics.sm.text"}}
  • {{t "about.fabrics.map.title"}}{{th "about.fabrics.map.text"}}
  • {{t "about.fabrics.racks.title"}}{{th "about.fabrics.racks.text"}}
  • {{t "about.fabrics.ports.title"}}{{t "about.fabrics.ports.text"}}

{{t "about.install.title"}}

{{t "about.install.intro"}}

{{th "about.install.guide" .InstallURL}}

{{t "about.install.cta"}} →

  1. {{t "about.install.binary.title"}}

    {{t "about.install.binary.text"}}

    VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}}   # from the releases page{{end}}
U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
curl -fsSLO $U/llmgw-$VERSION-linux-amd64
curl -fsSL  $U/checksums.txt | sha256sum -c --ignore-missing
sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw

  2. {{t "about.install.server.title"}}

    {{th "about.install.server.text"}}

    sudo llmgw
    {{.Still "setup-checks"}}

  3. {{t "about.install.node.title"}}

    {{th "about.install.node.text"}}

    sudo llmgw node add ubuntu@10.0.0.21 --fabric ib-a:mlx5_0,mlx5_1
    $ llmgw ctl nodes
NODE   STATE  GPUS      MEM   CLASSES   FABRICS  REPLICAS  LAST SEEN
gpu01  ready  8/8 free  503G  h100-80g  ib-a     0         2s

  4. {{t "about.install.model.title"}}

    {{th "about.install.model.text"}}

    curl https://llm.example.com/v1/chat/completions \
  -H "Authorization: Bearer $LLMGW_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"model": "coder", "messages": [{"role": "user", "content": "hello"}]}'

{{t "about.proxy.title"}}

{{t "about.proxy.intro"}}

{{th "about.proxy.loopback"}}

{{th "about.proxy.caddy"}} Caddyfile

{{.Caddy}}

{{th "about.proxy.nginx"}} nginx.conf

{{.Nginx}}
  • {{t "about.proxy.stream.title"}}{{t "about.proxy.stream.text"}}
  • {{t "about.proxy.timeout.title"}}{{t "about.proxy.timeout.text"}}
  • {{t "about.proxy.headers.title"}}{{th "about.proxy.headers.text"}}
  • {{t "about.proxy.body.title"}}{{t "about.proxy.body.text"}}

{{t "about.download.title"}}

{{th "about.download.intro"}}

{{if .Release}}
{{.Release}}{{with .Date}}{{t "about.download.released" .}}{{end}}
{{t "about.download.col.for"}}{{t "about.download.col.file"}}{{t "about.download.col.notes"}}
Linux x86-64llmgw-{{$.Release}}-linux-amd64{{t "about.download.static_any"}}
Linux ARM64llmgw-{{$.Release}}-linux-arm64{{t "about.download.static"}}
Linux x86-64, NVLinkllmgw-{{$.Release}}-linux-amd64-nvml{{t "about.download.nvml"}}
{{t "about.download.mac_arm"}}llmgw-{{$.Release}}-darwin-arm64{{th "about.download.mac_note"}}
macOS Intelllmgw-{{$.Release}}-darwin-amd64
CaddyCaddyfile{{t "about.download.proxy_note"}}
nginxnginx.conf{{t "about.download.proxy_note"}}
{{t "about.download.checksums"}}checksums.txt{{t "about.download.checksums_note"}}
{{else}}

{{th "about.download.dev" .Version}}

{{end}}

{{t "about.status.title"}}

{{t "about.status.intro"}}

{{t "about.status.proven"}}

  • {{t "about.status.proven.single"}}
  • {{t "about.status.proven.ondemand"}}
  • {{t "about.status.proven.keys"}}
  • {{t "about.status.proven.upgrades"}}
  • {{t "about.status.proven.stores"}}
  • {{t "about.status.proven.chat"}}
  • {{t "about.status.proven.ha"}}
  • {{t "about.status.proven.signin"}}
  • {{t "about.status.proven.kb"}}

{{t "about.status.open"}}

  • {{th "about.status.open.fabrics"}}
  • {{t "about.status.open.planes"}}
  • {{t "about.status.open.weights"}}
{{if not .Site}}{{end}}{{end}}