Serve the project's website at /about, with its fonts in the binary
The website was a file built by hand outside the repository. It is now a template in internal/web: every gateway serves it at /about, without a key, linked from the landing page's footer, and the published site is rendered from the same template. The fonts are vendored (Latin subsets, OFL) so the page sends nobody's browser to Google. The proxy configurations it shows are copies of deploy/proxy, kept equal by a test. A development build says so instead of linking to release files that do not exist. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed
1 parent
80e0dd252f
commit
e751c89937
16 files changed
+1016
-2
No files matched your search
@@ -150,6 +150,14 @@ The account page (was "Your keys") shows the last thirty days of a person's own
|
||||
use and, for each place they spend, a bar for every budget that applies — their
|
||||
own, or a project's shared with others, said as such.
|
||||
|
||||
### The project's website, in every gateway
|
||||
|
||||
`/about` is llmgw's own page: what it does, InfiniBand and NVLink, installing,
|
||||
the reverse proxy configurations, and the release's files to download. It is
|
||||
served without a key like `/`, whose footer links to it, and says nothing about
|
||||
the installation. Its fonts are in the binary, so it makes no request elsewhere.
|
||||
The published website is rendered from the same template.
|
||||
|
||||
### The admin pages
|
||||
|
||||
- The cluster page leads with capacity, nodes, models and replicas, then usage;
|
||||
|
||||
@@ -10,6 +10,9 @@ put it behind a proxy that holds a publicly trusted certificate.
|
||||
|
||||
Both are also attached to every release.
|
||||
|
||||
The `/about` page shows both, from copies in `internal/web/proxy`; a test fails
|
||||
when those differ from these, so change them here and copy them over.
|
||||
|
||||
Four things both files get right, and any other proxy has to as well:
|
||||
|
||||
- **Streaming is not buffered.** Replies are server-sent events; a buffering
|
||||
|
||||
@@ -117,6 +117,9 @@ func (r *Router) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil,
|
||||
r.accounts != nil && r.accounts.SignupOpen(),
|
||||
r.landing.Contact, r.landing.Notice))
|
||||
// What llmgw is and how to install it: about the software, not this
|
||||
// installation, so it needs no key either.
|
||||
mux.HandleFunc("GET /about", web.About(version.Version))
|
||||
mux.Handle("GET /_static/", web.Static())
|
||||
mux.HandleFunc("GET /me", r.handleMe)
|
||||
// Somebody has to be able to say "forget that", and a retention window is
|
||||
@@ -199,7 +202,7 @@ func (r *Router) Handler() http.Handler {
|
||||
// route must not be able to add an unauthenticated one by omission. Health
|
||||
// stays open because a load balancer has no key.
|
||||
return auth.Middleware(r.auth, r.log,
|
||||
"/health", "/ready", "/", "/_static/", "/claim/",
|
||||
"/health", "/ready", "/", "/about", "/_static/", "/claim/",
|
||||
"/signup", "/login", "/logout", "/profile", "/profile/",
|
||||
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The page about llmgw itself: what it is, how to install it, where to get
|
||||
// it. It is the project's website, carried by every gateway so an operator
|
||||
// has it where they already are, and served without a key like the landing
|
||||
// page — it says nothing about this installation, only about the software.
|
||||
//
|
||||
// The same template renders the published website (Site), which differs in
|
||||
// three ways: fonts from Google rather than from this binary, no link back to
|
||||
// a gateway, and features on main marked as coming in the next release.
|
||||
|
||||
// proxyConfigs are the reverse proxy configurations the page shows. They are
|
||||
// copies of deploy/proxy, which the release attaches; embed cannot reach
|
||||
// outside this package, and TestAboutProxyConfigsMatchDeploy keeps the two
|
||||
// the same.
|
||||
//
|
||||
//go:embed proxy
|
||||
var proxyConfigs embed.FS
|
||||
|
||||
// AboutData is what the page is rendered with.
|
||||
type AboutData struct {
|
||||
// Version is what this build calls itself; Release is the same when it is
|
||||
// a released version, and empty for a development build, which has no
|
||||
// files to download.
|
||||
Version, Release string
|
||||
// Download is where the release's files are, or for a development build
|
||||
// the proxy configurations on main.
|
||||
Download string
|
||||
// Date is the release date, when it is known. A binary does not know it.
|
||||
Date string
|
||||
Caddy, Nginx string
|
||||
Site bool
|
||||
}
|
||||
|
||||
// releaseTag is a version the releases page has files for.
|
||||
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
|
||||
|
||||
const repoURL = "https://gitea.mynodes.xyz/ilker/llmgw"
|
||||
|
||||
// NewAboutData fills the page's data for a version.
|
||||
func NewAboutData(version string) AboutData {
|
||||
d := AboutData{Version: version, Download: repoURL + "/raw/branch/main/deploy/proxy"}
|
||||
if releaseTag.MatchString(version) {
|
||||
d.Release = version
|
||||
d.Download = repoURL + "/releases/download/" + version
|
||||
}
|
||||
d.Caddy = proxyConfig("Caddyfile")
|
||||
d.Nginx = proxyConfig("nginx.conf")
|
||||
return d
|
||||
}
|
||||
|
||||
func proxyConfig(name string) string {
|
||||
b, err := proxyConfigs.ReadFile("proxy/" + name)
|
||||
if err != nil {
|
||||
panic("web: the proxy configuration " + name + " is missing from the binary: " + err.Error())
|
||||
}
|
||||
return strings.TrimRight(string(b), "\n")
|
||||
}
|
||||
|
||||
// RenderAbout writes the page, for the website's build as much as for About.
|
||||
func RenderAbout(w io.Writer, d AboutData) error {
|
||||
return tmpl.ExecuteTemplate(w, "about", d)
|
||||
}
|
||||
|
||||
// About serves the page at /about.
|
||||
func About(version string) http.HandlerFunc {
|
||||
d := NewAboutData(version)
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
render(w, "about", d)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAboutPageForARelease(t *testing.T) {
|
||||
code, body := get(t, About("v1.2.3"), "http://llmgw.example/about", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"<!doctype html>",
|
||||
repoURL + "/releases/download/v1.2.3/llmgw-v1.2.3-linux-amd64",
|
||||
"VERSION=v1.2.3",
|
||||
"reverse_proxy", // the Caddyfile, inline
|
||||
"proxy_read_timeout", // and nginx's
|
||||
`<a href="/">This gateway</a>`,
|
||||
"/_static/fonts/archivo-latin.woff2?v=",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page does not have %q", want)
|
||||
}
|
||||
}
|
||||
// Served to anyone, on networks that may reach nothing: the page sends no
|
||||
// browser elsewhere for its fonts, and does not announce unreleased work
|
||||
// in a binary that already has it.
|
||||
for _, not := range []string{"fonts.googleapis.com", "next release"} {
|
||||
if strings.Contains(body, not) {
|
||||
t.Errorf("the page has %q", not)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAboutPageForADevelopmentBuild(t *testing.T) {
|
||||
_, body := get(t, About("v1.2.3-4-gabcdef"), "http://llmgw.example/about", nil)
|
||||
if strings.Contains(body, "/releases/download/v1.2.3-4") || strings.Contains(body, `class="dl"`) {
|
||||
t.Error("a development build links to release files that do not exist")
|
||||
}
|
||||
if !strings.Contains(body, "development build (v1.2.3-4-gabcdef)") {
|
||||
t.Error("a development build does not say what it is")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAboutPageAsTheWebsite(t *testing.T) {
|
||||
d := NewAboutData("v1.2.3")
|
||||
d.Site, d.Date = true, "1 January 2027"
|
||||
var b bytes.Buffer
|
||||
if err := RenderAbout(&b, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := b.String()
|
||||
for _, want := range []string{"fonts.googleapis.com", "Released 1 January 2027", "next release"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the website does not have %q", want)
|
||||
}
|
||||
}
|
||||
for _, not := range []string{"<!doctype", "</body>", "This gateway", "/_static/"} {
|
||||
if strings.Contains(body, not) {
|
||||
t.Errorf("the website has %q", not)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAboutProxyConfigsMatchDeploy(t *testing.T) {
|
||||
for _, name := range []string{"Caddyfile", "nginx.conf"} {
|
||||
want, err := os.ReadFile("../../deploy/proxy/" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := proxyConfigs.ReadFile("proxy/" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("internal/web/proxy/%s differs from deploy/proxy/%[1]s: copy it over", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
# llmgw behind Caddy: the client-facing port (8080) on a public name, with a
|
||||
# certificate Caddy obtains and renews by itself.
|
||||
#
|
||||
# Replace llm.example.com, point its DNS at this machine, open 80 and 443, and
|
||||
# run: caddy run --config /etc/caddy/Caddyfile
|
||||
#
|
||||
# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's
|
||||
# own authority and must reach llmgw directly; 9090 and 9091 stay private.
|
||||
|
||||
llm.example.com {
|
||||
# The router reads at most 32 MiB of body (router.max_body_bytes). Refusing
|
||||
# more here gives the client a 413 before the upload, not after it.
|
||||
request_body {
|
||||
max_size 32MB
|
||||
}
|
||||
|
||||
reverse_proxy 127.0.0.1:8080 {
|
||||
# Tokens are written the moment they are generated. Without this a
|
||||
# streamed reply can arrive in bursts, and a client measuring time to
|
||||
# first token measures the proxy instead.
|
||||
flush_interval -1
|
||||
|
||||
# Caddy sets X-Forwarded-Proto and X-Forwarded-Host itself and, with
|
||||
# no trusted_proxies, discards whatever the client sent. llmgw relies
|
||||
# on that: the proto decides whether session cookies are Secure, and
|
||||
# the host is the base URL the landing page tells people to use.
|
||||
# Behind a load balancer of your own, name it in trusted_proxies.
|
||||
|
||||
transport http {
|
||||
# A long generation can be silent for a while before the first
|
||||
# token. llmgw has its own limits (timeouts.*); this must not be
|
||||
# the shorter one.
|
||||
response_header_timeout 15m
|
||||
}
|
||||
}
|
||||
|
||||
# No `encode` on purpose: compressing a stream buffers it.
|
||||
|
||||
# Access log to stderr, which under systemd is the journal.
|
||||
log
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
# llmgw behind nginx: the client-facing port (8080) on a public name.
|
||||
#
|
||||
# Put this in /etc/nginx/sites-available/llmgw (or conf.d/llmgw.conf), replace
|
||||
# llm.example.com, and get a certificate — certbot --nginx -d llm.example.com
|
||||
# fills in the ssl_ lines below. Then: nginx -t && systemctl reload nginx
|
||||
#
|
||||
# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's
|
||||
# own authority and must reach llmgw directly; 9090 and 9091 stay private.
|
||||
|
||||
upstream llmgw {
|
||||
server 127.0.0.1:8080;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name llm.example.com;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
# `listen ... http2` rather than `http2 on;`: the latter needs nginx
|
||||
# 1.25.1, and Ubuntu 22.04 and Debian 12 ship older ones. Newer nginx
|
||||
# accepts this form with a deprecation warning.
|
||||
listen 443 ssl http2;
|
||||
listen [::]:443 ssl http2;
|
||||
server_name llm.example.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/llm.example.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/llm.example.com/privkey.pem;
|
||||
|
||||
# The router reads at most 32 MiB of body (router.max_body_bytes).
|
||||
client_max_body_size 32m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://llmgw;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
# Set, never appended: llmgw decides from X-Forwarded-Proto whether
|
||||
# session cookies are Secure, and builds the base URL the landing page
|
||||
# shows from X-Forwarded-Host. A value passed through from the client
|
||||
# would be the client's choice.
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
||||
# Streamed replies: tokens go out as they are generated. Buffering
|
||||
# turns a stream into one late burst, and gzip buffers too.
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
gzip off;
|
||||
|
||||
# nginx's default is 60 seconds between two reads from upstream. A
|
||||
# large model can think longer than that before its first token, and
|
||||
# the client then sees a 504 for a request that was about to succeed.
|
||||
# llmgw has its own limits (timeouts.*); this must not be the shorter.
|
||||
proxy_read_timeout 15m;
|
||||
proxy_send_timeout 15m;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
Fonts for the /about page, vendored rather than loaded from Google Fonts: this
|
||||
gateway makes no outbound requests of its own, and a page served to anyone who
|
||||
can reach it should not send their browser to a third party either.
|
||||
|
||||
archivo-latin.woff2 Archivo, variable (width 112–125%, weight 600–800)
|
||||
plex-sans-latin.woff2 IBM Plex Sans, variable weight
|
||||
plex-mono-400-latin.woff2 IBM Plex Mono Regular
|
||||
plex-mono-500-latin.woff2 IBM Plex Mono Medium
|
||||
|
||||
The Latin subsets as Google Fonts serves them, unmodified. Both families are
|
||||
under the SIL Open Font License 1.1: OFL-Archivo.txt, OFL-IBMPlex.txt.
|
||||
|
||||
sha256:
|
||||
4c98b9d490d1698ec95f2ff17a6c7d0e72691864c0c5d7bc2a2c161b45afe5ad archivo-latin.woff2
|
||||
c36f509c0a8f9f85f29cb44bc8701d8a9e0b14c499e77a884f789ead7093a7ac plex-mono-400-latin.woff2
|
||||
a76f53ca6612e7b3828eec2311098675b7f9849ae4169a8bcef6302aec02a6c0 plex-mono-500-latin.woff2
|
||||
056e4e2459f57a0033c8c9c844ff19d6e42ac8602027803d4345823bcc939818 plex-sans-latin.woff2
|
||||
@@ -0,0 +1,93 @@
|
||||
Copyright 2020 The Archivo Project Authors (https://github.com/Omnibus-Type/Archivo)
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
http://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
@@ -0,0 +1,93 @@
|
||||
Copyright © 2017 IBM Corp. with Reserved Font Name "Plex"
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
|
||||
This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,531 @@
|
||||
{{define "about"}}{{if not .Site}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="description" content="llmgw serves models on your own GPUs behind one OpenAI-compatible API.">
|
||||
{{end}}<title>llmgw</title>
|
||||
{{if .Site}}<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wdth,wght@112,600;125,800&family=IBM+Plex+Sans:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500&display=swap">
|
||||
{{end}}<style>
|
||||
{{if not .Site}}@font-face{font-family:"Archivo";font-weight:600 800;font-stretch:112% 125%;font-display:swap;src:url("{{asset "fonts/archivo-latin.woff2"}}") format("woff2")}
|
||||
@font-face{font-family:"IBM Plex Sans";font-weight:100 700;font-display:swap;src:url("{{asset "fonts/plex-sans-latin.woff2"}}") format("woff2")}
|
||||
@font-face{font-family:"IBM Plex Mono";font-weight:400;font-display:swap;src:url("{{asset "fonts/plex-mono-400-latin.woff2"}}") format("woff2")}
|
||||
@font-face{font-family:"IBM Plex Mono";font-weight:500;font-display:swap;src:url("{{asset "fonts/plex-mono-500-latin.woff2"}}") format("woff2")}
|
||||
{{end}}:root{
|
||||
--bg:#F3F5F5; --surface:#FFFFFF; --ink:#142023; --muted:#56696C; --line:#D3DCDC;
|
||||
--fiber:#0B7F7D; --fiber-soft:#DDEFEE; --om1:#C4621A; --om1-soft:#F6E6D9;
|
||||
--code-bg:#15201F; --code-ink:#DCEAE8; --code-dim:#7F9A97;
|
||||
--display:"Archivo","Arial Narrow",system-ui,sans-serif;
|
||||
--body:"IBM Plex Sans",system-ui,-apple-system,"Segoe UI",sans-serif;
|
||||
--mono:"IBM Plex Mono",ui-monospace,"SFMono-Regular",Menlo,monospace;
|
||||
}
|
||||
@media (prefers-color-scheme:dark){
|
||||
:root:not([data-theme="light"]){
|
||||
--bg:#0E1617; --surface:#152022; --ink:#E2ECEB; --muted:#8FA5A4; --line:#26383A;
|
||||
--fiber:#3FC2BC; --fiber-soft:#133634; --om1:#EE9350; --om1-soft:#3A2717;
|
||||
--code-bg:#0A1011; --code-ink:#D5E5E3; --code-dim:#6E8886;
|
||||
color-scheme:dark;
|
||||
}
|
||||
}
|
||||
:root[data-theme="dark"]{
|
||||
--bg:#0E1617; --surface:#152022; --ink:#E2ECEB; --muted:#8FA5A4; --line:#26383A;
|
||||
--fiber:#3FC2BC; --fiber-soft:#133634; --om1:#EE9350; --om1-soft:#3A2717;
|
||||
--code-bg:#0A1011; --code-ink:#D5E5E3; --code-dim:#6E8886;
|
||||
color-scheme:dark;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
body{background:var(--bg);color:var(--ink);font:400 16px/1.6 var(--body);margin:0;padding-inline:20px;padding-block:0}
|
||||
.wrap{max-width:1080px;margin:0 auto}
|
||||
a{color:var(--fiber);text-underline-offset:3px}
|
||||
a:focus-visible,button:focus-visible,input:focus-visible+label{outline:2px solid var(--fiber);outline-offset:3px;border-radius:4px}
|
||||
h1,h2,h3{font-family:var(--display);text-wrap:balance;margin:0}
|
||||
h2{font-size:clamp(1.6rem,3.2vw,2.1rem);font-variation-settings:"wdth" 112;font-weight:600;letter-spacing:-.01em}
|
||||
h3{font-size:1.05rem;font-variation-settings:"wdth" 112;font-weight:600}
|
||||
p{margin:0}
|
||||
code,pre{font-family:var(--mono);font-size:.875rem}
|
||||
:not(pre)>code{background:var(--fiber-soft);padding:.1em .35em;border-radius:4px;font-size:.85em}
|
||||
.eyebrow{font:500 .75rem/1 var(--mono);letter-spacing:.12em;text-transform:uppercase;color:var(--muted)}
|
||||
|
||||
/* top bar */
|
||||
.bar{display:flex;align-items:center;justify-content:space-between;gap:16px;padding-block:22px;border-bottom:1px solid var(--line)}
|
||||
.mark{font:800 1.25rem/1 var(--display);font-variation-settings:"wdth" 125;letter-spacing:-.02em;color:var(--ink);text-decoration:none;display:flex;align-items:center;gap:10px}
|
||||
.mark svg{flex:none}
|
||||
.bar nav{display:flex;gap:20px;flex-wrap:wrap;font-size:.9rem}
|
||||
.bar nav a{color:var(--muted);text-decoration:none}
|
||||
.bar nav a:hover{color:var(--ink)}
|
||||
|
||||
/* hero */
|
||||
.hero{display:grid;grid-template-columns:1.05fr .95fr;gap:48px;align-items:center;padding-block:64px 56px}
|
||||
.hero h1{font-size:clamp(2.5rem,6vw,4.4rem);line-height:.98;font-weight:800;font-variation-settings:"wdth" 125;letter-spacing:-.03em}
|
||||
.hero h1 em{font-style:normal;color:var(--fiber)}
|
||||
.lede{margin-top:22px;font-size:1.12rem;color:var(--muted);max-width:34em}
|
||||
.cta{display:flex;flex-wrap:wrap;gap:12px;margin-top:30px}
|
||||
.btn{display:inline-flex;align-items:center;gap:8px;font:600 .95rem/1 var(--body);padding:13px 18px;border-radius:8px;text-decoration:none;border:1px solid var(--fiber);background:var(--fiber);color:var(--bg)}
|
||||
.btn.ghost{background:transparent;color:var(--ink);border-color:var(--line)}
|
||||
.btn.ghost:hover{border-color:var(--ink)}
|
||||
.oneline{margin-top:26px;display:flex;align-items:center;gap:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;padding:12px 12px 12px 16px;max-width:100%}
|
||||
.oneline code{flex:1;min-width:0;overflow-x:auto;white-space:nowrap;background:none;padding:0;font-size:.9rem;color:var(--code-ink)}
|
||||
.oneline .p{color:var(--code-dim);user-select:none}
|
||||
|
||||
/* schematic */
|
||||
figure{margin:0}
|
||||
.schem{background:var(--surface);border:1px solid var(--line);border-radius:12px;padding:18px}
|
||||
.schem svg{display:block;width:100%;height:auto}
|
||||
.schem figcaption{margin-top:12px;font-size:.85rem;color:var(--muted)}
|
||||
.s-label{font:500 11px var(--mono);fill:var(--muted)}
|
||||
.s-strong{font:600 13px var(--body);fill:var(--ink)}
|
||||
.s-box{fill:var(--bg);stroke:var(--line)}
|
||||
.s-gw{fill:var(--fiber-soft);stroke:var(--fiber);stroke-width:1.5}
|
||||
.s-fab{fill:none;stroke:var(--line);stroke-dasharray:5 4}
|
||||
.s-link{fill:none;stroke:var(--line);stroke-width:2}
|
||||
.s-gpu-a{fill:var(--fiber)} .s-gpu-b{fill:var(--om1)}
|
||||
.s-gpu-idle{fill:var(--line)}
|
||||
.pulse{fill:var(--fiber)}
|
||||
.pulse.b{fill:var(--om1)}
|
||||
@media (prefers-reduced-motion:reduce){.pulse{display:none}}
|
||||
|
||||
/* sections */
|
||||
section{padding-block:56px;border-top:1px solid var(--line)}
|
||||
.sec-head{display:grid;grid-template-columns:260px 1fr;gap:40px}
|
||||
.sec-head .intro{color:var(--muted);margin-top:12px;font-size:.95rem}
|
||||
|
||||
.facts{display:grid;grid-template-columns:1fr 1fr;gap:28px 40px}
|
||||
.fact h3{display:flex;gap:10px;align-items:baseline}
|
||||
.fact h3::before{content:"";width:10px;height:10px;border-radius:2px;background:var(--fiber);flex:none;transform:translateY(-1px)}
|
||||
.fact p{margin-top:6px;color:var(--muted);font-size:.95rem;max-width:36em}
|
||||
|
||||
/* steps */
|
||||
.steps{list-style:none;margin:0;padding:0;display:grid;gap:28px;counter-reset:s}
|
||||
.steps li{display:grid;grid-template-columns:44px 1fr;gap:4px 16px;counter-increment:s}
|
||||
.steps li>*{grid-column:2}
|
||||
.steps li::before{content:counter(s);grid-column:1;grid-row:span 4;font:800 1.5rem/1.1 var(--display);font-variation-settings:"wdth" 125;color:var(--fiber)}
|
||||
.steps p{color:var(--muted);font-size:.95rem;max-width:40em}
|
||||
.code{position:relative;margin-top:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;min-width:0}
|
||||
.code pre{margin:0;padding:16px 18px;overflow-x:auto;line-height:1.55}
|
||||
.code .c{color:var(--code-dim)}
|
||||
.copy{position:absolute;top:8px;right:8px;font:500 .75rem/1 var(--body);padding:7px 10px;border-radius:6px;border:1px solid #2d4442;background:#1c2b2a;color:var(--code-ink);cursor:pointer}
|
||||
.copy:hover{border-color:var(--code-dim)}
|
||||
|
||||
/* nodes table in step */
|
||||
.term{margin-top:10px;background:var(--code-bg);color:var(--code-ink);border-radius:8px;overflow-x:auto}
|
||||
.term pre{margin:0;padding:14px 18px;font-size:.8rem;line-height:1.6}
|
||||
.term .ok{color:#6FD3A6}
|
||||
|
||||
/* proxy tabs */
|
||||
.tabs{min-width:0}
|
||||
.tabs input{position:absolute;opacity:0;pointer-events:none}
|
||||
.tablist{display:flex;gap:4px;border-bottom:1px solid var(--line)}
|
||||
.tablist label{cursor:pointer;padding:10px 16px;font:600 .9rem/1 var(--body);color:var(--muted);border-bottom:2px solid transparent;margin-bottom:-1px}
|
||||
#t-caddy:checked~.tablist label[for="t-caddy"],#t-nginx:checked~.tablist label[for="t-nginx"]{color:var(--ink);border-color:var(--fiber)}
|
||||
.panel{display:none;margin-top:16px}
|
||||
#t-caddy:checked~.p-caddy,#t-nginx:checked~.p-nginx{display:block}
|
||||
.panel .meta{display:flex;flex-wrap:wrap;justify-content:space-between;gap:8px 16px;font-size:.88rem;color:var(--muted)}
|
||||
.panel .code pre{max-height:460px}
|
||||
.rules{margin-top:28px;display:grid;grid-template-columns:1fr 1fr;gap:18px 32px;padding:0;list-style:none}
|
||||
.rules li{font-size:.92rem;color:var(--muted);padding-left:16px;border-left:2px solid var(--om1)}
|
||||
.rules strong{color:var(--ink);font-weight:600;display:block}
|
||||
|
||||
/* downloads */
|
||||
.dl{width:100%;border-collapse:collapse;font-size:.92rem}
|
||||
.dl-wrap{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--surface)}
|
||||
.dl th{text-align:left;font:500 .72rem/1 var(--mono);letter-spacing:.1em;text-transform:uppercase;color:var(--muted);padding:14px 16px;border-bottom:1px solid var(--line)}
|
||||
.dl td{padding:12px 16px;border-bottom:1px solid var(--line);vertical-align:top}
|
||||
.dl tr:last-child td{border-bottom:0}
|
||||
.dl td:first-child{white-space:nowrap;font-weight:500}
|
||||
.dl code{background:none;padding:0;font-size:.85rem}
|
||||
.dl .note{color:var(--muted)}
|
||||
.ver{display:flex;flex-wrap:wrap;gap:10px 24px;align-items:baseline;margin-bottom:18px}
|
||||
.tag{font:500 .8rem/1 var(--mono);padding:6px 9px;border-radius:5px;background:var(--fiber-soft);color:var(--fiber)}
|
||||
|
||||
/* status */
|
||||
.status{display:grid;grid-template-columns:1fr 1fr;gap:32px}
|
||||
.status h3{margin-bottom:10px}
|
||||
.status ul{margin:0;padding:0;list-style:none;display:grid;gap:10px}
|
||||
.status li{font-size:.93rem;color:var(--muted);display:grid;grid-template-columns:18px 1fr;gap:8px}
|
||||
.status li::before{font:600 .9rem/1.6 var(--mono)}
|
||||
.proven li::before{content:"✓";color:var(--fiber)}
|
||||
.open li::before{content:"○";color:var(--om1)}
|
||||
|
||||
.next{font:500 .66rem/1 var(--mono);letter-spacing:.08em;text-transform:uppercase;padding:4px 6px;border-radius:4px;background:var(--om1-soft);color:var(--om1);white-space:nowrap;align-self:center}
|
||||
.fab-grid{display:grid;grid-template-columns:1fr;gap:28px;min-width:0}
|
||||
.term .warn{color:#E9B35A} .term .fail{color:#F07C6C} .term .dim{color:var(--code-dim)}
|
||||
.watch{display:grid;grid-template-columns:1fr 1fr;gap:18px 32px;padding:0;margin:0;list-style:none}
|
||||
.watch li{font-size:.92rem;color:var(--muted);padding-left:16px;border-left:2px solid var(--fiber)}
|
||||
.watch strong{color:var(--ink);font-weight:600;display:block}
|
||||
.watch code{font-size:.8em}
|
||||
.order{display:grid;grid-template-columns:repeat(3,1fr);gap:0;border:1px solid var(--line);border-radius:10px;background:var(--surface);overflow:hidden}
|
||||
.order div{padding:16px 18px;border-right:1px solid var(--line)}
|
||||
.order div:last-child{border-right:0}
|
||||
.order b{display:block;font:800 1.4rem/1 var(--display);font-variation-settings:"wdth" 125;color:var(--fiber);margin-bottom:8px}
|
||||
.order b.x{color:var(--om1)}
|
||||
.order h3{font-size:.95rem}
|
||||
.order p{font-size:.88rem;color:var(--muted);margin-top:4px}
|
||||
footer{border-top:1px solid var(--line);padding-block:28px 40px;display:flex;flex-wrap:wrap;justify-content:space-between;gap:12px;font-size:.85rem;color:var(--muted)}
|
||||
footer a{color:var(--muted)}
|
||||
|
||||
@media (max-width:860px){
|
||||
.hero{grid-template-columns:1fr;gap:36px;padding-block:44px 40px}
|
||||
.sec-head{grid-template-columns:1fr;gap:20px}
|
||||
.facts,.rules,.status,.watch,.order{grid-template-columns:1fr}
|
||||
.order div{border-right:0;border-bottom:1px solid var(--line)}
|
||||
.order div:last-child{border-bottom:0}
|
||||
.bar nav{display:none}
|
||||
}
|
||||
</style>
|
||||
{{if not .Site}}</head><body>{{end}}
|
||||
|
||||
<div class="wrap">
|
||||
<header class="bar">
|
||||
<a class="mark" href="#top" aria-label="llmgw home">
|
||||
<svg width="26" height="26" viewBox="0 0 26 26" aria-hidden="true">
|
||||
<rect x="1" y="1" width="24" height="24" rx="6" fill="none" stroke="currentColor" stroke-width="2"/>
|
||||
<path d="M7 9h12M7 13h12M7 17h12" stroke="var(--fiber)" stroke-width="2" stroke-linecap="round"/>
|
||||
</svg>
|
||||
llmgw
|
||||
</a>
|
||||
<nav aria-label="Sections">
|
||||
<a href="#what">What it does</a>
|
||||
<a href="#fabrics">Fabrics</a>
|
||||
<a href="#install">Install</a>
|
||||
<a href="#proxy">Reverse proxy</a>
|
||||
<a href="#download">Download</a>
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a>
|
||||
{{if not .Site}}<a href="/">This gateway</a>{{end}}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="hero" id="top">
|
||||
<div>
|
||||
<p class="eyebrow">Self-hosted LLM gateway · {{.Version}}</p>
|
||||
<h1 style="margin-top:14px">One endpoint in front of <em>your own GPUs</em>.</h1>
|
||||
<p class="lede">llmgw runs vLLM, SGLang and llama.cpp across your servers and serves them
|
||||
behind one OpenAI-compatible API. It knows which GPUs share an NVLink domain or an InfiniBand
|
||||
fabric, and a model split across machines never straddles two.</p>
|
||||
<div class="cta">
|
||||
<a class="btn" href="#download">Download{{with .Release}} {{.}}{{end}}</a>
|
||||
<a class="btn ghost" href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/install.md">Install guide</a>
|
||||
</div>
|
||||
<div class="oneline" aria-label="Set up a control plane">
|
||||
<code><span class="p">$ </span>sudo llmgw server init --admin you@example.com</code>
|
||||
<button class="copy" style="position:static" data-copy="sudo llmgw server init --admin you@example.com" type="button">Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<figure class="schem">
|
||||
<svg viewBox="0 0 440 250" role="img" aria-labelledby="schem-t">
|
||||
<title id="schem-t">Clients reach one llmgw address. llmgw places a model's shards on GPUs inside one fabric at a time.</title>
|
||||
<!-- clients -->
|
||||
<g>
|
||||
<rect class="s-box" x="4" y="58" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="77" text-anchor="middle">openai sdk</text>
|
||||
<rect class="s-box" x="4" y="110" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="129" text-anchor="middle">editor agent</text>
|
||||
<rect class="s-box" x="4" y="162" width="92" height="30" rx="6"/>
|
||||
<text class="s-label" x="50" y="181" text-anchor="middle">curl</text>
|
||||
</g>
|
||||
<path id="c1" class="s-link" d="M96 73 C130 73 130 118 160 118"/>
|
||||
<path id="c2" class="s-link" d="M96 125 H160"/>
|
||||
<path id="c3" class="s-link" d="M96 177 C130 177 130 132 160 132"/>
|
||||
<!-- gateway -->
|
||||
<rect class="s-gw" x="160" y="96" width="84" height="58" rx="8"/>
|
||||
<text class="s-strong" x="202" y="122" text-anchor="middle">llmgw</text>
|
||||
<text class="s-label" x="202" y="139" text-anchor="middle">:8080</text>
|
||||
<path id="fa" class="s-link" d="M244 114 C272 114 272 62 300 62"/>
|
||||
<path id="fb" class="s-link" d="M244 136 C272 136 272 188 300 188"/>
|
||||
<!-- fabric a -->
|
||||
<rect class="s-fab" x="300" y="14" width="136" height="98" rx="10"/>
|
||||
<text class="s-label" x="312" y="32">nvl72 · gb200</text>
|
||||
<g transform="translate(312 42)">
|
||||
<rect class="s-gpu-a" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-a" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="84" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-a" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-a" x="28" y="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-idle" x="56" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" y="28" width="22" height="22" rx="3"/>
|
||||
</g>
|
||||
<!-- fabric b -->
|
||||
<rect class="s-fab" x="300" y="138" width="136" height="98" rx="10"/>
|
||||
<text class="s-label" x="312" y="156">ib-b · h100-80g</text>
|
||||
<g transform="translate(312 166)">
|
||||
<rect class="s-gpu-b" width="22" height="22" rx="3"/><rect class="s-gpu-b" x="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-idle" x="56" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-idle" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="28" y="28" width="22" height="22" rx="3"/>
|
||||
<rect class="s-gpu-idle" x="56" y="28" width="22" height="22" rx="3"/><rect class="s-gpu-idle" x="84" y="28" width="22" height="22" rx="3"/>
|
||||
</g>
|
||||
<!-- requests in flight -->
|
||||
<circle class="pulse" r="4"><animateMotion dur="2.4s" repeatCount="indefinite"><mpath href="#c1"/></animateMotion></circle>
|
||||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="0.8s" repeatCount="indefinite"><mpath href="#c2"/></animateMotion></circle>
|
||||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="1.6s" repeatCount="indefinite"><mpath href="#c3"/></animateMotion></circle>
|
||||
<circle class="pulse" r="4"><animateMotion dur="2.4s" begin="1.2s" repeatCount="indefinite"><mpath href="#fa"/></animateMotion></circle>
|
||||
<circle class="pulse b" r="4"><animateMotion dur="2.4s" begin="2s" repeatCount="indefinite"><mpath href="#fb"/></animateMotion></circle>
|
||||
</svg>
|
||||
<figcaption>A replica spanning nodes goes into one NVLink domain if one has room, otherwise onto
|
||||
one InfiniBand fabric. Two fabrics that cannot see each other are never bridged: the all-reduce
|
||||
would hang.</figcaption>
|
||||
</figure>
|
||||
</div>
|
||||
|
||||
<section id="what">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>What it does</h2>
|
||||
<p class="intro">One static binary. It runs as the control plane, as the agent on every GPU
|
||||
node, and as the weights mirror.</p>
|
||||
</div>
|
||||
<div class="facts">
|
||||
<div class="fact"><h3>OpenAI API, your hardware</h3>
|
||||
<p>Point any OpenAI client at it. Only the base URL changes. Streaming, tool calls,
|
||||
<code>/v1/models</code> and <code>/v1/responses</code> work as your client expects.</p></div>
|
||||
<div class="fact"><h3>Placement that knows the fabric</h3>
|
||||
<p>Agents report GPU class, memory, NVLink peers and InfiniBand ports. Replicas go where they
|
||||
fit, including ones larger than one machine, on the GPUs with the most NVLink pairs.</p></div>
|
||||
<div class="fact"><h3>Models load when asked for</h3>
|
||||
<p>Keep more models in the catalog than your GPUs hold. The first request starts one, the
|
||||
chat page shows it loading, and it stops again after the idle timeout.</p></div>
|
||||
<div class="fact"><h3>Keys, budgets, rate limits</h3>
|
||||
<p>Organisations, teams and projects, each with limits. Priority bands decide who waits
|
||||
when the cluster is full. Managers see their organisation's spend; people see their own.</p></div>
|
||||
<div class="fact"><h3>Editors connect from the account page {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||||
<p>Pick Continue, Codex or Cline and a model. The page makes a key and hands over the config,
|
||||
filled in, or a one-line command that writes it.</p></div>
|
||||
<div class="fact"><h3>A chat page for quick questions {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||||
<p>Markdown and maths rendered, stop mid-reply, tokens per second and time to first token,
|
||||
a system prompt and a temperature. The gateway keeps no conversations.</p></div>
|
||||
<div class="fact"><h3>A mirror you browse {{if .Site}}<span class="next">next release</span>{{end}}</h3>
|
||||
<p>Explore what is trending on Hugging Face, read a model card under its row, and download
|
||||
once for the whole cluster. Only what your nodes can run is shown by default.</p></div>
|
||||
<div class="fact"><h3>A dashboard with nothing to set up</h3>
|
||||
<p><code>/admin</code> shows capacity per GPU class, CPU and memory per node, who is using
|
||||
what, each node's page and logs. It fits a phone. Prometheus is optional.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="fabrics">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>InfiniBand and NVLink, several of each</h2>
|
||||
<p class="intro">A site may have independent InfiniBand fabrics, eight-GPU HGX boards and NVL72
|
||||
racks side by side. llmgw tells them apart by what the adapters report, not by what a config
|
||||
file calls them.</p>
|
||||
{{if .Site}}<p class="intro"><span class="next">next release</span> Everything in this section is on main
|
||||
and ships in the next release.</p>{{end}}
|
||||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/fabrics.md">Fabrics and NVLink guide</a></p>
|
||||
</div>
|
||||
<div class="fab-grid">
|
||||
<div class="order" aria-label="Where a replica spanning nodes is placed, in order">
|
||||
<div><b>1</b><h3>One NVLink domain</h3><p>NVL72 or NVLink Switch. Every card joined, IMEX up.
|
||||
Shards get <code>NCCL_MNNVL_ENABLE=1</code>.</p></div>
|
||||
<div><b>2</b><h3>One InfiniBand or RoCE fabric</h3><p>Nodes under one leaf first, nodes with a
|
||||
degraded cable last.</p></div>
|
||||
<div><b class="x">×</b><h3>Never across two</h3><p>Two fabrics given one name are placed as
|
||||
two, and the cluster page says so.</p></div>
|
||||
</div>
|
||||
<div>
|
||||
<h3>Check a node in one command</h3>
|
||||
<div class="term" aria-label="Example output of llmgw doctor"><pre>$ sudo llmgw doctor
|
||||
<span class="fail">FAIL</span> nvlink this machine has NVSwitch but nvidia-fabricmanager is inactive
|
||||
<span class="dim">→ systemctl enable --now nvidia-fabricmanager</span>
|
||||
<span class="warn">WARN</span> fabric nvidia_peermem is not loaded: GPUDirect RDMA is off
|
||||
<span class="dim">→ modprobe nvidia_peermem</span>
|
||||
<span class="ok">OK</span> gpu 8 GPUs: 8× NVIDIA H100 80GB HBM3
|
||||
<span class="ok">OK</span> nvlink NVLink groups: [0 1 2 3 4 5 6 7]
|
||||
<span class="ok">OK</span> fabric mlx5_0/1 ACTIVE at 400 Gb/s (InfiniBand), subnet manager 0x248a07030030f0c0
|
||||
<span class="ok">OK</span> fabric subnet managers: 0x…f0c0 (master, priority 15), 0x…f1d4 (standby, priority 14)
|
||||
<span class="ok">OK</span> agent NCCL bootstraps over ens3 (10.1.0.21)</pre></div>
|
||||
</div>
|
||||
<ul class="watch">
|
||||
<li><strong>Subnet managers, watched</strong>Master, priority, standbys and failovers per fabric.
|
||||
A fabric with no standby is flagged. llmgw does not run one.</li>
|
||||
<li><strong>A map of each fabric</strong><code>/admin/fabrics</code> draws switches in tiers,
|
||||
cables coloured by rate, degraded ones in red, and live traffic on your adapters.</li>
|
||||
<li><strong>NVLink domains as racks</strong><code>/admin/nvlink</code> shows which trays are
|
||||
joined, whose IMEX is down, which link runs degraded.</li>
|
||||
<li><strong>Ports re-read every heartbeat</strong>A port that goes down takes its fabric off the
|
||||
node within seconds. Counters and rates are exported to Prometheus.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="install">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>From nothing to a first request</h2>
|
||||
<p class="intro">Debian, Ubuntu or RHEL 10. NVIDIA drivers and the InfiniBand stack are
|
||||
yours to install; llmgw does the rest.</p>
|
||||
<p class="intro"><a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/install.md">The full guide</a>
|
||||
covers the mirror, GPUs and what to do when something does not work.</p>
|
||||
</div>
|
||||
<ol class="steps">
|
||||
<li>
|
||||
<h3>Put the binary on every machine</h3>
|
||||
<p>Static, no dependencies, the same file everywhere.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}} <span class="c"># from the releases page</span>{{end}}
|
||||
U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
|
||||
curl -fsSLO $U/llmgw-$VERSION-linux-amd64
|
||||
curl -fsSL $U/checksums.txt | sha256sum -c --ignore-missing
|
||||
sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw</code></pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Set up the control plane</h3>
|
||||
<p>Creates the user, the certificate authority, the certificate, <code>server.yaml</code>
|
||||
and your administrator account, then starts the service. Safe to run again.
|
||||
Add <code>--host</code> for any name nodes use that the machine cannot see itself.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>sudo llmgw server init --admin you@example.com</code></pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Add a node</h3>
|
||||
<p>From the control plane, which holds the authority. The node gets its certificate,
|
||||
the binary and the agent, and installs its engine by itself.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>deploy/bootstrap/onboard-node.sh --node gpu01 --addr 10.0.0.141 \
|
||||
--server https://10.0.0.140:7443 --fabric ib-a:mlx5_0,mlx5_1</code></pre></div>
|
||||
<div class="term" aria-label="Example output of llmgw ctl nodes"><pre>$ llmgw ctl nodes
|
||||
NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
|
||||
gpu01 <span class="ok">ready</span> 8/8 free 503G h100-80g ib-a 0 2s</pre></div>
|
||||
</li>
|
||||
<li>
|
||||
<h3>Add a model and send a request</h3>
|
||||
<p>No restart. If it does not place, <code>llmgw ctl explain-placement</code> names every
|
||||
node and why it was turned down.</p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>llmgw ctl model add -f qwen-coder-3b.yaml
|
||||
|
||||
curl https://llm.example.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "coder", "messages": [{"role": "user", "content": "hello"}]}'</code></pre></div>
|
||||
</li>
|
||||
</ol>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="proxy">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Behind a reverse proxy</h2>
|
||||
<p class="intro">Port 8080 is plain HTTP. Put it behind Caddy or nginx with a public
|
||||
certificate. Only 8080 goes through the proxy: agents reach 7443 directly, because it is
|
||||
mutual TLS with the cluster's own authority.</p>
|
||||
<p class="intro">Bind llmgw to loopback when the proxy is on the same machine:
|
||||
<code>listen: "127.0.0.1:8080"</code>.</p>
|
||||
</div>
|
||||
<div class="tabs">
|
||||
<input type="radio" name="proxy" id="t-caddy" checked>
|
||||
<input type="radio" name="proxy" id="t-nginx">
|
||||
<div class="tablist" role="presentation">
|
||||
<label for="t-caddy">Caddy</label>
|
||||
<label for="t-nginx">nginx</label>
|
||||
</div>
|
||||
<div class="panel p-caddy">
|
||||
<p class="meta"><span>Obtains and renews its own certificate. Replace <code>llm.example.com</code>.</span>
|
||||
<a href="{{.Download}}/Caddyfile">Caddyfile</a></p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Caddy}}</code></pre></div>
|
||||
</div>
|
||||
<div class="panel p-nginx">
|
||||
<p class="meta"><span>Certificate from <code>certbot --nginx</code>. Works on nginx 1.18 and later.</span>
|
||||
<a href="{{.Download}}/nginx.conf">nginx.conf</a></p>
|
||||
<div class="code"><button class="copy" type="button">Copy</button><pre><code>{{.Nginx}}</code></pre></div>
|
||||
</div>
|
||||
<ul class="rules">
|
||||
<li><strong>Streams are not buffered</strong>Tokens reach the client as they are generated, not in one burst at the end.</li>
|
||||
<li><strong>The read timeout outlasts the model</strong>nginx's 60-second default turns a slow first token into a 504.</li>
|
||||
<li><strong>Forwarded headers are set, not passed on</strong>llmgw marks session cookies Secure from <code>X-Forwarded-Proto</code> and shows <code>X-Forwarded-Host</code> as the base URL.</li>
|
||||
<li><strong>Body limit matches the router</strong>32 MiB, so an oversized request is refused before it is uploaded.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="download">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Download</h2>
|
||||
<p class="intro">Built by CI from the tag and checked against <code>checksums.txt</code>.
|
||||
Every release is on the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>
|
||||
with its changelog.</p>
|
||||
</div>
|
||||
<div>
|
||||
{{if .Release}}
|
||||
<div class="ver"><span class="tag">{{.Release}}</span>{{with .Date}}<span class="eyebrow">Released {{.}}</span>{{end}}</div>
|
||||
<div class="dl-wrap">
|
||||
<table class="dl">
|
||||
<thead><tr><th>For</th><th>File</th><th>Notes</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>Linux x86-64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64"><code>llmgw-{{$.Release}}-linux-amd64</code></a></td><td class="note">Static. Runs on any libc.</td></tr>
|
||||
<tr><td>Linux ARM64</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-arm64"><code>llmgw-{{$.Release}}-linux-arm64</code></a></td><td class="note">Static.</td></tr>
|
||||
<tr><td>Linux x86-64, NVLink</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-linux-amd64-nvml"><code>llmgw-{{$.Release}}-linux-amd64-nvml</code></a></td><td class="note">Sees NVLink topology through NVML. For PCIe boxes with NVLink bridges. Needs the driver.</td></tr>
|
||||
<tr><td>macOS Apple silicon</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-arm64"><code>llmgw-{{$.Release}}-darwin-arm64</code></a></td><td class="note">For <code>llmgw ctl</code> and trying it on a laptop.</td></tr>
|
||||
<tr><td>macOS Intel</td><td><a href="{{$.Download}}/llmgw-{{$.Release}}-darwin-amd64"><code>llmgw-{{$.Release}}-darwin-amd64</code></a></td><td class="note"></td></tr>
|
||||
<tr><td>Caddy</td><td><a href="{{$.Download}}/Caddyfile"><code>Caddyfile</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||||
<tr><td>nginx</td><td><a href="{{$.Download}}/nginx.conf"><code>nginx.conf</code></a></td><td class="note">Reverse proxy for port 8080.</td></tr>
|
||||
<tr><td>Checksums</td><td><a href="{{$.Download}}/checksums.txt"><code>checksums.txt</code></a></td><td class="note">SHA-256 of every file above.</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}
|
||||
<p class="intro" style="margin-top:0">This is a development build ({{.Version}}). Take a release
|
||||
from the <a href="https://gitea.mynodes.xyz/ilker/llmgw/releases">releases page</a>, where every
|
||||
file is listed with its checksum.</p>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="status">
|
||||
<div class="sec-head">
|
||||
<div>
|
||||
<h2>Where it stands</h2>
|
||||
<p class="intro">Pre-1.0: configuration formats may change between minor versions.
|
||||
This is what has run on real hardware, and what has not yet.</p>
|
||||
</div>
|
||||
<div class="status">
|
||||
<div class="proven">
|
||||
<h3>Run on real clusters</h3>
|
||||
<ul>
|
||||
<li>Single-node replicas on real GPUs and on CPU</li>
|
||||
<li>On-demand loading and idle shutdown</li>
|
||||
<li>Keys, budgets, rate limits and priority bands</li>
|
||||
<li>Rolling upgrades with no dropped replicas</li>
|
||||
<li>SQLite and PostgreSQL stores</li>
|
||||
<li>Chat, editor setup and the Responses API against a six-node test cluster</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="open">
|
||||
<h3>Not yet proven</h3>
|
||||
<ul>
|
||||
<li>InfiniBand and NVLink on a real site: written and tested against the real output of
|
||||
<code>sminfo</code>, <code>ibnetdiscover</code> and <code>nvidia-smi</code>, but not yet run
|
||||
on the hardware. Run <code>llmgw doctor --json</code> on a few nodes first.</li>
|
||||
<li>Two control planes over long periods</li>
|
||||
<li>vLLM and SGLang are not installed by the node yet; name their paths yourself</li>
|
||||
<li>Nodes do not share downloaded weights; the mirror keeps each download local</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<span>llmgw {{.Version}}</span>
|
||||
<span><a href="https://gitea.mynodes.xyz/ilker/llmgw">Source</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/CHANGELOG.md">Changelog</a> ·
|
||||
<a href="https://gitea.mynodes.xyz/ilker/llmgw/src/branch/main/docs/reference.md">Configuration reference</a></span>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
document.querySelectorAll('.copy').forEach(function (b) {
|
||||
b.addEventListener('click', function () {
|
||||
var text = b.dataset.copy || b.parentElement.querySelector('pre').innerText;
|
||||
var done = function (label) { b.textContent = label; setTimeout(function () { b.textContent = 'Copy'; }, 1600); };
|
||||
if (navigator.clipboard && navigator.clipboard.writeText) {
|
||||
navigator.clipboard.writeText(text).then(function () { done('Copied'); }, function () { select(); });
|
||||
} else { select(); }
|
||||
function select() {
|
||||
var el = b.dataset.copy ? b.parentElement.querySelector('code') : b.parentElement.querySelector('pre');
|
||||
var r = document.createRange(); r.selectNodeContents(el);
|
||||
var s = window.getSelection(); s.removeAllRanges(); s.addRange(r);
|
||||
done('Selected');
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
{{if not .Site}}</body></html>{{end}}{{end}}
|
||||
@@ -185,7 +185,7 @@ is listed here — there is nothing llmgw-specific to install.</p>
|
||||
<li><code>GET {{.BaseURL}}/me</code><span>its limits, and what it has spent</span></li>
|
||||
</ul>
|
||||
|
||||
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
|
||||
<footer><span><a href="/about">llmgw {{.Version}}</a></span><span>{{.Now}}</span></footer>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
|
||||
Reference in new issue
Block a user