diff --git a/CHANGELOG.md b/CHANGELOG.md index 278ba2a..a98eb61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -150,6 +150,14 @@ The account page (was "Your keys") shows the last thirty days of a person's own use and, for each place they spend, a bar for every budget that applies — their own, or a project's shared with others, said as such. +### The project's website, in every gateway + +`/about` is llmgw's own page: what it does, InfiniBand and NVLink, installing, +the reverse proxy configurations, and the release's files to download. It is +served without a key like `/`, whose footer links to it, and says nothing about +the installation. Its fonts are in the binary, so it makes no request elsewhere. +The published website is rendered from the same template. + ### The admin pages - The cluster page leads with capacity, nodes, models and replicas, then usage; diff --git a/deploy/proxy/README.md b/deploy/proxy/README.md index e82424c..71bccd4 100644 --- a/deploy/proxy/README.md +++ b/deploy/proxy/README.md @@ -10,6 +10,9 @@ put it behind a proxy that holds a publicly trusted certificate. Both are also attached to every release. +The `/about` page shows both, from copies in `internal/web/proxy`; a test fails +when those differ from these, so change them here and copy them over. + Four things both files get right, and any other proxy has to as well: - **Streaming is not buffered.** Replies are server-sent events; a buffering diff --git a/internal/router/handlers.go b/internal/router/handlers.go index ca6638a..f71a526 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -117,6 +117,9 @@ func (r *Router) Handler() http.Handler { mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil, r.accounts != nil && r.accounts.SignupOpen(), r.landing.Contact, r.landing.Notice)) + // What llmgw is and how to install it: about the software, not this + // installation, so it needs no key either. + mux.HandleFunc("GET /about", web.About(version.Version)) mux.Handle("GET /_static/", web.Static()) mux.HandleFunc("GET /me", r.handleMe) // Somebody has to be able to say "forget that", and a retention window is @@ -199,7 +202,7 @@ func (r *Router) Handler() http.Handler { // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. return auth.Middleware(r.auth, r.log, - "/health", "/ready", "/", "/_static/", "/claim/", + "/health", "/ready", "/", "/about", "/_static/", "/claim/", "/signup", "/login", "/logout", "/profile", "/profile/", "/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux) } diff --git a/internal/web/about.go b/internal/web/about.go new file mode 100644 index 0000000..71ede4d --- /dev/null +++ b/internal/web/about.go @@ -0,0 +1,79 @@ +package web + +import ( + "embed" + "io" + "net/http" + "regexp" + "strings" +) + +// The page about llmgw itself: what it is, how to install it, where to get +// it. It is the project's website, carried by every gateway so an operator +// has it where they already are, and served without a key like the landing +// page — it says nothing about this installation, only about the software. +// +// The same template renders the published website (Site), which differs in +// three ways: fonts from Google rather than from this binary, no link back to +// a gateway, and features on main marked as coming in the next release. + +// proxyConfigs are the reverse proxy configurations the page shows. They are +// copies of deploy/proxy, which the release attaches; embed cannot reach +// outside this package, and TestAboutProxyConfigsMatchDeploy keeps the two +// the same. +// +//go:embed proxy +var proxyConfigs embed.FS + +// AboutData is what the page is rendered with. +type AboutData struct { + // Version is what this build calls itself; Release is the same when it is + // a released version, and empty for a development build, which has no + // files to download. + Version, Release string + // Download is where the release's files are, or for a development build + // the proxy configurations on main. + Download string + // Date is the release date, when it is known. A binary does not know it. + Date string + Caddy, Nginx string + Site bool +} + +// releaseTag is a version the releases page has files for. +var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+$`) + +const repoURL = "https://gitea.mynodes.xyz/ilker/llmgw" + +// NewAboutData fills the page's data for a version. +func NewAboutData(version string) AboutData { + d := AboutData{Version: version, Download: repoURL + "/raw/branch/main/deploy/proxy"} + if releaseTag.MatchString(version) { + d.Release = version + d.Download = repoURL + "/releases/download/" + version + } + d.Caddy = proxyConfig("Caddyfile") + d.Nginx = proxyConfig("nginx.conf") + return d +} + +func proxyConfig(name string) string { + b, err := proxyConfigs.ReadFile("proxy/" + name) + if err != nil { + panic("web: the proxy configuration " + name + " is missing from the binary: " + err.Error()) + } + return strings.TrimRight(string(b), "\n") +} + +// RenderAbout writes the page, for the website's build as much as for About. +func RenderAbout(w io.Writer, d AboutData) error { + return tmpl.ExecuteTemplate(w, "about", d) +} + +// About serves the page at /about. +func About(version string) http.HandlerFunc { + d := NewAboutData(version) + return func(w http.ResponseWriter, r *http.Request) { + render(w, "about", d) + } +} diff --git a/internal/web/about_test.go b/internal/web/about_test.go new file mode 100644 index 0000000..ca151d7 --- /dev/null +++ b/internal/web/about_test.go @@ -0,0 +1,83 @@ +package web + +import ( + "bytes" + "net/http" + "os" + "strings" + "testing" +) + +func TestAboutPageForARelease(t *testing.T) { + code, body := get(t, About("v1.2.3"), "http://llmgw.example/about", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + for _, want := range []string{ + "", + repoURL + "/releases/download/v1.2.3/llmgw-v1.2.3-linux-amd64", + "VERSION=v1.2.3", + "reverse_proxy", // the Caddyfile, inline + "proxy_read_timeout", // and nginx's + `This gateway`, + "/_static/fonts/archivo-latin.woff2?v=", + } { + if !strings.Contains(body, want) { + t.Errorf("the page does not have %q", want) + } + } + // Served to anyone, on networks that may reach nothing: the page sends no + // browser elsewhere for its fonts, and does not announce unreleased work + // in a binary that already has it. + for _, not := range []string{"fonts.googleapis.com", "next release"} { + if strings.Contains(body, not) { + t.Errorf("the page has %q", not) + } + } +} + +func TestAboutPageForADevelopmentBuild(t *testing.T) { + _, body := get(t, About("v1.2.3-4-gabcdef"), "http://llmgw.example/about", nil) + if strings.Contains(body, "/releases/download/v1.2.3-4") || strings.Contains(body, `class="dl"`) { + t.Error("a development build links to release files that do not exist") + } + if !strings.Contains(body, "development build (v1.2.3-4-gabcdef)") { + t.Error("a development build does not say what it is") + } +} + +func TestAboutPageAsTheWebsite(t *testing.T) { + d := NewAboutData("v1.2.3") + d.Site, d.Date = true, "1 January 2027" + var b bytes.Buffer + if err := RenderAbout(&b, d); err != nil { + t.Fatal(err) + } + body := b.String() + for _, want := range []string{"fonts.googleapis.com", "Released 1 January 2027", "next release"} { + if !strings.Contains(body, want) { + t.Errorf("the website does not have %q", want) + } + } + for _, not := range []string{"", "This gateway", "/_static/"} { + if strings.Contains(body, not) { + t.Errorf("the website has %q", not) + } + } +} + +func TestAboutProxyConfigsMatchDeploy(t *testing.T) { + for _, name := range []string{"Caddyfile", "nginx.conf"} { + want, err := os.ReadFile("../../deploy/proxy/" + name) + if err != nil { + t.Fatal(err) + } + got, err := proxyConfigs.ReadFile("proxy/" + name) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("internal/web/proxy/%s differs from deploy/proxy/%[1]s: copy it over", name) + } + } +} diff --git a/internal/web/proxy/Caddyfile b/internal/web/proxy/Caddyfile new file mode 100644 index 0000000..a27034e --- /dev/null +++ b/internal/web/proxy/Caddyfile @@ -0,0 +1,41 @@ +# llmgw behind Caddy: the client-facing port (8080) on a public name, with a +# certificate Caddy obtains and renews by itself. +# +# Replace llm.example.com, point its DNS at this machine, open 80 and 443, and +# run: caddy run --config /etc/caddy/Caddyfile +# +# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's +# own authority and must reach llmgw directly; 9090 and 9091 stay private. + +llm.example.com { + # The router reads at most 32 MiB of body (router.max_body_bytes). Refusing + # more here gives the client a 413 before the upload, not after it. + request_body { + max_size 32MB + } + + reverse_proxy 127.0.0.1:8080 { + # Tokens are written the moment they are generated. Without this a + # streamed reply can arrive in bursts, and a client measuring time to + # first token measures the proxy instead. + flush_interval -1 + + # Caddy sets X-Forwarded-Proto and X-Forwarded-Host itself and, with + # no trusted_proxies, discards whatever the client sent. llmgw relies + # on that: the proto decides whether session cookies are Secure, and + # the host is the base URL the landing page tells people to use. + # Behind a load balancer of your own, name it in trusted_proxies. + + transport http { + # A long generation can be silent for a while before the first + # token. llmgw has its own limits (timeouts.*); this must not be + # the shorter one. + response_header_timeout 15m + } + } + + # No `encode` on purpose: compressing a stream buffers it. + + # Access log to stderr, which under systemd is the journal. + log +} diff --git a/internal/web/proxy/nginx.conf b/internal/web/proxy/nginx.conf new file mode 100644 index 0000000..29157aa --- /dev/null +++ b/internal/web/proxy/nginx.conf @@ -0,0 +1,63 @@ +# llmgw behind nginx: the client-facing port (8080) on a public name. +# +# Put this in /etc/nginx/sites-available/llmgw (or conf.d/llmgw.conf), replace +# llm.example.com, and get a certificate — certbot --nginx -d llm.example.com +# fills in the ssl_ lines below. Then: nginx -t && systemctl reload nginx +# +# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's +# own authority and must reach llmgw directly; 9090 and 9091 stay private. + +upstream llmgw { + server 127.0.0.1:8080; + keepalive 32; +} + +server { + listen 80; + listen [::]:80; + server_name llm.example.com; + return 301 https://$host$request_uri; +} + +server { + # `listen ... http2` rather than `http2 on;`: the latter needs nginx + # 1.25.1, and Ubuntu 22.04 and Debian 12 ship older ones. Newer nginx + # accepts this form with a deprecation warning. + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name llm.example.com; + + ssl_certificate /etc/letsencrypt/live/llm.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/llm.example.com/privkey.pem; + + # The router reads at most 32 MiB of body (router.max_body_bytes). + client_max_body_size 32m; + + location / { + proxy_pass http://llmgw; + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Set, never appended: llmgw decides from X-Forwarded-Proto whether + # session cookies are Secure, and builds the base URL the landing page + # shows from X-Forwarded-Host. A value passed through from the client + # would be the client's choice. + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Streamed replies: tokens go out as they are generated. Buffering + # turns a stream into one late burst, and gzip buffers too. + proxy_buffering off; + proxy_cache off; + gzip off; + + # nginx's default is 60 seconds between two reads from upstream. A + # large model can think longer than that before its first token, and + # the client then sees a 504 for a request that was about to succeed. + # llmgw has its own limits (timeouts.*); this must not be the shorter. + proxy_read_timeout 15m; + proxy_send_timeout 15m; + } +} diff --git a/internal/web/static/fonts/NOTICE b/internal/web/static/fonts/NOTICE new file mode 100644 index 0000000..fa218e6 --- /dev/null +++ b/internal/web/static/fonts/NOTICE @@ -0,0 +1,17 @@ +Fonts for the /about page, vendored rather than loaded from Google Fonts: this +gateway makes no outbound requests of its own, and a page served to anyone who +can reach it should not send their browser to a third party either. + + archivo-latin.woff2 Archivo, variable (width 112–125%, weight 600–800) + plex-sans-latin.woff2 IBM Plex Sans, variable weight + plex-mono-400-latin.woff2 IBM Plex Mono Regular + plex-mono-500-latin.woff2 IBM Plex Mono Medium + +The Latin subsets as Google Fonts serves them, unmodified. Both families are +under the SIL Open Font License 1.1: OFL-Archivo.txt, OFL-IBMPlex.txt. + +sha256: + 4c98b9d490d1698ec95f2ff17a6c7d0e72691864c0c5d7bc2a2c161b45afe5ad archivo-latin.woff2 + c36f509c0a8f9f85f29cb44bc8701d8a9e0b14c499e77a884f789ead7093a7ac plex-mono-400-latin.woff2 + a76f53ca6612e7b3828eec2311098675b7f9849ae4169a8bcef6302aec02a6c0 plex-mono-500-latin.woff2 + 056e4e2459f57a0033c8c9c844ff19d6e42ac8602027803d4345823bcc939818 plex-sans-latin.woff2 diff --git a/internal/web/static/fonts/OFL-Archivo.txt b/internal/web/static/fonts/OFL-Archivo.txt new file mode 100644 index 0000000..8597481 --- /dev/null +++ b/internal/web/static/fonts/OFL-Archivo.txt @@ -0,0 +1,93 @@ +Copyright 2020 The Archivo Project Authors (https://github.com/Omnibus-Type/Archivo) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/internal/web/static/fonts/OFL-IBMPlex.txt b/internal/web/static/fonts/OFL-IBMPlex.txt new file mode 100644 index 0000000..c35c4c6 --- /dev/null +++ b/internal/web/static/fonts/OFL-IBMPlex.txt @@ -0,0 +1,93 @@ +Copyright © 2017 IBM Corp. with Reserved Font Name "Plex" + +This Font Software is licensed under the SIL Open Font License, Version 1.1. + +This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/internal/web/static/fonts/archivo-latin.woff2 b/internal/web/static/fonts/archivo-latin.woff2 new file mode 100644 index 0000000..7af3f7d Binary files /dev/null and b/internal/web/static/fonts/archivo-latin.woff2 differ diff --git a/internal/web/static/fonts/plex-mono-400-latin.woff2 b/internal/web/static/fonts/plex-mono-400-latin.woff2 new file mode 100644 index 0000000..52b6c75 Binary files /dev/null and b/internal/web/static/fonts/plex-mono-400-latin.woff2 differ diff --git a/internal/web/static/fonts/plex-mono-500-latin.woff2 b/internal/web/static/fonts/plex-mono-500-latin.woff2 new file mode 100644 index 0000000..3308bce Binary files /dev/null and b/internal/web/static/fonts/plex-mono-500-latin.woff2 differ diff --git a/internal/web/static/fonts/plex-sans-latin.woff2 b/internal/web/static/fonts/plex-sans-latin.woff2 new file mode 100644 index 0000000..e5ebabb Binary files /dev/null and b/internal/web/static/fonts/plex-sans-latin.woff2 differ diff --git a/internal/web/templates/about.html b/internal/web/templates/about.html new file mode 100644 index 0000000..d860719 --- /dev/null +++ b/internal/web/templates/about.html @@ -0,0 +1,531 @@ +{{define "about"}}{{if not .Site}} + + + +{{end}}llmgw +{{if .Site}} + + +{{end}} +{{if not .Site}}{{end}} + +
+
+ + + llmgw + + +
+ +
+
+

Self-hosted LLM gateway · {{.Version}}

+

One endpoint in front of your own GPUs.

+

llmgw runs vLLM, SGLang and llama.cpp across your servers and serves them + behind one OpenAI-compatible API. It knows which GPUs share an NVLink domain or an InfiniBand + fabric, and a model split across machines never straddles two.

+
+ Download{{with .Release}} {{.}}{{end}} + Install guide +
+
+ $ sudo llmgw server init --admin you@example.com + +
+
+ +
+ + Clients reach one llmgw address. llmgw places a model's shards on GPUs inside one fabric at a time. + + + + openai sdk + + editor agent + + curl + + + + + + + llmgw + :8080 + + + + + nvl72 · gb200 + + + + + + + + + ib-b · h100-80g + + + + + + + + + + + + + +
A replica spanning nodes goes into one NVLink domain if one has room, otherwise onto + one InfiniBand fabric. Two fabrics that cannot see each other are never bridged: the all-reduce + would hang.
+
+
+ +
+
+
+

What it does

+

One static binary. It runs as the control plane, as the agent on every GPU + node, and as the weights mirror.

+
+
+

OpenAI API, your hardware

+

Point any OpenAI client at it. Only the base URL changes. Streaming, tool calls, + /v1/models and /v1/responses work as your client expects.

+

Placement that knows the fabric

+

Agents report GPU class, memory, NVLink peers and InfiniBand ports. Replicas go where they + fit, including ones larger than one machine, on the GPUs with the most NVLink pairs.

+

Models load when asked for

+

Keep more models in the catalog than your GPUs hold. The first request starts one, the + chat page shows it loading, and it stops again after the idle timeout.

+

Keys, budgets, rate limits

+

Organisations, teams and projects, each with limits. Priority bands decide who waits + when the cluster is full. Managers see their organisation's spend; people see their own.

+

Editors connect from the account page {{if .Site}}next release{{end}}

+

Pick Continue, Codex or Cline and a model. The page makes a key and hands over the config, + filled in, or a one-line command that writes it.

+

A chat page for quick questions {{if .Site}}next release{{end}}

+

Markdown and maths rendered, stop mid-reply, tokens per second and time to first token, + a system prompt and a temperature. The gateway keeps no conversations.

+

A mirror you browse {{if .Site}}next release{{end}}

+

Explore what is trending on Hugging Face, read a model card under its row, and download + once for the whole cluster. Only what your nodes can run is shown by default.

+

A dashboard with nothing to set up

+

/admin shows capacity per GPU class, CPU and memory per node, who is using + what, each node's page and logs. It fits a phone. Prometheus is optional.

+
+
+
+ +
+
+
+

InfiniBand and NVLink, several of each

+

A site may have independent InfiniBand fabrics, eight-GPU HGX boards and NVL72 + racks side by side. llmgw tells them apart by what the adapters report, not by what a config + file calls them.

+ {{if .Site}}

next release Everything in this section is on main + and ships in the next release.

{{end}} +

Fabrics and NVLink guide

+
+
+
+
1

One NVLink domain

NVL72 or NVLink Switch. Every card joined, IMEX up. + Shards get NCCL_MNNVL_ENABLE=1.

+
2

One InfiniBand or RoCE fabric

Nodes under one leaf first, nodes with a + degraded cable last.

+
×

Never across two

Two fabrics given one name are placed as + two, and the cluster page says so.

+
+
+

Check a node in one command

+ 
$ sudo llmgw doctor
+FAIL  nvlink  this machine has NVSwitch but nvidia-fabricmanager is inactive
+              → systemctl enable --now nvidia-fabricmanager
+WARN  fabric  nvidia_peermem is not loaded: GPUDirect RDMA is off
+              → modprobe nvidia_peermem
+OK    gpu     8 GPUs: 8× NVIDIA H100 80GB HBM3
+OK    nvlink  NVLink groups: [0 1 2 3 4 5 6 7]
+OK    fabric  mlx5_0/1 ACTIVE at 400 Gb/s (InfiniBand), subnet manager 0x248a07030030f0c0
+OK    fabric  subnet managers: 0x…f0c0 (master, priority 15), 0x…f1d4 (standby, priority 14)
+OK    agent   NCCL bootstraps over ens3 (10.1.0.21)
+
+
    +
  • Subnet managers, watchedMaster, priority, standbys and failovers per fabric. + A fabric with no standby is flagged. llmgw does not run one.
    • +
  • A map of each fabric/admin/fabrics draws switches in tiers, + cables coloured by rate, degraded ones in red, and live traffic on your adapters.
    • +
  • NVLink domains as racks/admin/nvlink shows which trays are + joined, whose IMEX is down, which link runs degraded.
    • +
  • Ports re-read every heartbeatA port that goes down takes its fabric off the + node within seconds. Counters and rates are exported to Prometheus.
    • +
+
+
+
+ +
+
+
+

From nothing to a first request

+

Debian, Ubuntu or RHEL 10. NVIDIA drivers and the InfiniBand stack are + yours to install; llmgw does the rest.

+

The full guide + covers the mirror, GPUs and what to do when something does not work.

+
+
    +
  1. +

    Put the binary on every machine

    +

    Static, no dependencies, the same file everywhere.

    + 
    VERSION={{or .Release "vX.Y.Z"}}{{if not .Release}}   # from the releases page{{end}}
+U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
+curl -fsSLO $U/llmgw-$VERSION-linux-amd64
+curl -fsSL  $U/checksums.txt | sha256sum -c --ignore-missing
+sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw
    +
    2. +
  2. +

    Set up the control plane

    +

    Creates the user, the certificate authority, the certificate, server.yaml + and your administrator account, then starts the service. Safe to run again. + Add --host for any name nodes use that the machine cannot see itself.

    + 
    sudo llmgw server init --admin you@example.com
    +
    3. +
  3. +

    Add a node

    +

    From the control plane, which holds the authority. The node gets its certificate, + the binary and the agent, and installs its engine by itself.

    + 
    deploy/bootstrap/onboard-node.sh --node gpu01 --addr 10.0.0.141 \
+  --server https://10.0.0.140:7443 --fabric ib-a:mlx5_0,mlx5_1
    + 
    $ llmgw ctl nodes
+NODE   STATE  GPUS      MEM   CLASSES   FABRICS  REPLICAS  LAST SEEN
+gpu01  ready  8/8 free  503G  h100-80g  ib-a     0         2s
    +
    4. +
  4. +

    Add a model and send a request

    +

    No restart. If it does not place, llmgw ctl explain-placement names every + node and why it was turned down.

    + 
    llmgw ctl model add -f qwen-coder-3b.yaml
+
+curl https://llm.example.com/v1/chat/completions \
+  -H "Authorization: Bearer $LLMGW_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "coder", "messages": [{"role": "user", "content": "hello"}]}'
    +
    5. +
+
+
+ +
+
+
+

Behind a reverse proxy

+

Port 8080 is plain HTTP. Put it behind Caddy or nginx with a public + certificate. Only 8080 goes through the proxy: agents reach 7443 directly, because it is + mutual TLS with the cluster's own authority.

+

Bind llmgw to loopback when the proxy is on the same machine: + listen: "127.0.0.1:8080".

+
+
+ + + +
+

Obtains and renews its own certificate. Replace llm.example.com. + Caddyfile

+ 
{{.Caddy}}
+
+
+

Certificate from certbot --nginx. Works on nginx 1.18 and later. + nginx.conf

+ 
{{.Nginx}}
+
+
    +
  • Streams are not bufferedTokens reach the client as they are generated, not in one burst at the end.
    • +
  • The read timeout outlasts the modelnginx's 60-second default turns a slow first token into a 504.
    • +
  • Forwarded headers are set, not passed onllmgw marks session cookies Secure from X-Forwarded-Proto and shows X-Forwarded-Host as the base URL.
    • +
  • Body limit matches the router32 MiB, so an oversized request is refused before it is uploaded.
    • +
+
+
+
+ +
+
+
+

Download

+

Built by CI from the tag and checked against checksums.txt. + Every release is on the releases page + with its changelog.

+
+
+{{if .Release}} +
{{.Release}}{{with .Date}}Released {{.}}{{end}}
+
+ + + + + + + + + + + + +
ForFileNotes
Linux x86-64llmgw-{{$.Release}}-linux-amd64Static. Runs on any libc.
Linux ARM64llmgw-{{$.Release}}-linux-arm64Static.
Linux x86-64, NVLinkllmgw-{{$.Release}}-linux-amd64-nvmlSees NVLink topology through NVML. For PCIe boxes with NVLink bridges. Needs the driver.
macOS Apple siliconllmgw-{{$.Release}}-darwin-arm64For llmgw ctl and trying it on a laptop.
macOS Intelllmgw-{{$.Release}}-darwin-amd64
CaddyCaddyfileReverse proxy for port 8080.
nginxnginx.confReverse proxy for port 8080.
Checksumschecksums.txtSHA-256 of every file above.
+
+ {{else}} +

This is a development build ({{.Version}}). Take a release + from the releases page, where every + file is listed with its checksum.

+ {{end}} +
+
+
+ +
+
+
+

Where it stands

+

Pre-1.0: configuration formats may change between minor versions. + This is what has run on real hardware, and what has not yet.

+
+
+
+

Run on real clusters

+
    +
  • Single-node replicas on real GPUs and on CPU
    • +
  • On-demand loading and idle shutdown
    • +
  • Keys, budgets, rate limits and priority bands
    • +
  • Rolling upgrades with no dropped replicas
    • +
  • SQLite and PostgreSQL stores
    • +
  • Chat, editor setup and the Responses API against a six-node test cluster
    • +
+
+
+

Not yet proven

+
    +
  • InfiniBand and NVLink on a real site: written and tested against the real output of + sminfo, ibnetdiscover and nvidia-smi, but not yet run + on the hardware. Run llmgw doctor --json on a few nodes first.
    • +
  • Two control planes over long periods
    • +
  • vLLM and SGLang are not installed by the node yet; name their paths yourself
    • +
  • Nodes do not share downloaded weights; the mirror keeps each download local
    • +
+
+
+
+
+ + +
+ + +{{if not .Site}}{{end}}{{end}} diff --git a/internal/web/templates/landing.html b/internal/web/templates/landing.html index 1450d02..f26e95a 100644 --- a/internal/web/templates/landing.html +++ b/internal/web/templates/landing.html @@ -185,7 +185,7 @@ is listed here — there is nothing llmgw-specific to install.

  • GET {{.BaseURL}}/meits limits, and what it has spent
    • - +