Setup lets every part be chosen, each with its settings; three machines each run the gateway
CI / test (push) Failing after 30m4s
CI / test (push) Failing after 30m4s
The setup screens list the parts of an install: control plane, store, model mirror, this machine as a node, starter model, pictures and 3D, the judge, firewall, notifications, limits, on-demand turns and an offline bundle. Each says what it is or what is missing without it. Space turns a part on or off; Enter opens its settings in a dialog that explains it. A plan page lists what is installed and what is left out, and the same install as a command. Every setting is a server init flag too, written into server.yaml when init writes it. With three machines, the two database machines become control planes too, standing by (server add's steps), with an optional floating address. The mirror's management API answers all three control planes, so a standby that comes to lead can still fetch models. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
733e20924c
commit
94401e21e6
16 files changed
+2255
-483
No files matched your search
@@ -10,6 +10,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -120,10 +121,36 @@ func (in *serverInit) writeUnit(path, body string) (bool, error) {
|
||||
|
||||
func (in *serverInit) mirrorUnit() string {
|
||||
base := "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
|
||||
return mirrorUnitFor(mirrorDir, mirrorAPI, base, nil,
|
||||
if in.standbys {
|
||||
return mirrorUnitFor(in.mirrorRoot(), ":8082", base, in.mirrorAllow(),
|
||||
"'llmgw server init'. Bytes on :8081 for every node; the\n# management API for the control planes only, whichever leads.")
|
||||
}
|
||||
return mirrorUnitFor(in.mirrorRoot(), mirrorAPI, base, nil,
|
||||
"'llmgw server init'. Bytes on :8081 for every node; the\n# management API on loopback only, for the control plane beside it.")
|
||||
}
|
||||
|
||||
// localMirrorAPI is the management API of the mirror beside this control
|
||||
// plane: on loopback, or, with standbys that may lead instead, on an address
|
||||
// they reach.
|
||||
func (in *serverInit) localMirrorAPI() string {
|
||||
if in.standbys {
|
||||
return "http://" + net.JoinHostPort(in.reachableAddr(), "8082")
|
||||
}
|
||||
return "http://" + mirrorAPI
|
||||
}
|
||||
|
||||
// mirrorAllow are the control planes the mirror's management API answers
|
||||
// besides loopback: this one by its own address, and each standby.
|
||||
func (in *serverInit) mirrorAllow() []string {
|
||||
out := []string{in.reachableAddr()}
|
||||
for _, n := range in.patroniNodes {
|
||||
if !slices.Contains(out, n.IP) {
|
||||
out = append(out, n.IP)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mirrorUnitFor is a mirror's systemd unit. allow is who, besides the
|
||||
// machine itself, may use the management API.
|
||||
func mirrorUnitFor(dir, apiListen, base string, allow []string, about string) string {
|
||||
@@ -160,7 +187,7 @@ func (in *serverInit) mirrorAPIURL() string {
|
||||
if in.mirrorURL != "" {
|
||||
return in.mirrorURL
|
||||
}
|
||||
return "http://" + mirrorAPI
|
||||
return in.localMirrorAPI()
|
||||
}
|
||||
|
||||
// useMirror points the control plane at a mirror on another machine, set up
|
||||
@@ -202,15 +229,16 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Mirror.API != "" && cfg.Mirror.API != "http://"+mirrorAPI {
|
||||
if cfg.Mirror.API != "" && cfg.Mirror.API != "http://"+mirrorAPI && cfg.Mirror.API != in.localMirrorAPI() {
|
||||
in.kept("mirror at %s, as server.yaml says", cfg.Mirror.API)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(mirrorDir, 0o755); err != nil {
|
||||
dir := in.mirrorRoot()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in.chown(mirrorDir, true); err != nil {
|
||||
if err := in.chown(dir, true); err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := in.writeUnit(mirrorUnitPath, in.mirrorUnit())
|
||||
@@ -227,21 +255,25 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
|
||||
if err := in.systemctl(ctx, "restart", "llmgw-mirror"); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("model mirror in %s, serving on :8081", mirrorDir)
|
||||
in.done("model mirror in %s, serving on :8081", dir)
|
||||
} else {
|
||||
in.kept("model mirror in %s", mirrorDir)
|
||||
in.kept("model mirror in %s", dir)
|
||||
}
|
||||
|
||||
if cfg.Mirror.API == "" {
|
||||
if err := appendMirrorBlockFor(in.configPath(), "http://"+mirrorAPI,
|
||||
"# The model mirror on this machine, added by 'llmgw server init'. Its\n"+
|
||||
"# management API is on loopback, so it needs no certificate."); err != nil {
|
||||
comment := "# The model mirror on this machine, added by 'llmgw server init'. Its\n" +
|
||||
"# management API is on loopback, so it needs no certificate."
|
||||
if in.standbys {
|
||||
comment = "# The model mirror on this machine, added by 'llmgw server init'. Its\n" +
|
||||
"# management API answers the control planes' addresses only."
|
||||
}
|
||||
if err := appendMirrorBlockFor(in.configPath(), in.localMirrorAPI(), comment); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("server.yaml points at the mirror")
|
||||
}
|
||||
|
||||
m, err := mirror.NewClient("http://"+mirrorAPI, tlsconf.Config{}, 5*time.Second)
|
||||
m, err := mirror.NewClient(in.localMirrorAPI(), tlsconf.Config{}, 5*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -414,8 +446,9 @@ func hasNVIDIADevice() bool {
|
||||
// --- the starter model ---
|
||||
|
||||
// starterModel is the catalog entry: a GPU profile first when there is a card,
|
||||
// and a CPU profile always, both pinned to this machine.
|
||||
func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model {
|
||||
// and a CPU profile always, both pinned to this machine. ctx is the context a
|
||||
// client may use; each engine holds four times that, for four at once.
|
||||
func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model {
|
||||
var pin map[string]string
|
||||
if pinned {
|
||||
pin = map[string]string{localLabelKey: localLabelValue}
|
||||
@@ -424,21 +457,25 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
for _, class := range gpuClasses {
|
||||
profiles = append(profiles, catalog.Profile{
|
||||
Name: "gpu-" + class, Engine: "llamacpp", GPUClass: class,
|
||||
Priority: 200, MinMemoryGB: 2, NodeSelector: pin,
|
||||
Args: []string{"--ctx-size", "32768"},
|
||||
Priority: 200, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(max(32768, 4*ctx))},
|
||||
})
|
||||
}
|
||||
profiles = append(profiles, catalog.Profile{
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2, NodeSelector: pin,
|
||||
Args: []string{"--ctx-size", "16384"},
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))},
|
||||
})
|
||||
var aliases []string
|
||||
if spec.Alias != "" {
|
||||
aliases = []string{spec.Alias}
|
||||
}
|
||||
return catalog.Model{
|
||||
Name: starterName,
|
||||
Aliases: []string{starterAlias},
|
||||
Description: "A small model init put here so there is something to try; fast, not clever.",
|
||||
Name: spec.Name,
|
||||
Aliases: aliases,
|
||||
Description: "The model init put here so there is something to try: " + spec.About + ".",
|
||||
Source: source,
|
||||
RoutingProfile: "chat",
|
||||
ContextLength: 4096,
|
||||
ContextLength: ctx,
|
||||
MaxOutput: 1024,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: profiles,
|
||||
@@ -446,8 +483,8 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
}
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room.
|
||||
func guardModel(source string) catalog.Model {
|
||||
// the models people use, on whichever node has room; replicas copies of it.
|
||||
func guardModel(source string, replicas int) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: guardModelName,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
@@ -456,7 +493,7 @@ func guardModel(source string) catalog.Model {
|
||||
Guard: &catalog.GuardModel{Format: "qwen3guard"},
|
||||
// Two: every judged request waits on it, and with fail: closed a
|
||||
// judge that is not there refuses everything.
|
||||
Scaling: catalog.Scaling{Min: 2, Max: 2},
|
||||
Scaling: catalog.Scaling{Min: replicas, Max: replicas},
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
|
||||
Args: []string{"--ctx-size", "8192"},
|
||||
@@ -469,15 +506,16 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
// Served here when this machine is a node, pinned so it does not wander
|
||||
// to the first GPU node that joins; otherwise on whichever node joins.
|
||||
pinned := !in.noNode
|
||||
return in.ensureServed(ctx, starterName, starterSource, starterVersion, pinned,
|
||||
func(url string, classes []string) catalog.Model { return starterModel(url, classes, pinned) })
|
||||
spec, n := in.starterModelSpec(), in.starterContext()
|
||||
return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned,
|
||||
func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) })
|
||||
}
|
||||
|
||||
// ensureGuardModel fetches the guardrails' judge model into the mirror and
|
||||
// adds it to the catalog, to run on any node with room.
|
||||
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
|
||||
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
|
||||
func(url string, _ []string) catalog.Model { return guardModel(url) })
|
||||
func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) })
|
||||
}
|
||||
|
||||
// ensureServed fetches a model into the mirror, adds it to the catalog, and
|
||||
|
||||
Reference in new issue
Block a user