Setup lets every part be chosen, each with its settings; three machines each run the gateway
CI / test (push) Failing after 30m4s

The setup screens list the parts of an install: control plane, store, model
mirror, this machine as a node, starter model, pictures and 3D, the judge,
firewall, notifications, limits, on-demand turns and an offline bundle. Each
says what it is or what is missing without it. Space turns a part on or off;
Enter opens its settings in a dialog that explains it. A plan page lists what
is installed and what is left out, and the same install as a command. Every
setting is a server init flag too, written into server.yaml when init writes
it.

With three machines, the two database machines become control planes too,
standing by (server add's steps), with an optional floating address. The
mirror's management API answers all three control planes, so a standby that
comes to lead can still fetch models.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 11:22:36 +02:00
1 parent 733e20924c
commit 94401e21e6
16 files changed
+2255 -483

No files matched your search

+104 -18
View File
@@ -95,9 +95,31 @@ type serverInit struct {
// bundle is an offline bundle to install from, and bundleSHA its
// manifest's sha256: the cluster is then set up offline.
bundle, bundleSHA string
// mirrorPath is the mirror's directory here; "" is /srv/models. A seam
// for tests.
// mirrorPath is the mirror's directory here; "" is /srv/models.
mirrorPath string
// What setup lets a person choose besides the parts: zero is the
// default (serverinit_choices.go).
starter starterSpec
starterCtx int
judge judgeSpec
// language is the portal's for a browser that asks for neither of its
// own; "" leaves it English.
language string
signupOpen bool
defaultRPM, defaultTPM int
maxKeyTTL time.Duration
// media has the nodes install ComfyUI and the leader add a picture model
// and a 3D model once a card fits one.
media bool
// evictAfterIdle and minLifetime are on_demand's; zero leaves its own.
evictAfterIdle, minLifetime time.Duration
// standbys makes each machine of the replicated store a control plane
// too, standing by while this one leads: three machines, each able to
// take over. vip is a floating address, as 10.0.0.160/24, kept on
// whichever leads.
standbys bool
vip string
// pgSetup prepares PostgreSQL and returns the store address; a seam for
// tests. Nil does it for real.
pgSetup func(context.Context) (string, error)
@@ -122,7 +144,10 @@ func newDefaultServerInit() serverInit {
func newServerInitCmd() *cobra.Command {
in := newDefaultServerInit()
var plain bool
var (
plain bool
starter, judgeCats string
)
cmd := &cobra.Command{
Use: "init",
Short: "Set this machine up as a control plane and start it",
@@ -158,6 +183,9 @@ func newServerInitCmd() *cobra.Command {
if os.Geteuid() != 0 {
return errors.New("init creates a user, writes /etc and starts a service: run it as root")
}
if err := in.readChoices(starter, judgeCats); err != nil {
return err
}
in.out = cmd.OutOrStdout()
if !plain && isTerminal(in.out) {
return in.runInitTUI(cmd.Context(), in.out)
@@ -191,6 +219,34 @@ func newServerInitCmd() *cobra.Command {
f.StringArrayVar(&in.patroni, "patroni", nil,
"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
// What server.yaml is written with; each is in it only when init writes
// it, never into one already there.
f.StringVar(&in.mirrorPath, "mirror-dir", mirrorDir, "the directory the mirror on this machine keeps model files in")
f.StringVar(&starter, "starter", starterName,
"the starter model: "+strings.Join(starterNames(), ", ")+", or hf:<owner>/<repo>/<file>.gguf")
f.IntVar(&in.starterCtx, "starter-context", defaultContext, "the context, in tokens, a client may use of the starter model")
f.StringVar(&in.judge.Fail, "judge-fail", "closed",
"a request no judge read in time: closed refuses it, open serves it flagged")
f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","),
"what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", "))
f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs")
f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge")
f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr")
f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator")
f.IntVar(&in.defaultRPM, "default-rpm", defaultRPM, "requests a minute a key may make when nothing gives it a limit")
f.IntVar(&in.defaultTPM, "default-tpm", defaultTPM, "tokens a minute a key may use when nothing gives it a limit")
f.DurationVar(&in.maxKeyTTL, "max-key-ttl", defaultMaxKeyTTL, "the longest an API key lives")
f.BoolVar(&in.media, "media", false,
"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
f.DurationVar(&in.minLifetime, "min-lifetime", 0,
"how long a model just loaded on demand is safe from that (0: 5m)")
f.BoolVar(&in.standbys, "standbys", false,
"with --patroni, make each of those machines a control plane too, standing by while this one leads")
f.StringVar(&in.vip, "vip", "",
"with --standbys, a floating address for whichever control plane leads, as 10.0.0.160/24 (keepalived on each)")
return cmd
}
@@ -274,6 +330,11 @@ func (in *serverInit) run(ctx context.Context) error {
return fmt.Errorf("starter model: %w", err)
}
}
if in.standbys {
if err := in.ensureStandbys(ctx); err != nil {
return fmt.Errorf("standby control planes: %w", err)
}
}
in.nextSteps()
return nil
}
@@ -310,29 +371,45 @@ func (in *serverInit) configJudgesWith(model string) bool {
}
// guardSection is server.yaml's guard: the baseline every request passes,
// with the judge model when there is one.
// with the judge model, as init sets it up unless told otherwise, when there
// is one.
func guardSection(judge bool) string {
j := ` # A judge model reads what patterns cannot: violence, weapons, abuse,
if !judge {
return guardSectionFor(nil)
}
j := (&serverInit{}).judgeFor()
return guardSectionFor(&j)
}
// guardSectionFor is server.yaml's guard with this judge, or none.
func guardSectionFor(j *judgeSpec) string {
fail := "closed"
jb := ` # A judge model reads what patterns cannot: violence, weapons, abuse,
# self-harm. Installed by init; --no-guard-model leaves it out.
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
`
if judge {
j = ` # A judge model reads what patterns cannot: violence, crime, sexual
# content, self-harm, jailbreaks. Two replicas run on CPUs, about two
if j != nil {
fail = j.Fail
copies := map[int]string{1: "One replica runs", 2: "Two replicas run", 3: "Three replicas run"}[j.Replicas]
if copies == "" {
copies = fmt.Sprintf("%d replicas run", j.Replicas)
}
jb = ` # A judge model reads what patterns cannot: violence, crime, sexual
# content, self-harm, jailbreaks. ` + copies + ` on CPUs, about two
# seconds for a short text; an answer is read every 600 characters as
# it streams. With fail: closed, a request no judge can read within
# latency_budget is refused (503); open serves it, flagged, unjudged.
# PII is the patterns' (above), and copyright, unethical acts and
# politically sensitive topics are left out: they refuse too much
# ordinary work. Controversial counts as safe; controversial: unsafe
# makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then
# llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
# politically sensitive topics are left out unless chosen: they refuse
# too much ordinary work. Controversial counts as safe; controversial:
# unsafe makes it count. With a GPU: llmgw ctl guard judge enable --gpu,
# then llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
judge:
models: [` + guardModelName + `]
input: block
output: block
categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak]
latency_budget: 5s
categories: [` + strings.Join(j.Categories, ", ") + `]
latency_budget: ` + durationYAML(j.Budget) + `
`
}
return `# Guardrails (docs/guardrails.md). The baseline applies to every request;
@@ -342,13 +419,13 @@ guard:
default: [baseline]
policies:
baseline:
fail: closed
fail: ` + fail + `
input:
pii: {secret: block, card: redact}
injection: flag
output:
pii: {secret: block, card: redact}
` + j
` + jb
}
func (in *serverInit) done(format string, a ...any) {
@@ -527,6 +604,10 @@ func (in *serverInit) certHosts() []string {
for _, h := range in.hosts {
add(h)
}
// The floating address is this machine's whenever it leads.
if ip, _, err := net.ParseCIDR(in.vip); err == nil {
add(ip.String())
}
if name, err := os.Hostname(); err == nil {
add(name)
}
@@ -565,8 +646,13 @@ func (in *serverInit) ensureConfig(context.Context) error {
if contact == "" {
contact = "platform@example.com"
}
body := fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
in.storeAddress(), contact, guardSection(in.judgeWanted()))
var judge *judgeSpec
if in.judgeWanted() {
j := in.judgeFor()
judge = &j
}
body := in.tuneConfig(fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
in.storeAddress(), contact, guardSectionFor(judge)))
if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
return err
}