Setup lets every part be chosen, each with its settings; three machines each run the gateway
The setup screens list the parts of an install: control plane, store, model mirror, this machine as a node, starter model, pictures and 3D, the judge, firewall, notifications, limits, on-demand turns and an offline bundle. Each says what it is or what is missing without it. Space turns a part on or off; Enter opens its settings in a dialog that explains it. A plan page lists what is installed and what is left out, and the same install as a command. Every setting is a server init flag too, written into server.yaml when init writes it. With three machines, the two database machines become control planes too, standing by (server add's steps), with an optional floating address. The mirror's management API answers all three control planes, so a standby that comes to lead can still fetch models. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
733e20924c
commit
94401e21e6
16 files changed
+2255
-483
No files matched your search
+104
-18
@@ -95,9 +95,31 @@ type serverInit struct {
|
||||
// bundle is an offline bundle to install from, and bundleSHA its
|
||||
// manifest's sha256: the cluster is then set up offline.
|
||||
bundle, bundleSHA string
|
||||
// mirrorPath is the mirror's directory here; "" is /srv/models. A seam
|
||||
// for tests.
|
||||
// mirrorPath is the mirror's directory here; "" is /srv/models.
|
||||
mirrorPath string
|
||||
|
||||
// What setup lets a person choose besides the parts: zero is the
|
||||
// default (serverinit_choices.go).
|
||||
starter starterSpec
|
||||
starterCtx int
|
||||
judge judgeSpec
|
||||
// language is the portal's for a browser that asks for neither of its
|
||||
// own; "" leaves it English.
|
||||
language string
|
||||
signupOpen bool
|
||||
defaultRPM, defaultTPM int
|
||||
maxKeyTTL time.Duration
|
||||
// media has the nodes install ComfyUI and the leader add a picture model
|
||||
// and a 3D model once a card fits one.
|
||||
media bool
|
||||
// evictAfterIdle and minLifetime are on_demand's; zero leaves its own.
|
||||
evictAfterIdle, minLifetime time.Duration
|
||||
// standbys makes each machine of the replicated store a control plane
|
||||
// too, standing by while this one leads: three machines, each able to
|
||||
// take over. vip is a floating address, as 10.0.0.160/24, kept on
|
||||
// whichever leads.
|
||||
standbys bool
|
||||
vip string
|
||||
// pgSetup prepares PostgreSQL and returns the store address; a seam for
|
||||
// tests. Nil does it for real.
|
||||
pgSetup func(context.Context) (string, error)
|
||||
@@ -122,7 +144,10 @@ func newDefaultServerInit() serverInit {
|
||||
|
||||
func newServerInitCmd() *cobra.Command {
|
||||
in := newDefaultServerInit()
|
||||
var plain bool
|
||||
var (
|
||||
plain bool
|
||||
starter, judgeCats string
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Set this machine up as a control plane and start it",
|
||||
@@ -158,6 +183,9 @@ func newServerInitCmd() *cobra.Command {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("init creates a user, writes /etc and starts a service: run it as root")
|
||||
}
|
||||
if err := in.readChoices(starter, judgeCats); err != nil {
|
||||
return err
|
||||
}
|
||||
in.out = cmd.OutOrStdout()
|
||||
if !plain && isTerminal(in.out) {
|
||||
return in.runInitTUI(cmd.Context(), in.out)
|
||||
@@ -191,6 +219,34 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.StringArrayVar(&in.patroni, "patroni", nil,
|
||||
"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
|
||||
"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
|
||||
|
||||
// What server.yaml is written with; each is in it only when init writes
|
||||
// it, never into one already there.
|
||||
f.StringVar(&in.mirrorPath, "mirror-dir", mirrorDir, "the directory the mirror on this machine keeps model files in")
|
||||
f.StringVar(&starter, "starter", starterName,
|
||||
"the starter model: "+strings.Join(starterNames(), ", ")+", or hf:<owner>/<repo>/<file>.gguf")
|
||||
f.IntVar(&in.starterCtx, "starter-context", defaultContext, "the context, in tokens, a client may use of the starter model")
|
||||
f.StringVar(&in.judge.Fail, "judge-fail", "closed",
|
||||
"a request no judge read in time: closed refuses it, open serves it flagged")
|
||||
f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","),
|
||||
"what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", "))
|
||||
f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs")
|
||||
f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge")
|
||||
f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr")
|
||||
f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator")
|
||||
f.IntVar(&in.defaultRPM, "default-rpm", defaultRPM, "requests a minute a key may make when nothing gives it a limit")
|
||||
f.IntVar(&in.defaultTPM, "default-tpm", defaultTPM, "tokens a minute a key may use when nothing gives it a limit")
|
||||
f.DurationVar(&in.maxKeyTTL, "max-key-ttl", defaultMaxKeyTTL, "the longest an API key lives")
|
||||
f.BoolVar(&in.media, "media", false,
|
||||
"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
|
||||
f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
|
||||
"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
|
||||
f.DurationVar(&in.minLifetime, "min-lifetime", 0,
|
||||
"how long a model just loaded on demand is safe from that (0: 5m)")
|
||||
f.BoolVar(&in.standbys, "standbys", false,
|
||||
"with --patroni, make each of those machines a control plane too, standing by while this one leads")
|
||||
f.StringVar(&in.vip, "vip", "",
|
||||
"with --standbys, a floating address for whichever control plane leads, as 10.0.0.160/24 (keepalived on each)")
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -274,6 +330,11 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
return fmt.Errorf("starter model: %w", err)
|
||||
}
|
||||
}
|
||||
if in.standbys {
|
||||
if err := in.ensureStandbys(ctx); err != nil {
|
||||
return fmt.Errorf("standby control planes: %w", err)
|
||||
}
|
||||
}
|
||||
in.nextSteps()
|
||||
return nil
|
||||
}
|
||||
@@ -310,29 +371,45 @@ func (in *serverInit) configJudgesWith(model string) bool {
|
||||
}
|
||||
|
||||
// guardSection is server.yaml's guard: the baseline every request passes,
|
||||
// with the judge model when there is one.
|
||||
// with the judge model, as init sets it up unless told otherwise, when there
|
||||
// is one.
|
||||
func guardSection(judge bool) string {
|
||||
j := ` # A judge model reads what patterns cannot: violence, weapons, abuse,
|
||||
if !judge {
|
||||
return guardSectionFor(nil)
|
||||
}
|
||||
j := (&serverInit{}).judgeFor()
|
||||
return guardSectionFor(&j)
|
||||
}
|
||||
|
||||
// guardSectionFor is server.yaml's guard with this judge, or none.
|
||||
func guardSectionFor(j *judgeSpec) string {
|
||||
fail := "closed"
|
||||
jb := ` # A judge model reads what patterns cannot: violence, weapons, abuse,
|
||||
# self-harm. Installed by init; --no-guard-model leaves it out.
|
||||
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
|
||||
`
|
||||
if judge {
|
||||
j = ` # A judge model reads what patterns cannot: violence, crime, sexual
|
||||
# content, self-harm, jailbreaks. Two replicas run on CPUs, about two
|
||||
if j != nil {
|
||||
fail = j.Fail
|
||||
copies := map[int]string{1: "One replica runs", 2: "Two replicas run", 3: "Three replicas run"}[j.Replicas]
|
||||
if copies == "" {
|
||||
copies = fmt.Sprintf("%d replicas run", j.Replicas)
|
||||
}
|
||||
jb = ` # A judge model reads what patterns cannot: violence, crime, sexual
|
||||
# content, self-harm, jailbreaks. ` + copies + ` on CPUs, about two
|
||||
# seconds for a short text; an answer is read every 600 characters as
|
||||
# it streams. With fail: closed, a request no judge can read within
|
||||
# latency_budget is refused (503); open serves it, flagged, unjudged.
|
||||
# PII is the patterns' (above), and copyright, unethical acts and
|
||||
# politically sensitive topics are left out: they refuse too much
|
||||
# ordinary work. Controversial counts as safe; controversial: unsafe
|
||||
# makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then
|
||||
# llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
|
||||
# politically sensitive topics are left out unless chosen: they refuse
|
||||
# too much ordinary work. Controversial counts as safe; controversial:
|
||||
# unsafe makes it count. With a GPU: llmgw ctl guard judge enable --gpu,
|
||||
# then llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
|
||||
judge:
|
||||
models: [` + guardModelName + `]
|
||||
input: block
|
||||
output: block
|
||||
categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak]
|
||||
latency_budget: 5s
|
||||
categories: [` + strings.Join(j.Categories, ", ") + `]
|
||||
latency_budget: ` + durationYAML(j.Budget) + `
|
||||
`
|
||||
}
|
||||
return `# Guardrails (docs/guardrails.md). The baseline applies to every request;
|
||||
@@ -342,13 +419,13 @@ guard:
|
||||
default: [baseline]
|
||||
policies:
|
||||
baseline:
|
||||
fail: closed
|
||||
fail: ` + fail + `
|
||||
input:
|
||||
pii: {secret: block, card: redact}
|
||||
injection: flag
|
||||
output:
|
||||
pii: {secret: block, card: redact}
|
||||
` + j
|
||||
` + jb
|
||||
}
|
||||
|
||||
func (in *serverInit) done(format string, a ...any) {
|
||||
@@ -527,6 +604,10 @@ func (in *serverInit) certHosts() []string {
|
||||
for _, h := range in.hosts {
|
||||
add(h)
|
||||
}
|
||||
// The floating address is this machine's whenever it leads.
|
||||
if ip, _, err := net.ParseCIDR(in.vip); err == nil {
|
||||
add(ip.String())
|
||||
}
|
||||
if name, err := os.Hostname(); err == nil {
|
||||
add(name)
|
||||
}
|
||||
@@ -565,8 +646,13 @@ func (in *serverInit) ensureConfig(context.Context) error {
|
||||
if contact == "" {
|
||||
contact = "platform@example.com"
|
||||
}
|
||||
body := fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
|
||||
in.storeAddress(), contact, guardSection(in.judgeWanted()))
|
||||
var judge *judgeSpec
|
||||
if in.judgeWanted() {
|
||||
j := in.judgeFor()
|
||||
judge = &j
|
||||
}
|
||||
body := in.tuneConfig(fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
|
||||
in.storeAddress(), contact, guardSectionFor(judge)))
|
||||
if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user