Put the mirror on its own machine in two commands

mirror init sets a machine up as the model mirror — the user, /srv/models,
and a unit serving files on :8081 and the management API on :8082 — and
server init --mirror points the control plane at it instead of running one.
Before, that took a unit written by hand and an edit to server.yaml.

The management API can fetch and delete models and answered anyone who
could reach it. mirror serve --api-allow limits it to the addresses given
and the machine itself; mirror init passes the control plane's. The unit
text is shared with the one server init writes for a mirror beside it.

With --no-node the starter model is still fetched and catalogued, unpinned,
so it starts on the first node added, and the init form shows the mirror in
use instead of offering to run one.

Tried on three empty VMs: the mirror answered the control plane and refused
the node, and the starter model came up on the node when it was added.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 22:17:24 +02:00
1 parent a1b5daceb8
commit b5f412f9df
9 files changed
+379 -33

No files matched your search

+79 -16
View File
@@ -111,15 +111,25 @@ func (in *serverInit) writeUnit(path, body string) (bool, error) {
func (in *serverInit) mirrorUnit() string {
base := "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
return mirrorUnitFor(mirrorDir, mirrorAPI, base, nil,
"'llmgw server init'. Bytes on :8081 for every node; the\n# management API on loopback only, for the control plane beside it.")
}
// mirrorUnitFor is a mirror's systemd unit. allow is who, besides the
// machine itself, may use the management API.
func mirrorUnitFor(dir, apiListen, base string, allow []string, about string) string {
exec := "/usr/local/bin/llmgw mirror serve --dir " + dir + " --listen :8081 --api-listen " + apiListen + " --base " + base
for _, a := range allow {
exec += " --api-allow " + a
}
return `[Unit]
Description=llmgw model mirror
After=network-online.target
Wants=network-online.target
[Service]
# Written by 'llmgw server init'. Bytes on :8081 for every node; the
# management API on loopback only, for the control plane beside it.
ExecStart=/usr/local/bin/llmgw mirror serve --dir ` + mirrorDir + ` --listen :8081 --api-listen ` + mirrorAPI + ` --base ` + base + `
# Written by ` + about + `
ExecStart=` + exec + `
User=llmgw
Group=llmgw
Restart=on-failure
@@ -128,13 +138,53 @@ NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
ReadWritePaths=` + mirrorDir + `
ReadWritePaths=` + dir + `
[Install]
WantedBy=multi-user.target
`
}
// mirrorAPIURL is the mirror's management API: one on its own machine, or
// the one beside this control plane.
func (in *serverInit) mirrorAPIURL() string {
if in.mirrorURL != "" {
return in.mirrorURL
}
return "http://" + mirrorAPI
}
// useMirror points the control plane at a mirror on another machine, set up
// there by 'llmgw mirror init'.
func (in *serverInit) useMirror(ctx context.Context) error {
m, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 5*time.Second)
if err != nil {
return err
}
if err := m.Health(ctx); err != nil {
return fmt.Errorf("the mirror at %s does not answer: %v. On that machine: sudo llmgw mirror init --control-plane <this machine's address>", in.mirrorURL, err)
}
cfg, err := config.Load(in.configPath())
if err != nil {
return err
}
switch cfg.Mirror.API {
case in.mirrorURL:
in.kept("mirror at %s", in.mirrorURL)
return nil
case "":
default:
in.kept("mirror at %s, as server.yaml says; change it there to use %s", cfg.Mirror.API, in.mirrorURL)
return nil
}
if err := appendMirrorBlockFor(in.configPath(), in.mirrorURL,
"# The model mirror, on its own machine, added by 'llmgw server init'."); err != nil {
return err
}
in.done("server.yaml points at the mirror at %s", in.mirrorURL)
return nil
}
// ensureMirror runs a mirror on this machine and points the control plane at
// it. Its management API is on loopback, which is why it needs no certificate:
// nothing but this machine can reach it.
@@ -174,7 +224,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
}
if cfg.Mirror.API == "" {
if err := appendMirrorBlock(in.configPath()); err != nil {
if err := appendMirrorBlockFor(in.configPath(), "http://"+mirrorAPI,
"# The model mirror on this machine, added by 'llmgw server init'. Its\n"+
"# management API is on loopback, so it needs no certificate."); err != nil {
return err
}
in.done("server.yaml points at the mirror")
@@ -189,9 +241,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
})
}
// appendMirrorBlock adds the mirror to server.yaml without touching the rest,
// keeping the file as it was beside it.
func appendMirrorBlock(path string) error {
// appendMirrorBlockFor adds the mirror to server.yaml without touching the
// rest, keeping the file as it was beside it.
func appendMirrorBlockFor(path, api, comment string) error {
b, err := os.ReadFile(path)
if err != nil {
return err
@@ -200,9 +252,7 @@ func appendMirrorBlock(path string) error {
if err := os.WriteFile(backup, b, 0o640); err != nil {
return err
}
block := "\n# The model mirror on this machine, added by 'llmgw server init'. Its\n" +
"# management API is on loopback, so it needs no certificate.\n" +
"mirror:\n api: http://" + mirrorAPI + "\n"
block := "\n" + comment + "\nmirror:\n api: " + api + "\n"
info, err := os.Stat(path)
if err != nil {
return err
@@ -356,8 +406,11 @@ func hasNVIDIADevice() bool {
// starterModel is the catalog entry: a GPU profile first when there is a card,
// and a CPU profile always, both pinned to this machine.
func starterModel(source string, gpuClasses []string) catalog.Model {
pin := map[string]string{localLabelKey: localLabelValue}
func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model {
var pin map[string]string
if pinned {
pin = map[string]string{localLabelKey: localLabelValue}
}
var profiles []catalog.Profile
for _, class := range gpuClasses {
profiles = append(profiles, catalog.Profile{
@@ -401,7 +454,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
if !ok {
return fmt.Errorf("cannot turn %s into a download", starterSource)
}
m, err := mirror.NewClient("http://"+mirrorAPI, tlsconf.Config{}, 15*time.Second)
m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
if err != nil {
return err
}
@@ -444,14 +497,24 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
}
in.done("%s in the mirror", starterName)
classes, _ := gpuClasses(ctx)
body, err := json.Marshal(starterModel(st.URL, classes))
// Served here when this machine is a node, pinned so it does not wander
// to the first GPU node that joins; otherwise on whichever node joins.
pinned := !in.noNode
var classes []string
if pinned {
classes, _ = gpuClasses(ctx)
}
body, err := json.Marshal(starterModel(st.URL, classes, pinned))
if err != nil {
return err
}
if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
}
if !pinned {
in.done("%s in the catalog; it starts on the first node that joins", starterName)
return nil
}
where := "on CPU"
if len(classes) > 0 {
where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"