Put the mirror on its own machine in two commands
mirror init sets a machine up as the model mirror — the user, /srv/models, and a unit serving files on :8081 and the management API on :8082 — and server init --mirror points the control plane at it instead of running one. Before, that took a unit written by hand and an edit to server.yaml. The management API can fetch and delete models and answered anyone who could reach it. mirror serve --api-allow limits it to the addresses given and the machine itself; mirror init passes the control plane's. The unit text is shared with the one server init writes for a mirror beside it. With --no-node the starter model is still fetched and catalogued, unpinned, so it starts on the first node added, and the init form shows the mirror in use instead of offering to run one. Tried on three empty VMs: the mirror answered the control plane and refused the node, and the starter model came up on the node when it was added. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a1b5daceb8
commit
b5f412f9df
9 files changed
+379
-33
No files matched your search
@@ -111,15 +111,25 @@ func (in *serverInit) writeUnit(path, body string) (bool, error) {
|
||||
|
||||
func (in *serverInit) mirrorUnit() string {
|
||||
base := "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
|
||||
return mirrorUnitFor(mirrorDir, mirrorAPI, base, nil,
|
||||
"'llmgw server init'. Bytes on :8081 for every node; the\n# management API on loopback only, for the control plane beside it.")
|
||||
}
|
||||
|
||||
// mirrorUnitFor is a mirror's systemd unit. allow is who, besides the
|
||||
// machine itself, may use the management API.
|
||||
func mirrorUnitFor(dir, apiListen, base string, allow []string, about string) string {
|
||||
exec := "/usr/local/bin/llmgw mirror serve --dir " + dir + " --listen :8081 --api-listen " + apiListen + " --base " + base
|
||||
for _, a := range allow {
|
||||
exec += " --api-allow " + a
|
||||
}
|
||||
return `[Unit]
|
||||
Description=llmgw model mirror
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
# Written by 'llmgw server init'. Bytes on :8081 for every node; the
|
||||
# management API on loopback only, for the control plane beside it.
|
||||
ExecStart=/usr/local/bin/llmgw mirror serve --dir ` + mirrorDir + ` --listen :8081 --api-listen ` + mirrorAPI + ` --base ` + base + `
|
||||
# Written by ` + about + `
|
||||
ExecStart=` + exec + `
|
||||
User=llmgw
|
||||
Group=llmgw
|
||||
Restart=on-failure
|
||||
@@ -128,13 +138,53 @@ NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=` + mirrorDir + `
|
||||
ReadWritePaths=` + dir + `
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
}
|
||||
|
||||
// mirrorAPIURL is the mirror's management API: one on its own machine, or
|
||||
// the one beside this control plane.
|
||||
func (in *serverInit) mirrorAPIURL() string {
|
||||
if in.mirrorURL != "" {
|
||||
return in.mirrorURL
|
||||
}
|
||||
return "http://" + mirrorAPI
|
||||
}
|
||||
|
||||
// useMirror points the control plane at a mirror on another machine, set up
|
||||
// there by 'llmgw mirror init'.
|
||||
func (in *serverInit) useMirror(ctx context.Context) error {
|
||||
m, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 5*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.Health(ctx); err != nil {
|
||||
return fmt.Errorf("the mirror at %s does not answer: %v. On that machine: sudo llmgw mirror init --control-plane <this machine's address>", in.mirrorURL, err)
|
||||
}
|
||||
cfg, err := config.Load(in.configPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch cfg.Mirror.API {
|
||||
case in.mirrorURL:
|
||||
in.kept("mirror at %s", in.mirrorURL)
|
||||
return nil
|
||||
case "":
|
||||
default:
|
||||
in.kept("mirror at %s, as server.yaml says; change it there to use %s", cfg.Mirror.API, in.mirrorURL)
|
||||
return nil
|
||||
}
|
||||
if err := appendMirrorBlockFor(in.configPath(), in.mirrorURL,
|
||||
"# The model mirror, on its own machine, added by 'llmgw server init'."); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("server.yaml points at the mirror at %s", in.mirrorURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureMirror runs a mirror on this machine and points the control plane at
|
||||
// it. Its management API is on loopback, which is why it needs no certificate:
|
||||
// nothing but this machine can reach it.
|
||||
@@ -174,7 +224,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if cfg.Mirror.API == "" {
|
||||
if err := appendMirrorBlock(in.configPath()); err != nil {
|
||||
if err := appendMirrorBlockFor(in.configPath(), "http://"+mirrorAPI,
|
||||
"# The model mirror on this machine, added by 'llmgw server init'. Its\n"+
|
||||
"# management API is on loopback, so it needs no certificate."); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("server.yaml points at the mirror")
|
||||
@@ -189,9 +241,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
|
||||
})
|
||||
}
|
||||
|
||||
// appendMirrorBlock adds the mirror to server.yaml without touching the rest,
|
||||
// keeping the file as it was beside it.
|
||||
func appendMirrorBlock(path string) error {
|
||||
// appendMirrorBlockFor adds the mirror to server.yaml without touching the
|
||||
// rest, keeping the file as it was beside it.
|
||||
func appendMirrorBlockFor(path, api, comment string) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -200,9 +252,7 @@ func appendMirrorBlock(path string) error {
|
||||
if err := os.WriteFile(backup, b, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
block := "\n# The model mirror on this machine, added by 'llmgw server init'. Its\n" +
|
||||
"# management API is on loopback, so it needs no certificate.\n" +
|
||||
"mirror:\n api: http://" + mirrorAPI + "\n"
|
||||
block := "\n" + comment + "\nmirror:\n api: " + api + "\n"
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -356,8 +406,11 @@ func hasNVIDIADevice() bool {
|
||||
|
||||
// starterModel is the catalog entry: a GPU profile first when there is a card,
|
||||
// and a CPU profile always, both pinned to this machine.
|
||||
func starterModel(source string, gpuClasses []string) catalog.Model {
|
||||
pin := map[string]string{localLabelKey: localLabelValue}
|
||||
func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model {
|
||||
var pin map[string]string
|
||||
if pinned {
|
||||
pin = map[string]string{localLabelKey: localLabelValue}
|
||||
}
|
||||
var profiles []catalog.Profile
|
||||
for _, class := range gpuClasses {
|
||||
profiles = append(profiles, catalog.Profile{
|
||||
@@ -401,7 +454,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
if !ok {
|
||||
return fmt.Errorf("cannot turn %s into a download", starterSource)
|
||||
}
|
||||
m, err := mirror.NewClient("http://"+mirrorAPI, tlsconf.Config{}, 15*time.Second)
|
||||
m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -444,14 +497,24 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
}
|
||||
in.done("%s in the mirror", starterName)
|
||||
|
||||
classes, _ := gpuClasses(ctx)
|
||||
body, err := json.Marshal(starterModel(st.URL, classes))
|
||||
// Served here when this machine is a node, pinned so it does not wander
|
||||
// to the first GPU node that joins; otherwise on whichever node joins.
|
||||
pinned := !in.noNode
|
||||
var classes []string
|
||||
if pinned {
|
||||
classes, _ = gpuClasses(ctx)
|
||||
}
|
||||
body, err := json.Marshal(starterModel(st.URL, classes, pinned))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
|
||||
return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
|
||||
}
|
||||
if !pinned {
|
||||
in.done("%s in the catalog; it starts on the first node that joins", starterName)
|
||||
return nil
|
||||
}
|
||||
where := "on CPU"
|
||||
if len(classes) > 0 {
|
||||
where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"
|
||||
|
||||
Reference in new issue
Block a user