Merge C18 offline bundles and the Helm chart

Conflicts: server.go imports (both kept); agent.pb.go regenerated with buf
from the merged proto.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:15:47 +02:00
commit 6f85c17fc5
51 files changed
+4177 -63

No files matched your search

+341
View File
@@ -0,0 +1,341 @@
package cli
import (
"errors"
"fmt"
"io"
"os"
"strings"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// newBundleCmd carries a cluster's downloads to a site with no internet.
func newBundleCmd() *cobra.Command {
c := &cobra.Command{
Use: "bundle",
Short: "Carry releases, engines and models to a site with no internet",
Long: "A cluster fetches llmgw's releases, the llama.cpp builds its nodes install,\n" +
"and model files from the internet. A site that has none gets them in a\n" +
"bundle: made where there is internet with 'bundle create', carried in, and\n" +
"put into the cluster's mirror with 'bundle import'. Every file is checked\n" +
"as a node online would check it — releases against the project's signing\n" +
"key, engines against their publisher's checksums — when the bundle is made,\n" +
"when it is imported, and again on the node that installs it.",
}
c.AddCommand(newBundleCreateCmd(), newBundleImportCmd(), newBundleTrustCmd(), newBundleShowCmd())
return c
}
func newBundleCreateCmd() *cobra.Command {
var (
out, rel, releases, registry string
arches, engines, variants []string
models []string
noRelease, noStarter, pack bool
)
c := &cobra.Command{
Use: "create",
Short: "Make a bundle, on a machine with internet",
Long: "Downloads into --out, checking each file as it arrives: llmgw's release for\n" +
"each --arch with its signed checksums, every build of each --engine for those\n" +
"architectures (or the --variant builds), and each --model's file from Hugging\n" +
"Face, pinned to the commit it was taken at. The starter model and the\n" +
"guardrails' judge that 'server init' serves are carried unless\n" +
"--no-starter-models.\n\n" +
"Run again with the same --out to add to a bundle: what is there and right is\n" +
"not downloaded again.\n\n" +
"It prints the manifest's sha256. Carry that apart from the bundle — read it\n" +
"out over the phone, write it in the ticket — and give it to 'bundle import':\n" +
"it is what says the bundle that arrived is the one that was made.\n\n" +
"vLLM and SGLang are Python environments from PyPI, and no bundle carries\n" +
"them; offline nodes serve with llama.cpp, or have them installed by hand.",
Example: ` llmgw bundle create --out llmgw-bundle --arch amd64 --tar
llmgw bundle create --out site-b --engine llamacpp=b10991 --variant cpu --variant cuda-13.3 \
--model qwen2.5-14b-instruct/2026-10=hf:Qwen/Qwen2.5-14B-Instruct-GGUF/qwen2.5-14b-instruct-q4_k_m.gguf`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if out == "" {
return errors.New("--out: the directory to make the bundle in")
}
o := bundle.CreateOptions{Out: out, Arches: arches, Releases: releases, Registry: registry}
switch {
case noRelease:
case rel != "":
o.Release = rel
case release.Tag.MatchString(version.Version):
o.Release = version.Version
default:
return fmt.Errorf("this llmgw is %s, which has no release to carry: give --version v0.x.y, "+
"or --no-release to carry none", version.Version)
}
if len(engines) == 0 {
engines = []string{"llamacpp=" + config.DefaultEngines["llamacpp"]}
}
for _, e := range engines {
name, v, ok := strings.Cut(e, "=")
if !ok || name == "" || v == "" {
return fmt.Errorf("--engine %q: engine=version, llamacpp=%s", e, config.DefaultEngines["llamacpp"])
}
o.Engines = append(o.Engines, bundle.EngineSpec{Engine: name, Version: v, Variants: variants})
}
if !noStarter {
o.Models = append(o.Models,
bundle.ModelSpec{Name: starterName, Version: starterVersion, Source: starterSource},
bundle.ModelSpec{Name: guardModelName, Version: guardModelVersion, Source: guardModelSource})
}
for _, m := range models {
spec, err := parseModelSpec(m)
if err != nil {
return err
}
o.Models = append(o.Models, spec)
}
w := cmd.OutOrStdout()
o.Log = func(format string, args ...any) { fmt.Fprintf(w, format+"\n", args...) }
m, digest, err := bundle.Create(cmd.Context(), o)
if err != nil {
return err
}
where := out
if pack {
where = strings.TrimSuffix(out, "/") + ".tar"
fmt.Fprintf(w, "packing it into %s\n", where)
if err := bundle.Pack(out, where); err != nil {
return err
}
if err := os.RemoveAll(out); err != nil {
return err
}
}
printManifest(w, m)
fmt.Fprintf(w, "\nThe bundle is %s. Its sha256, to carry apart from it:\n\n %s\n\n", where, digest)
fmt.Fprintf(w, "At the site, on the machine with the mirror (and the control plane):\n\n"+
" sudo llmgw bundle import %s --sha256 %s\n\n"+
"or, installing a control plane from it:\n\n"+
" sudo llmgw server init --bundle %s --bundle-sha256 %s\n", where, digest, where, digest)
return nil
},
}
f := c.Flags()
f.StringVar(&out, "out", "", "the directory to make the bundle in; added to when it holds one already")
f.BoolVar(&pack, "tar", false, "pack the bundle into <out>.tar, one file to carry, and remove the directory")
f.StringVar(&rel, "version", "", "llmgw's release to carry (default: this llmgw's own)")
f.BoolVar(&noRelease, "no-release", false, "carry no llmgw release")
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)")
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
f.StringVar(&registry, "registry", "", "the model registry, when not the public Hugging Face")
return c
}
// parseModelSpec reads name/version=source.
func parseModelSpec(s string) (bundle.ModelSpec, error) {
where, source, ok := strings.Cut(s, "=")
name, ver, ok2 := strings.Cut(where, "/")
if !ok || !ok2 || name == "" || ver == "" || source == "" {
return bundle.ModelSpec{}, fmt.Errorf("--model %q: name/version=hf:owner/repo/file", s)
}
return bundle.ModelSpec{Name: name, Version: ver, Source: source}, nil
}
func newBundleImportCmd() *cobra.Command {
var (
dir, digest, trust, serverCfg string
unchecked, noTrust bool
)
c := &cobra.Command{
Use: "import PATH",
Short: "Put a bundle into this machine's mirror",
Long: "Checks every file of the bundle — against its manifest, and llmgw's release\n" +
"against the project's signing key — and only then puts them into the mirror:\n" +
"releases and engines under _llmgw/, models where the mirror keeps models.\n" +
"A file the mirror has already, the same, is kept; a different one under the\n" +
"same version is refused, because a version is never replaced.\n\n" +
"On a control plane (server.yaml is there) it also records the engines'\n" +
"checksums in offline.checksums, which the nodes check each engine file they\n" +
"fetch against. When the mirror is on another machine, import there, and\n" +
"run 'llmgw bundle trust' with the same bundle on the control plane.\n\n" +
"PATH is the bundle's directory or its .tar.",
Example: ` sudo llmgw bundle import /media/usb/llmgw-bundle.tar --sha256 3f2a…`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
w := cmd.OutOrStdout()
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
return fmt.Errorf("%s is not the mirror's directory: give --dir, or set the mirror up first "+
"(sudo llmgw mirror init, or server init)", dir)
}
fmt.Fprintf(w, "checking %s and putting it into the mirror at %s\n", args[0], dir)
res, err := bundle.Import(args[0], bundle.ImportOptions{
Mirror: dir, SHA256: digest, Unchecked: unchecked,
Progress: func(f bundle.File, kept bool) {
if kept {
fmt.Fprintf(w, " kept %s\n", f.Path)
} else {
fmt.Fprintf(w, " ok %s (%s)\n", f.Path, fileSize(f.Bytes))
}
},
})
if err != nil {
return err
}
if unchecked {
fmt.Fprintf(w, "\nTaken unchecked: its manifest's sha256 is %s; compare it with what was made.\n", res.Digest)
}
fmt.Fprintf(w, "\n%d files put in the mirror (%s), %d already there.\n", res.Written, fileSize(res.Bytes), res.Kept)
if noTrust {
return nil
}
cfg, haveCfg := loadServerConfig(serverCfg)
if trust == "" && !haveCfg {
fmt.Fprintf(w, "\nThis machine is not a control plane. On the control plane:\n\n"+
" sudo llmgw bundle trust %s --sha256 %s\n", args[0], res.Digest)
return nil
}
if trust == "" {
trust = cfg.Offline.ChecksumsFile()
}
n, err := bundle.TrustManifest(res.Manifest, trust)
if err != nil {
return fmt.Errorf("recording the engines' checksums in %s: %w", trust, err)
}
fmt.Fprintf(w, "The control plane trusts %d engine files (%s).\n", n, trust)
offlineHint(w, cfg, haveCfg)
return nil
},
}
f := c.Flags()
f.StringVar(&dir, "dir", mirrorDir, "the mirror's directory")
f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it")
f.BoolVar(&unchecked, "unchecked", false, "take the bundle without its sha256 (its files are still checked against its manifest)")
f.StringVar(&trust, "trust", "", "where to record the engines' checksums (default: offline.checksums, on a control plane)")
f.BoolVar(&noTrust, "no-trust", false, "do not record the engines' checksums here")
f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, when this is one")
return c
}
func newBundleTrustCmd() *cobra.Command {
var digest, file, serverCfg string
var unchecked bool
c := &cobra.Command{
Use: "trust PATH",
Short: "Have this control plane trust a bundle's engine checksums",
Long: "Records the checksum of every engine file in the bundle in offline.checksums,\n" +
"where the control plane reads what it tells each node a file from the\n" +
"mirror must be. Only the manifest is read, so this is the step for a control\n" +
"plane whose mirror is on another machine: import the bundle there, and run\n" +
"this here with the same bundle and sha256. The control plane sees the new\n" +
"checksums without a restart.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
w := cmd.OutOrStdout()
cfg, haveCfg := loadServerConfig(serverCfg)
if file == "" {
file = cfg.Offline.ChecksumsFile()
}
m, n, err := bundle.Trust(args[0], file, digest, unchecked)
if err != nil {
return err
}
printManifest(w, m)
fmt.Fprintf(w, "\nThe control plane trusts %d engine files (%s).\n", n, file)
offlineHint(w, cfg, haveCfg)
return nil
},
}
f := c.Flags()
f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it")
f.BoolVar(&unchecked, "unchecked", false, "take the manifest without its sha256")
f.StringVar(&file, "file", "", "where to record them (default: offline.checksums from server.yaml, else "+config.DefaultOfflineChecksums+")")
f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
return c
}
func newBundleShowCmd() *cobra.Command {
var verify bool
var digest string
c := &cobra.Command{
Use: "show PATH",
Short: "Say what a bundle holds, and check it",
Long: "Prints what the bundle carries and its sha256. With --verify every file is\n" +
"read and checked, as an import would, and nothing is written: worth doing\n" +
"after copying a bundle onto the disk that carries it.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
w := cmd.OutOrStdout()
m, d, err := bundle.ReadManifest(args[0])
if err != nil {
return err
}
printManifest(w, m)
fmt.Fprintf(w, "\nsha256 %s\n", d)
if !verify {
return nil
}
res, err := bundle.Import(args[0], bundle.ImportOptions{SHA256: digest, Unchecked: digest == ""})
if err != nil {
return err
}
fmt.Fprintf(w, "Every file is as it was made (%d files, %s).\n", res.Written, fileSize(res.Bytes))
return nil
},
}
c.Flags().BoolVar(&verify, "verify", false, "read and check every file")
c.Flags().StringVar(&digest, "sha256", "", "with --verify, the sha256 the bundle should have")
return c
}
// printManifest says what a bundle carries.
func printManifest(w io.Writer, m bundle.Manifest) {
var total int64
for _, f := range m.Files {
total += f.Bytes
}
fmt.Fprintf(w, "\nA bundle made %s by llmgw %s: %d files, %s.\n",
m.Created.Format("2006-01-02 15:04 UTC"), m.MadeBy, len(m.Files), fileSize(total))
for _, r := range m.Releases {
fmt.Fprintf(w, " llmgw %s for %s\n", r.Version, strings.Join(r.Arches, ", "))
}
for _, e := range m.Engines {
fmt.Fprintf(w, " %-9s %s for %s: %s\n", e.Engine, e.Version, strings.Join(e.Arches, ", "), strings.Join(e.Variants, ", "))
}
for _, md := range m.Models {
at := ""
if md.Revision != "" {
at = " at " + md.Revision[:min(len(md.Revision), 8)]
}
fmt.Fprintf(w, " model %s/%s from %s%s\n", md.Name, md.Version, md.Source, at)
}
}
// loadServerConfig reads a control plane's configuration, when this
// machine is one.
func loadServerConfig(path string) (config.Server, bool) {
if _, err := os.Stat(path); err != nil {
return config.Server{}, false
}
cfg, err := config.Load(path)
if err != nil {
return config.Server{}, false
}
return cfg, true
}
// offlineHint says what is left once a bundle is trusted.
func offlineHint(w io.Writer, cfg config.Server, haveCfg bool) {
if haveCfg && !cfg.Offline.Enabled() {
fmt.Fprintf(w, "\nserver.yaml does not yet say the cluster is offline. Add the mirror's file\n"+
"address, as nodes reach it, and restart the control plane:\n\n"+
" offline:\n mirror: http://<mirror>:8081\n\n"+
" sudo systemctl restart llmgw-server\n")
}
}
+12 -1
View File
@@ -50,6 +50,16 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
}
out := cmd.OutOrStdout()
ctx := cmd.Context()
if releases == "" {
// Where the control plane sends its nodes: an offline
// cluster's mirror, so this machine needs no internet either.
if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok {
releases = cfg.ReleasesBase()
}
if releases == "" {
releases = release.DefaultBase
}
}
if !skipCP {
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
return err
@@ -60,7 +70,8 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
}
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
c.Flags().StringVar(&releases, "releases", "", "where release binaries are fetched from, for this machine "+
"(default: as server.yaml says, the mirror's copy on an offline cluster, else the project's)")
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
return c
+7 -1
View File
@@ -53,9 +53,15 @@ func newJoinCmd() *cobra.Command {
// screen that cuts long ones off.
func joinInstall() []string {
v := version.Version
// From where this cluster takes its releases: on an offline one, its
// mirror, which the new machine can reach when the internet is not.
base := release.DefaultBase
if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok && cfg.ReleasesBase() != "" {
base = cfg.ReleasesBase()
}
return []string{
`a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`,
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v),
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", base, v, v),
"sudo chmod 755 /usr/local/bin/llmgw",
}
}
+12 -1
View File
@@ -17,6 +17,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
@@ -120,7 +121,7 @@ engines are kept.`,
f.StringVar(&na.opts.NvidiaSMI, "nvidia-smi", "", "where nvidia-smi is, when it is not on PATH")
f.StringVar(&na.opts.IBRoot, "ib-root", "", "the sysfs InfiniBand directory, when not /sys/class/infiniband")
f.StringVar(&na.opts.PKI, "pki", defaultPKIDir, "directory holding the certificate authority")
f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
f.StringVar(&na.opts.Releases, "releases", "", "where release files are downloaded from (default: releases in server.yaml, the mirror's copy on an offline cluster, else the project's)")
f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
f.BoolVar(&na.opts.NoNodeExporter, "no-node-exporter", false, "leave Prometheus' node exporter off the machine")
f.BoolVar(&na.opts.NoFirewall, "no-firewall", false, "leave the machine's firewall alone, instead of keeping llmgw's ports to the cluster's networks")
@@ -176,6 +177,16 @@ func (na *nodeAdd) prepare() error {
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
}
// Where the control plane upgrades nodes from, so a node joins on the
// same release from the same place: an offline cluster's mirror.
if o.Releases == "" {
if cfg, err := config.Load(na.serverCfg); err == nil {
o.Releases = cfg.ReleasesBase()
}
if o.Releases == "" {
o.Releases = release.DefaultBase
}
}
o.Version = version.Version
if o.Binary == "" {
exe, err := os.Executable()
+1
View File
@@ -52,6 +52,7 @@ func Root() *cobra.Command {
newUsersCmd(),
newOrgCmd(),
newMirrorCmd(),
newBundleCmd(),
newSimEngineCmd(),
newDoctorCmd(),
newNodeCmd(),
+22
View File
@@ -34,6 +34,7 @@ import (
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
@@ -103,6 +104,12 @@ func newServerCmd() *cobra.Command {
return errors.New("nothing to serve: declare models with profiles, " +
"or give --endpoint for a router-only run")
}
// Offline, nothing is asked of the internet: releases come from
// the mirror's copy, and no registry is asked about newer weights.
if cfg.Offline.Enabled() {
cfg.Releases = cfg.ReleasesBase()
cfg.Mirror.UpdateCheck = 0
}
// Before the router, so its tracer is the configured one. The
// host tells two control planes' traces apart.
@@ -685,6 +692,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels)
})
log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides))
if cfg.Offline.Enabled() {
// From the mirror, each file held to the checksum the bundle gave,
// which this control plane keeps and the mirror has no say in.
trusted := bundle.NewTrusted(cfg.Offline.ChecksumsFile())
svc.SetOffline(func(engine, version string) (string, map[string]string) {
return bundle.EngineBase(cfg.Offline.Mirror, engine, version), trusted.For(engine, version)
})
if v, ok := cfg.WantedEngines()["llamacpp"]; ok && len(trusted.For("llamacpp", v)) == 0 {
log.Warn("offline, and no bundle imported here carries llama.cpp at this version: nodes cannot "+
"install it until one does (llmgw bundle create --engine llamacpp="+v+", then bundle import or trust)",
"version", v, "checksums", cfg.Offline.ChecksumsFile(), "err", trusted.Err())
}
log.Info("offline: nodes install engines from the mirror", "mirror", cfg.Offline.Mirror,
"releases", cfg.Releases)
}
if ch := ctl.EngineChoices(); ch != nil {
svc.SetEngineChoices(ch.For)
ch.OnChange(svc.Reprovision)
+24
View File
@@ -89,6 +89,12 @@ type serverInit struct {
noFirewall bool
// mirrorURL is a mirror on another machine to use instead of running one.
mirrorURL string
// bundle is an offline bundle to install from, and bundleSHA its
// manifest's sha256: the cluster is then set up offline.
bundle, bundleSHA string
// mirrorPath is the mirror's directory here; "" is /srv/models. A seam
// for tests.
mirrorPath string
// pgSetup prepares PostgreSQL and returns the store address; a seam for
// tests. Nil does it for real.
pgSetup func(context.Context) (string, error)
@@ -165,6 +171,9 @@ func newServerInitCmd() *cobra.Command {
f.BoolVar(&in.noMirror, "no-mirror", false, "do not run a model mirror on this machine")
f.StringVar(&in.mirrorURL, "mirror", "",
"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
f.StringVar(&in.bundle, "bundle", "",
"install offline from this bundle ('llmgw bundle create'): into the mirror, with the cluster set to reach nothing outside")
f.StringVar(&in.bundleSHA, "bundle-sha256", "", "the bundle's sha256, as 'llmgw bundle create' printed it")
f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
f.BoolVar(&in.noGuardModel, "no-guard-model", false,
@@ -180,6 +189,9 @@ func newServerInitCmd() *cobra.Command {
}
func (in *serverInit) run(ctx context.Context) error {
if err := in.checkBundle(); err != nil {
return fmt.Errorf("offline bundle: %w", err)
}
steps := []struct {
name string
fn func(context.Context) error
@@ -214,6 +226,13 @@ func (in *serverInit) run(ctx context.Context) error {
return fmt.Errorf("mirror: %w", err)
}
}
// Into the mirror, and server.yaml told, before the service starts:
// the control plane reads at start that the cluster is offline.
if in.bundle != "" {
if err := in.ensureBundle(ctx); err != nil {
return fmt.Errorf("offline bundle: %w", err)
}
}
if err := in.ensureService(ctx); err != nil {
return fmt.Errorf("service: %w", err)
}
@@ -256,6 +275,11 @@ func (in *serverInit) judgeWanted() bool {
if in.noGuardModel || (in.noMirror && in.mirrorURL == "") || in.unitPath == "" {
return false
}
// Offline, only a judge the bundle carries: a policy asking a judge
// that never comes refuses every request.
if in.bundle != "" && !in.bundleCarries(guardModelName) {
return false
}
_, err := exec.LookPath("systemctl")
return err == nil
}
+158
View File
@@ -0,0 +1,158 @@
package cli
import (
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// --- installing from an offline bundle ---
// checkBundle refuses, before anything is done, a bundle init could not
// install from: one that is not there, or not the one that was made.
func (in *serverInit) checkBundle() error {
if in.bundle == "" {
return nil
}
if in.noMirror && in.mirrorURL == "" {
return errors.New("a bundle goes into the cluster's mirror: leave out --no-mirror, or give --mirror")
}
if in.bundleSHA == "" {
return errors.New("give --bundle-sha256, the sha256 'llmgw bundle create' printed: " +
"it is what says the bundle that arrived is the one that was made")
}
_, digest, err := bundle.ReadManifest(in.bundle)
if err != nil {
return err
}
if digest != in.bundleSHA {
return fmt.Errorf("the bundle's sha256 is %s, not the %s given: it is not the bundle that was made, "+
"or the digest was mistyped", digest, in.bundleSHA)
}
return nil
}
// bundleCarries says whether the bundle holds a model of this name.
func (in *serverInit) bundleCarries(model string) bool {
m, _, err := bundle.ReadManifest(in.bundle)
if err != nil {
return false
}
for _, md := range m.Models {
if md.Name == model {
return true
}
}
return false
}
// offline says whether this cluster is set up to reach nothing outside.
func (in *serverInit) offline() bool {
if in.bundle != "" {
return true
}
cfg, err := config.Load(in.configPath())
return err == nil && cfg.Offline.Enabled()
}
func (in *serverInit) mirrorRoot() string {
if in.mirrorPath != "" {
return in.mirrorPath
}
return mirrorDir
}
// ensureBundle puts the bundle into the mirror beside this control plane,
// has the control plane trust its engines' checksums, and says in
// server.yaml that the cluster is offline. With a mirror on another
// machine, the files are imported there; here only the checksums are kept.
func (in *serverInit) ensureBundle(ctx context.Context) error {
cfg, err := config.Load(in.configPath())
if err != nil {
return err
}
var (
m bundle.Manifest
files string
)
if in.mirrorURL == "" {
res, err := bundle.Import(in.bundle, bundle.ImportOptions{Mirror: in.mirrorRoot(), SHA256: in.bundleSHA})
if err != nil {
return err
}
if res.Written > 0 {
in.done("offline bundle in the mirror: %d files, %s, each checked", res.Written, fileSize(res.Bytes))
} else {
in.kept("offline bundle in the mirror (%d files)", res.Kept)
}
m = res.Manifest
files = "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
} else {
mc, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 10*time.Second)
if err != nil {
return err
}
list, err := mc.List(ctx)
if err != nil {
return fmt.Errorf("asking the mirror at %s where nodes reach it: %w", in.mirrorURL, err)
}
files = list.Base
if m, _, err = bundle.ReadManifest(in.bundle); err != nil {
return err
}
fmt.Fprintf(in.out, " !! the mirror is on another machine: import the bundle there too, "+
"sudo llmgw bundle import <bundle> --sha256 %s\n", in.bundleSHA)
}
sums := cfg.Offline.Checksums
if sums == "" {
sums = filepath.Join(in.etcDir, "offline.sha256")
}
n, err := bundle.TrustManifest(m, sums)
if err != nil {
return err
}
in.done("the control plane trusts %d engine files from the bundle (%s)", n, sums)
if cfg.Offline.Enabled() {
in.kept("offline, installing from %s, as server.yaml says", cfg.Offline.Mirror)
return nil
}
if sums == config.DefaultOfflineChecksums {
sums = ""
}
if err := appendOfflineBlock(in.configPath(), files, sums); err != nil {
return err
}
in.done("server.yaml: offline, nodes install from the mirror at %s", files)
return nil
}
// appendOfflineBlock says in server.yaml that the cluster is offline,
// without touching the rest.
func appendOfflineBlock(path, files, sums string) error {
b, err := os.ReadFile(path) //nolint:gosec // server.yaml
if err != nil {
return err
}
info, err := os.Stat(path)
if err != nil {
return err
}
block := "\n# Offline: nothing in the cluster reaches the internet. Nodes install\n" +
"# engines, and upgrades fetch releases, from the mirror, out of the bundle\n" +
"# 'llmgw server init --bundle' imported; 'llmgw bundle import' adds more.\n" +
"offline:\n mirror: " + files + "\n"
if sums != "" {
block += " checksums: " + sums + "\n"
}
return os.WriteFile(path, append(b, []byte(block)...), info.Mode().Perm())
}
+86
View File
@@ -0,0 +1,86 @@
package cli
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
)
// handMadeBundle is a bundle as 'bundle create' leaves it: an engine build
// and the starter model, with their checksums in the manifest.
func handMadeBundle(t *testing.T) (dir, digest string) {
t.Helper()
dir = t.TempDir()
m := bundle.Manifest{Format: bundle.Format, MadeBy: "test",
Models: []bundle.Model{{Name: starterName, Version: starterVersion, Source: starterSource}}}
for p, body := range map[string]string{
"engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz": "an engine",
"models/" + starterName + "/" + starterVersion + "/qwen-q4_k_m.gguf": "weights",
} {
full := filepath.Join(dir, p)
_ = os.MkdirAll(filepath.Dir(full), 0o755)
_ = os.WriteFile(full, []byte(body), 0o644)
sum := sha256.Sum256([]byte(body))
m.Files = append(m.Files, bundle.File{Path: p, Bytes: int64(len(body)), SHA256: hex.EncodeToString(sum[:])})
}
raw, _ := json.Marshal(m)
_ = os.WriteFile(filepath.Join(dir, bundle.ManifestName), raw, 0o644)
return dir, bundle.Digest(raw)
}
func TestInitFromABundleLeavesAnOfflineClusterThatTrustsItsEngines(t *testing.T) {
in, out := newTestInit(t)
dir, digest := handMadeBundle(t)
in.bundle, in.hosts, in.mirrorPath = dir, []string{"10.9.8.7"}, t.TempDir()
in.bundleSHA = ""
if err := in.checkBundle(); err == nil || !strings.Contains(err.Error(), "--bundle-sha256") {
t.Errorf("a bundle with no digest: %v", err)
}
in.bundleSHA = strings.Repeat("0", 64)
if err := in.checkBundle(); err == nil {
t.Error("a bundle not matching its digest was taken")
}
in.bundleSHA = digest
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
if err := in.ensureBundle(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
cfg, err := config.Load(in.configPath())
if err != nil {
t.Fatal(err)
}
if cfg.Offline.Mirror != "http://10.9.8.7:8081" || cfg.ReleasesBase() != "http://10.9.8.7:8081/_llmgw/releases" {
t.Errorf("server.yaml: offline %+v, releases %s", cfg.Offline, cfg.ReleasesBase())
}
got := bundle.NewTrusted(cfg.Offline.ChecksumsFile()).For("llamacpp", "b1")
if len(got) != 1 {
t.Errorf("the control plane trusts %v", got)
}
if _, err := os.Stat(filepath.Join(in.mirrorPath, starterName, starterVersion, "qwen-q4_k_m.gguf")); err != nil {
t.Errorf("the starter model is not in the mirror: %v", err)
}
if !in.bundleCarries(starterName) || in.bundleCarries(guardModelName) {
t.Error("what the bundle carries was misread")
}
// Again, and nothing changes: not the mirror, not server.yaml.
before, _ := os.ReadFile(in.configPath())
if err := in.ensureBundle(context.Background()); err != nil {
t.Fatal(err)
}
if after, _ := os.ReadFile(in.configPath()); string(after) != string(before) {
t.Error("init from the same bundle changed server.yaml again")
}
}
+5
View File
@@ -516,6 +516,11 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st
var apiErr *mirror.APIError
switch {
case err == nil:
case errors.As(err, &apiErr) && apiErr.Status == 404 && in.offline():
// Asked of the internet, it would fail after a timeout and stop init
// over a model nobody carried in.
fmt.Fprintf(in.out, " skip %s: not in the mirror, and an offline cluster does not download it\n", name)
return nil
case errors.As(err, &apiErr) && apiErr.Status == 404:
if st, err = m.Fetch(ctx, mirror.FetchRequest{
Source: upstream, Model: name, Version: version,
+47 -1
View File
@@ -55,6 +55,8 @@ const (
fNode
fModel
fGuard
fBundle
fBundleSHA
fStart
fCount
)
@@ -68,6 +70,8 @@ type initModel struct {
focus int
admin textField
hosts textField
bundle textField
bundleSHA textField
toggles [fCount]bool
formError string
@@ -99,8 +103,19 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel {
hosts.SetValue(strings.Join(in.hosts, ", "))
hosts.CharLimit = 400
hosts.Width = 40
bundle := newTextField()
bundle.Placeholder = "/media/usb/llmgw-bundle.tar"
bundle.SetValue(in.bundle)
bundle.CharLimit = 400
bundle.Width = 40
bundleSHA := newTextField()
bundleSHA.Placeholder = "the sha256 'llmgw bundle create' printed"
bundleSHA.SetValue(in.bundleSHA)
bundleSHA.CharLimit = 64
bundleSHA.Width = 64
sp := newSpinner()
m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, spin: sp, width: 100}
m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts,
bundle: bundle, bundleSHA: bundleSHA, spin: sp, width: 100}
m.toggles[fPostgres] = in.postgres
m.toggles[fMirror] = !in.noMirror
m.toggles[fNode] = !in.noNode
@@ -207,6 +222,20 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, nil
}
}
bundle, sum := strings.TrimSpace(m.bundle.Value()), strings.ToLower(strings.TrimSpace(m.bundleSHA.Value()))
switch {
case bundle != "" && sum == "":
m.formError = "give the bundle's sha256, as 'llmgw bundle create' printed it"
m.focus = fBundleSHA
m.moveFocus(0)
return m, nil
case bundle != "" && !m.toggles[fMirror] && m.in.mirrorURL == "":
m.formError = "an offline bundle goes into the model mirror: tick it"
m.focus = fMirror
m.moveFocus(0)
return m, nil
}
m.in.bundle, m.in.bundleSHA = bundle, sum
m.in.admin = admin
m.in.hosts = nil
for _, h := range strings.Split(m.hosts.Value(), ",") {
@@ -231,6 +260,14 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
var cmd tea.Cmd
m.hosts, cmd = m.hosts.Update(k)
return m, cmd
case fBundle:
var cmd tea.Cmd
m.bundle, cmd = m.bundle.Update(k)
return m, cmd
case fBundleSHA:
var cmd tea.Cmd
m.bundleSHA, cmd = m.bundleSHA.Update(k)
return m, cmd
case fPostgres, fMirror, fNode, fModel, fGuard:
if k.String() == " " || k.String() == "x" {
m.toggles[m.focus] = !m.toggles[m.focus]
@@ -247,11 +284,17 @@ func (m *initModel) moveFocus(d int) {
}
m.admin.Blur()
m.hosts.Blur()
m.bundle.Blur()
m.bundleSHA.Blur()
switch m.focus {
case fAdmin:
m.admin.Focus()
case fHosts:
m.hosts.Focus()
case fBundle:
m.bundle.Focus()
case fBundleSHA:
m.bundleSHA.Focus()
}
}
@@ -375,6 +418,9 @@ func (m initModel) formView() string {
b.WriteString(check(fNode, "This machine as a node", "serves models here too, on its GPUs or CPU"))
b.WriteString(check(fModel, "Starter model", "a small Qwen, so there is something to ask at once"))
b.WriteString(check(fGuard, "Guardrails' judge", "Llama Guard 3 1B on a CPU, refusing violent, criminal and abusive requests"))
fmt.Fprintf(&b, "\n%s\n %s\n", label(fBundle, "Offline bundle"), m.bundle.View())
b.WriteString(" " + sDim.Render("For a site with no internet: engines, releases and models from 'llmgw bundle create'. Empty downloads them.") + "\n")
fmt.Fprintf(&b, "%s\n %s\n", label(fBundleSHA, "Its sha256"), m.bundleSHA.View())
b.WriteString("\n" + label(fStart, lipgloss.NewStyle().Bold(true).Render("Set up")) + "\n")
if m.formError != "" {
b.WriteString("\n" + sBad.Render(m.formError) + "\n")
+24 -2
View File
@@ -23,8 +23,9 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab},
tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}) // no bundle
m = send(m, tea.KeyMsg{Type: tea.KeyEnter})
if in.admin != "ops@example.com" || strings.Join(in.hosts, "|") != "llm.example.com|203.0.113.10" {
t.Errorf("admin %q hosts %q", in.admin, in.hosts)
@@ -37,6 +38,27 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
}
}
func TestAnOfflineBundleIsGivenInTheFormWithItsDigest(t *testing.T) {
in := &serverInit{}
var m tea.Model = newInitModel(in, nil, true)
if v := m.View(); !strings.Contains(v, "Offline bundle") {
t.Fatalf("the form offers no bundle:\n%s", v)
}
for range fBundle {
m = send(m, tea.KeyMsg{Type: tea.KeyTab})
}
m = typeIn(m, "/media/usb/b.tar")
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
if !m.(initModel).form || !strings.Contains(m.View(), "sha256") || m.(initModel).focus != fBundleSHA {
t.Fatalf("a bundle without its digest was accepted:\n%s", m.View())
}
m = typeIn(m, strings.Repeat("AB", 32))
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
if m.(initModel).form || in.bundle != "/media/usb/b.tar" || in.bundleSHA != strings.Repeat("ab", 32) {
t.Errorf("bundle %q sha %q, form up %v", in.bundle, in.bundleSHA, m.(initModel).form)
}
}
func TestAnEmailThatIsNotOneIsSaidSo(t *testing.T) {
in := &serverInit{}
var m tea.Model = newInitModel(in, nil, true)