From 0bd5305e9af13d9664b2090f565b4d682e875d46 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 09:48:03 +0200 Subject: [PATCH] Install only releases signed with the key built in; fetch a node's release on the control plane trusted.pub now holds the release signing key, so self-upgrade, agent upgrade and node onboarding refuse a release whose checksums are not signed with it. Onboarding fetched the release on the node and compared it with a checksum from the same server; the control plane now fetches it for the node's architecture with release.Fetch, checks signature and checksum, and hands the bytes over. Release addresses must be https (loopback excepted). LLMGW_RELEASE_KEY overrides the key only in development builds. The CI release job fails instead of publishing an unsigned release while a key is built in. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitea/workflows/release.yml | 19 +++++--- CHANGELOG.md | 17 +++++++ internal/agent/upgrade_test.go | 12 ++++- internal/onboard/onboard.go | 58 +++++++++++++++------- internal/onboard/onboard_test.go | 84 ++++++++++++++++++++++++++------ internal/release/release.go | 74 ++++++++++++++++++++++++++++ internal/release/release_test.go | 45 +++++++++++++++++ internal/release/sign.go | 15 ++++-- internal/release/sign_test.go | 19 ++++++++ internal/release/trusted.pub | 1 + 10 files changed, 300 insertions(+), 44 deletions(-) diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 80c85ae..a557848 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -93,11 +93,10 @@ jobs: ./dist/llmgw-${version}-linux-amd64 --version cat dist/checksums.txt - # Signed when the repository has a signing key, so a machine that trusts - # it installs nothing the server alone could have put there. Without the - # secret nothing is signed and releases are checked on their checksums, - # as before; a build that trusts a key refuses those, so the secret has - # to be in place before trusted.pub is filled in. + # Signed, so a machine that trusts the key built into it installs + # nothing the server alone could have put there. `release sign` checks + # the signature against that key before writing it, so a secret that + # does not match trusted.pub stops the release too. - name: Sign the checksums env: RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }} @@ -105,7 +104,15 @@ jobs: set -euo pipefail version="${GITHUB_REF_NAME:-$(git describe --tags --always)}" if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then - echo "no RELEASE_SIGNING_KEY secret; releases are not signed" + # Builds that carry a trusted key install only signed releases: + # publishing an unsigned one would be a release no upgraded + # machine accepts, so it stops here instead. + if [ -s internal/release/trusted.pub ]; then + echo "internal/release/trusted.pub holds a key, and the RELEASE_SIGNING_KEY secret is not set:" + echo "add it under Settings -> Secrets, or this release could not be installed anywhere." + exit 1 + fi + echo "no RELEASE_SIGNING_KEY secret and no trusted key; releases are not signed" exit 0 fi ./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d6e08..e5b952d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,23 @@ APIs change between minors without a deprecation period. ## Unreleased +### Security + +- **Built with Go 1.26.6.** Eleven standard-library vulnerabilities reachable + from this code at 1.26.3 (net/http, crypto/x509, os.Root, net/textproto) + are fixed by it. `make vuln` runs govulncheck; `make check` runs it with the + tests and vet. +- **Releases are signed, and only signed releases are installed.** The build + now carries the release signing key (`internal/release/trusted.pub`); the + control plane upgrading itself, an agent told to upgrade and a node being + added all refuse a release whose checksums are not signed with it. The CI + release job stops rather than publish an unsigned release. Adding a node + no longer has the node fetch the release and compare it with a checksum + from the same server: the control plane fetches it for the node's + architecture, checks the signature and the checksum, and hands it over. + Releases are fetched over https only (loopback excepted), and + `LLMGW_RELEASE_KEY` replaces the trusted key in development builds only. + ### Everything that can be changed on a running cluster, from the console `sudo llmgw` only watched the cluster and edited the configuration files; diff --git a/internal/agent/upgrade_test.go b/internal/agent/upgrade_test.go index 5a67824..9756cbb 100644 --- a/internal/agent/upgrade_test.go +++ b/internal/agent/upgrade_test.go @@ -19,10 +19,20 @@ func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) { body := []byte("the new agent") sum := sha256.Sum256(body) asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH) + // Signed, as every release is: with a key this test's build trusts. + pub, priv, kerr := release.GenerateKey() + if kerr != nil { + t.Fatal(kerr) + } + t.Setenv("LLMGW_RELEASE_KEY", pub) + sums := hex.EncodeToString(sum[:]) + " " + asset + "\n" + sig, _ := release.Sign(priv, []byte(sums)) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v9.9.9/checksums.txt": - _, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n")) + _, _ = w.Write([]byte(sums)) + case "/v9.9.9/" + release.SignatureName: + _, _ = w.Write([]byte(sig)) case "/v9.9.9/" + asset: _, _ = w.Write(body) default: diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 4217337..2986503 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net" + "net/http" "os" "path/filepath" "regexp" @@ -88,6 +89,8 @@ type Options struct { Binary, Version string // Releases is where release files are downloaded from. Releases string + // ReleaseClient fetches them; nil is a default client. + ReleaseClient *http.Client // IgnoreChecks goes on when doctor finds a problem. IgnoreChecks bool // DisableEngines are engines this node never installs or advertises, @@ -354,25 +357,25 @@ var releaseTag = release.Tag func (r *Runner) binary(ctx context.Context, res *Result) error { v := r.Opts.Version if releaseTag.MatchString(v) { - r.emit(StepBinary, Running, "downloading llmgw "+v+" on the node", "") - name := "llmgw-" + v + "-linux-" + res.Arch - base := strings.TrimSuffix(r.Opts.Releases, "/") + "/" + v - // Fetched by the node, for its own architecture, and checked against - // the release's checksums, rather than handed whatever this machine - // happens to run. - script := fmt.Sprintf(`set -e -d=$(mktemp -d); cd "$d" -curl -fsSL -o %[1]s '%[2]s/%[1]s' -curl -fsSL -o checksums.txt '%[2]s/checksums.txt' -grep " %[1]s$" checksums.txt | sha256sum -c - >/dev/null -sudo install -m 0755 %[1]s /usr/local/bin/llmgw -cd /; rm -rf "$d"`, name, base) - out, err := r.Remote.Run(ctx, script, nil) + // Fetched here, for the node's architecture, and checked here — + // signature and checksum, by the same code that upgrades this + // machine — then handed to the node. The node checking a release + // itself could only compare it with a checksum from the same + // server, which proves nothing about who put both there. + r.emit(StepBinary, Running, "downloading llmgw "+v+" for "+res.Arch+" and checking it", "") + b, err := release.Fetch(ctx, r.Opts.ReleaseClient, r.Opts.Releases, v, "linux", goArch(res.Arch)) if err == nil { - r.emit(StepBinary, Done, "llmgw "+v+" from the release, checksum verified", "") + if err := r.installBinary(ctx, b); err != nil { + return err + } + what := "checksum verified" + if release.TrustedKey() != "" { + what = "signature and checksum verified" + } + r.emit(StepBinary, Done, "llmgw "+v+" from the release, "+what, "") return nil } - r.emit(StepBinary, Warn, "the node could not download the release: "+lastLine(out), "") + r.emit(StepBinary, Warn, "the release could not be fetched and checked here: "+err.Error(), "") } // A development build has no release to fetch, and a node that cannot // reach the releases gets this machine's copy — when it can run it. @@ -385,13 +388,32 @@ cd /; rm -rf "$d"`, name, base) if err != nil { return fmt.Errorf("reading %s: %w", r.Opts.Binary, err) } - if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil { - return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out)) + if err := r.installBinary(ctx, b); err != nil { + return err } r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "") return nil } +// installBinary puts bytes at /usr/local/bin/llmgw on the node. +func (r *Runner) installBinary(ctx context.Context, b []byte) error { + if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil { + return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out)) + } + return nil +} + +// goArch is Go's name for what uname -m says. +func goArch(uname string) string { + switch uname { + case "x86_64", "amd64": + return "amd64" + case "aarch64", "arm64": + return "arm64" + } + return uname +} + // --- check --- func (r *Runner) check(ctx context.Context, res *Result) error { diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go index 81991ae..2b2ae55 100644 --- a/internal/onboard/onboard_test.go +++ b/internal/onboard/onboard_test.go @@ -2,8 +2,12 @@ package onboard import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" + "net/http" + "net/http/httptest" "regexp" "runtime" "strings" @@ -12,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/doctor" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) @@ -47,11 +52,6 @@ func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byt return []byte("sudo: a password is required"), errors.New("exit 1") } return nil, nil - case strings.Contains(script, "checksums.txt"): - if f.download != nil { - return []byte("curl: (6) Could not resolve host"), f.download - } - return nil, nil case strings.Contains(script, "install -m 0755") && stdin != nil: f.files["/usr/local/bin/llmgw"] = stdin return nil, nil @@ -132,31 +132,85 @@ func TestANodeIsAddedFromItsAddressAlone(t *testing.T) { } } -func TestAReleaseIsFetchedByTheNodeAndCheckedThere(t *testing.T) { +// releaseFor serves one release for linux/, its checksums signed with +// a key the test then trusts; tamper changes the binary after signing. +func releaseFor(t *testing.T, version, arch string, body []byte, signed, tamper bool) string { + t.Helper() + pub, priv, err := release.GenerateKey() + if err != nil { + t.Fatal(err) + } + t.Setenv("LLMGW_RELEASE_KEY", pub) + asset := release.AssetName(version, "linux", arch) + sum := sha256.Sum256(body) + sums := hex.EncodeToString(sum[:]) + " " + asset + "\n" + sig, _ := release.Sign(priv, []byte(sums)) + if tamper { + body = []byte("tampered") + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/" + version + "/checksums.txt": + _, _ = w.Write([]byte(sums)) + case "/" + version + "/" + release.SignatureName: + if !signed { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(sig)) + case "/" + version + "/" + asset: + _, _ = w.Write(body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv.URL +} + +func TestAReleaseIsFetchedAndCheckedHereThenGivenToTheNode(t *testing.T) { m := newFake() - r, _ := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test/releases/download"}) + base := releaseFor(t, "v1.2.3", hostArch(), []byte("the release"), true, false) + r, events := runner(t, m, Options{Version: "v1.2.3", Releases: base}) if _, err := r.Run(t.Context()); err != nil { t.Fatal(err) } - if !m.ran("sha256sum -c") || !m.ran("https://example.test/releases/download/v1.2.3") { - t.Error("the release was not fetched and verified on the node") + if got := string(m.files["/usr/local/bin/llmgw"]); got != "the release" { + t.Errorf("the node was given %q", got) } - if _, copied := m.files["/usr/local/bin/llmgw"]; copied { - t.Error("this machine's binary was copied although the node fetched the release") + if m.ran("curl") { + t.Error("the node fetched something itself") + } + if !said(*events, Done, "signature and checksum verified") { + t.Errorf("the check was not reported: %v", *events) } } -func TestANodeThatCannotDownloadGetsThisMachinesCopy(t *testing.T) { +func TestAnUnsignedOrAlteredReleaseIsNotGivenToTheNode(t *testing.T) { + for name, c := range map[string][2]bool{"unsigned": {false, false}, "altered": {true, true}} { + m := newFake() + base := releaseFor(t, "v1.2.3", hostArch(), []byte("the release"), c[0], c[1]) + r, events := runner(t, m, Options{Version: "v1.2.3", Releases: base}) + _, _ = r.Run(t.Context()) + if got := string(m.files["/usr/local/bin/llmgw"]); got == "the release" || got == "tampered" { + t.Errorf("%s: the node was given the release", name) + } + if !said(*events, Warn, "could not be fetched and checked") { + t.Errorf("%s: the refusal was not said", name) + } + } +} + +func TestANodeGetsThisMachinesCopyWhenTheReleaseCannotBeReached(t *testing.T) { m := newFake() - m.download = errors.New("exit 6") - r, events := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test"}) + r, events := runner(t, m, Options{Version: "v1.2.3", Releases: "https://127.0.0.1:1"}) if _, err := r.Run(t.Context()); err != nil { t.Fatal(err) } if len(m.files["/usr/local/bin/llmgw"]) == 0 { t.Error("the binary was not copied") } - if !said(*events, Warn, "could not download") { + if !said(*events, Warn, "could not be fetched") { t.Error("the failed download was not mentioned") } } diff --git a/internal/release/release.go b/internal/release/release.go index b1d1cab..1cd656a 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -15,7 +15,9 @@ import ( "encoding/hex" "fmt" "io" + "net" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -41,6 +43,9 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar if !Tag.MatchString(version) { return "", fmt.Errorf("%q is not a release version", version) } + if err := CheckBase(base); err != nil { + return "", err + } if client == nil { client = &http.Client{Timeout: 10 * time.Minute} } @@ -86,6 +91,75 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar return backup, nil } +// CheckBase refuses a release address a binary should not come from: one +// over plain HTTP, where anyone on the path can swap the bytes and, without a +// signing key, the checksums with them. Loopback is allowed, for a mirror on +// this machine and for tests. +func CheckBase(base string) error { + u, err := url.Parse(base) + if err != nil || u.Host == "" { + return fmt.Errorf("%q is not a release address", base) + } + switch u.Scheme { + case "https": + return nil + case "http": + if ip := net.ParseIP(u.Hostname()); (ip != nil && ip.IsLoopback()) || u.Hostname() == "localhost" { + return nil + } + } + return fmt.Errorf("releases are fetched over https, and %s is not", base) +} + +// maxBinary bounds a release binary read into memory. +const maxBinary = 512 << 20 + +// Fetch downloads a release binary for a platform and checks it exactly as +// Install does — signature when this build trusts a key, then checksum — +// returning its bytes: for a machine that installs it on another, which +// then runs nothing that was not checked here. +func Fetch(ctx context.Context, client *http.Client, base, version, goos, goarch string) ([]byte, error) { + if !Tag.MatchString(version) { + return nil, fmt.Errorf("%q is not a release version", version) + } + if err := CheckBase(base); err != nil { + return nil, err + } + if client == nil { + client = &http.Client{Timeout: 10 * time.Minute} + } + asset := AssetName(version, goos, goarch) + root := strings.TrimSuffix(base, "/") + "/" + version + "/" + want, err := checksum(ctx, client, root, asset) + if err != nil { + return nil, err + } + var buf bytes.Buffer + got, err := download(ctx, client, root+asset, &limitWriter{w: &buf, n: maxBinary}) + if err != nil { + return nil, err + } + if got != want { + return nil, fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)", + asset, got[:12], want[:12]) + } + return buf.Bytes(), nil +} + +// limitWriter refuses to grow past n bytes. +type limitWriter struct { + w io.Writer + n int64 +} + +func (l *limitWriter) Write(p []byte) (int, error) { + if int64(len(p)) > l.n { + return 0, fmt.Errorf("the release binary is larger than %d bytes", maxBinary) + } + l.n -= int64(len(p)) + return l.w.Write(p) +} + // checksum reads the release's checksum for asset — after checking the // checksums' signature, when this build trusts a signing key. A release with // no signature is then refused rather than taken on its checksums: a missing diff --git a/internal/release/release_test.go b/internal/release/release_test.go index b57f0bd..ac2d895 100644 --- a/internal/release/release_test.go +++ b/internal/release/release_test.go @@ -26,7 +26,17 @@ func releaseServer(t *testing.T, version, asset string, body []byte, sum string) return srv } +// unsigned has this build trust no key for the test: the checksum alone. +func unsigned(t *testing.T) { + t.Helper() + was := trustedPub + trustedPub = "" + t.Setenv("LLMGW_RELEASE_KEY", "") + t.Cleanup(func() { trustedPub = was }) +} + func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) { + unsigned(t) dir := t.TempDir() target := filepath.Join(dir, "llmgw") _ = os.WriteFile(target, []byte("old"), 0o755) @@ -59,3 +69,38 @@ func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) { t.Error("a version that is not one was fetched") } } + +func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) { + for base, ok := range map[string]bool{ + "https://gitea.example/releases/download": true, + "http://127.0.0.1:8080/r": true, + "http://localhost/r": true, + "http://gitea.example/releases/download": false, + "http://10.0.0.5/r": false, + "ftp://x/r": false, + "not a url": false, + } { + if err := CheckBase(base); (err == nil) != ok { + t.Errorf("%s: %v", base, err) + } + } + if _, err := Install(t.Context(), nil, "http://gitea.example/r", "v1.2.3", "linux", "amd64", "/nonexistent"); err == nil { + t.Error("a release over plain http was installed") + } +} + +func TestFetchGivesOnlyACheckedBinary(t *testing.T) { + unsigned(t) + body := []byte("for another machine") + sum := sha256.Sum256(body) + asset := AssetName("v1.2.3", "linux", "arm64") + srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:])) + got, err := Fetch(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "arm64") + if err != nil || string(got) != string(body) { + t.Fatalf("%q %v", got, err) + } + bad := releaseServer(t, "v1.2.3", asset, []byte("swapped"), hex.EncodeToString(sum[:])) + if _, err := Fetch(t.Context(), bad.Client(), bad.URL, "v1.2.3", "linux", "arm64"); err == nil { + t.Error("a binary not matching its checksum was given") + } +} diff --git a/internal/release/sign.go b/internal/release/sign.go index c89caf5..c001488 100644 --- a/internal/release/sign.go +++ b/internal/release/sign.go @@ -9,6 +9,8 @@ import ( "fmt" "os" "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) // trustedPub is the public key releases are signed with, base64, built into @@ -24,11 +26,16 @@ import ( //go:embed trusted.pub var trustedPub string -// TrustedKey is the key releases are verified with: LLMGW_RELEASE_KEY when set -// (for rotating a key, or testing), the built-in one otherwise; empty is none. +// TrustedKey is the key releases are verified with: the built-in one; empty +// is none. LLMGW_RELEASE_KEY replaces it in a development build — for testing +// a release pipeline — and never in a release: a key taken from the +// environment would let whatever sets the environment of an upgrading +// process choose what it trusts. func TrustedKey() string { - if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" { - return k + if !Tag.MatchString(version.Version) { + if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" { + return k + } } return strings.TrimSpace(trustedPub) } diff --git a/internal/release/sign_test.go b/internal/release/sign_test.go index 130da1f..c2a429e 100644 --- a/internal/release/sign_test.go +++ b/internal/release/sign_test.go @@ -9,6 +9,8 @@ import ( "path/filepath" "strings" "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) // signedServer publishes a release whose checksums are signed with seed, @@ -97,3 +99,20 @@ func TestAKeyThatIsNotOneIsSaid(t *testing.T) { t.Errorf("a malformed trusted key: %v", err) } } + +func TestAReleaseBuildTakesNoKeyFromTheEnvironment(t *testing.T) { + pub, _, _ := GenerateKey() + t.Setenv("LLMGW_RELEASE_KEY", pub) + was, built := version.Version, trustedPub + t.Cleanup(func() { version.Version, trustedPub = was, built }) + trustedPub = "the built-in key" + + version.Version = "dev" + if TrustedKey() != pub { + t.Error("a development build ignored LLMGW_RELEASE_KEY") + } + version.Version = "v1.2.3" + if TrustedKey() != "the built-in key" { + t.Error("a release build took its trusted key from the environment") + } +} diff --git a/internal/release/trusted.pub b/internal/release/trusted.pub index e69de29..69a4024 100644 --- a/internal/release/trusted.pub +++ b/internal/release/trusted.pub @@ -0,0 +1 @@ +DrmKRQmeI7w0xSnBThCpfMvyqBVrxYcbJGJT52fl6J8=