Install only releases signed with the key built in; fetch a node's release on the control plane

trusted.pub now holds the release signing key, so self-upgrade, agent
upgrade and node onboarding refuse a release whose checksums are not signed
with it. Onboarding fetched the release on the node and compared it with a
checksum from the same server; the control plane now fetches it for the
node's architecture with release.Fetch, checks signature and checksum, and
hands the bytes over. Release addresses must be https (loopback excepted).
LLMGW_RELEASE_KEY overrides the key only in development builds. The CI
release job fails instead of publishing an unsigned release while a key is
built in.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 09:48:03 +02:00
1 parent 9461a7fc2d
commit 0bd5305e9a
10 files changed
+300 -44

No files matched your search

+13 -6
View File
@@ -93,11 +93,10 @@ jobs:
./dist/llmgw-${version}-linux-amd64 --version
cat dist/checksums.txt
# Signed when the repository has a signing key, so a machine that trusts
# it installs nothing the server alone could have put there. Without the
# secret nothing is signed and releases are checked on their checksums,
# as before; a build that trusts a key refuses those, so the secret has
# to be in place before trusted.pub is filled in.
# Signed, so a machine that trusts the key built into it installs
# nothing the server alone could have put there. `release sign` checks
# the signature against that key before writing it, so a secret that
# does not match trusted.pub stops the release too.
- name: Sign the checksums
env:
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
@@ -105,7 +104,15 @@ jobs:
set -euo pipefail
version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then
echo "no RELEASE_SIGNING_KEY secret; releases are not signed"
# Builds that carry a trusted key install only signed releases:
# publishing an unsigned one would be a release no upgraded
# machine accepts, so it stops here instead.
if [ -s internal/release/trusted.pub ]; then
echo "internal/release/trusted.pub holds a key, and the RELEASE_SIGNING_KEY secret is not set:"
echo "add it under Settings -> Secrets, or this release could not be installed anywhere."
exit 1
fi
echo "no RELEASE_SIGNING_KEY secret and no trusted key; releases are not signed"
exit 0
fi
./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt
+17
View File
@@ -23,6 +23,23 @@ APIs change between minors without a deprecation period.
## Unreleased
### Security
- **Built with Go 1.26.6.** Eleven standard-library vulnerabilities reachable
from this code at 1.26.3 (net/http, crypto/x509, os.Root, net/textproto)
are fixed by it. `make vuln` runs govulncheck; `make check` runs it with the
tests and vet.
- **Releases are signed, and only signed releases are installed.** The build
now carries the release signing key (`internal/release/trusted.pub`); the
control plane upgrading itself, an agent told to upgrade and a node being
added all refuse a release whose checksums are not signed with it. The CI
release job stops rather than publish an unsigned release. Adding a node
no longer has the node fetch the release and compare it with a checksum
from the same server: the control plane fetches it for the node's
architecture, checks the signature and the checksum, and hands it over.
Releases are fetched over https only (loopback excepted), and
`LLMGW_RELEASE_KEY` replaces the trusted key in development builds only.
### Everything that can be changed on a running cluster, from the console
`sudo llmgw` only watched the cluster and edited the configuration files;
+11 -1
View File
@@ -19,10 +19,20 @@ func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
body := []byte("the new agent")
sum := sha256.Sum256(body)
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
// Signed, as every release is: with a key this test's build trusts.
pub, priv, kerr := release.GenerateKey()
if kerr != nil {
t.Fatal(kerr)
}
t.Setenv("LLMGW_RELEASE_KEY", pub)
sums := hex.EncodeToString(sum[:]) + " " + asset + "\n"
sig, _ := release.Sign(priv, []byte(sums))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v9.9.9/checksums.txt":
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
_, _ = w.Write([]byte(sums))
case "/v9.9.9/" + release.SignatureName:
_, _ = w.Write([]byte(sig))
case "/v9.9.9/" + asset:
_, _ = w.Write(body)
default:
+40 -18
View File
@@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
@@ -88,6 +89,8 @@ type Options struct {
Binary, Version string
// Releases is where release files are downloaded from.
Releases string
// ReleaseClient fetches them; nil is a default client.
ReleaseClient *http.Client
// IgnoreChecks goes on when doctor finds a problem.
IgnoreChecks bool
// DisableEngines are engines this node never installs or advertises,
@@ -354,25 +357,25 @@ var releaseTag = release.Tag
func (r *Runner) binary(ctx context.Context, res *Result) error {
v := r.Opts.Version
if releaseTag.MatchString(v) {
r.emit(StepBinary, Running, "downloading llmgw "+v+" on the node", "")
name := "llmgw-" + v + "-linux-" + res.Arch
base := strings.TrimSuffix(r.Opts.Releases, "/") + "/" + v
// Fetched by the node, for its own architecture, and checked against
// the release's checksums, rather than handed whatever this machine
// happens to run.
script := fmt.Sprintf(`set -e
d=$(mktemp -d); cd "$d"
curl -fsSL -o %[1]s '%[2]s/%[1]s'
curl -fsSL -o checksums.txt '%[2]s/checksums.txt'
grep " %[1]s$" checksums.txt | sha256sum -c - >/dev/null
sudo install -m 0755 %[1]s /usr/local/bin/llmgw
cd /; rm -rf "$d"`, name, base)
out, err := r.Remote.Run(ctx, script, nil)
// Fetched here, for the node's architecture, and checked here —
// signature and checksum, by the same code that upgrades this
// machine — then handed to the node. The node checking a release
// itself could only compare it with a checksum from the same
// server, which proves nothing about who put both there.
r.emit(StepBinary, Running, "downloading llmgw "+v+" for "+res.Arch+" and checking it", "")
b, err := release.Fetch(ctx, r.Opts.ReleaseClient, r.Opts.Releases, v, "linux", goArch(res.Arch))
if err == nil {
r.emit(StepBinary, Done, "llmgw "+v+" from the release, checksum verified", "")
if err := r.installBinary(ctx, b); err != nil {
return err
}
what := "checksum verified"
if release.TrustedKey() != "" {
what = "signature and checksum verified"
}
r.emit(StepBinary, Done, "llmgw "+v+" from the release, "+what, "")
return nil
}
r.emit(StepBinary, Warn, "the node could not download the release: "+lastLine(out), "")
r.emit(StepBinary, Warn, "the release could not be fetched and checked here: "+err.Error(), "")
}
// A development build has no release to fetch, and a node that cannot
// reach the releases gets this machine's copy — when it can run it.
@@ -385,13 +388,32 @@ cd /; rm -rf "$d"`, name, base)
if err != nil {
return fmt.Errorf("reading %s: %w", r.Opts.Binary, err)
}
if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out))
if err := r.installBinary(ctx, b); err != nil {
return err
}
r.emit(StepBinary, Done, "llmgw "+orUnknown(v)+", copied from this machine", "")
return nil
}
// installBinary puts bytes at /usr/local/bin/llmgw on the node.
func (r *Runner) installBinary(ctx context.Context, b []byte) error {
if out, err := r.Remote.Run(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
return fmt.Errorf("installing the binary: %v: %s", err, lastLine(out))
}
return nil
}
// goArch is Go's name for what uname -m says.
func goArch(uname string) string {
switch uname {
case "x86_64", "amd64":
return "amd64"
case "aarch64", "arm64":
return "arm64"
}
return uname
}
// --- check ---
func (r *Runner) check(ctx context.Context, res *Result) error {
+69 -15
View File
@@ -2,8 +2,12 @@ package onboard
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"regexp"
"runtime"
"strings"
@@ -12,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -47,11 +52,6 @@ func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byt
return []byte("sudo: a password is required"), errors.New("exit 1")
}
return nil, nil
case strings.Contains(script, "checksums.txt"):
if f.download != nil {
return []byte("curl: (6) Could not resolve host"), f.download
}
return nil, nil
case strings.Contains(script, "install -m 0755") && stdin != nil:
f.files["/usr/local/bin/llmgw"] = stdin
return nil, nil
@@ -132,31 +132,85 @@ func TestANodeIsAddedFromItsAddressAlone(t *testing.T) {
}
}
func TestAReleaseIsFetchedByTheNodeAndCheckedThere(t *testing.T) {
// releaseFor serves one release for linux/<arch>, its checksums signed with
// a key the test then trusts; tamper changes the binary after signing.
func releaseFor(t *testing.T, version, arch string, body []byte, signed, tamper bool) string {
t.Helper()
pub, priv, err := release.GenerateKey()
if err != nil {
t.Fatal(err)
}
t.Setenv("LLMGW_RELEASE_KEY", pub)
asset := release.AssetName(version, "linux", arch)
sum := sha256.Sum256(body)
sums := hex.EncodeToString(sum[:]) + " " + asset + "\n"
sig, _ := release.Sign(priv, []byte(sums))
if tamper {
body = []byte("tampered")
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/" + version + "/checksums.txt":
_, _ = w.Write([]byte(sums))
case "/" + version + "/" + release.SignatureName:
if !signed {
http.NotFound(w, r)
return
}
_, _ = w.Write([]byte(sig))
case "/" + version + "/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv.URL
}
func TestAReleaseIsFetchedAndCheckedHereThenGivenToTheNode(t *testing.T) {
m := newFake()
r, _ := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test/releases/download"})
base := releaseFor(t, "v1.2.3", hostArch(), []byte("the release"), true, false)
r, events := runner(t, m, Options{Version: "v1.2.3", Releases: base})
if _, err := r.Run(t.Context()); err != nil {
t.Fatal(err)
}
if !m.ran("sha256sum -c") || !m.ran("https://example.test/releases/download/v1.2.3") {
t.Error("the release was not fetched and verified on the node")
if got := string(m.files["/usr/local/bin/llmgw"]); got != "the release" {
t.Errorf("the node was given %q", got)
}
if _, copied := m.files["/usr/local/bin/llmgw"]; copied {
t.Error("this machine's binary was copied although the node fetched the release")
if m.ran("curl") {
t.Error("the node fetched something itself")
}
if !said(*events, Done, "signature and checksum verified") {
t.Errorf("the check was not reported: %v", *events)
}
}
func TestANodeThatCannotDownloadGetsThisMachinesCopy(t *testing.T) {
func TestAnUnsignedOrAlteredReleaseIsNotGivenToTheNode(t *testing.T) {
for name, c := range map[string][2]bool{"unsigned": {false, false}, "altered": {true, true}} {
m := newFake()
base := releaseFor(t, "v1.2.3", hostArch(), []byte("the release"), c[0], c[1])
r, events := runner(t, m, Options{Version: "v1.2.3", Releases: base})
_, _ = r.Run(t.Context())
if got := string(m.files["/usr/local/bin/llmgw"]); got == "the release" || got == "tampered" {
t.Errorf("%s: the node was given the release", name)
}
if !said(*events, Warn, "could not be fetched and checked") {
t.Errorf("%s: the refusal was not said", name)
}
}
}
func TestANodeGetsThisMachinesCopyWhenTheReleaseCannotBeReached(t *testing.T) {
m := newFake()
m.download = errors.New("exit 6")
r, events := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test"})
r, events := runner(t, m, Options{Version: "v1.2.3", Releases: "https://127.0.0.1:1"})
if _, err := r.Run(t.Context()); err != nil {
t.Fatal(err)
}
if len(m.files["/usr/local/bin/llmgw"]) == 0 {
t.Error("the binary was not copied")
}
if !said(*events, Warn, "could not download") {
if !said(*events, Warn, "could not be fetched") {
t.Error("the failed download was not mentioned")
}
}
+74
View File
@@ -15,7 +15,9 @@ import (
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
@@ -41,6 +43,9 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar
if !Tag.MatchString(version) {
return "", fmt.Errorf("%q is not a release version", version)
}
if err := CheckBase(base); err != nil {
return "", err
}
if client == nil {
client = &http.Client{Timeout: 10 * time.Minute}
}
@@ -86,6 +91,75 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar
return backup, nil
}
// CheckBase refuses a release address a binary should not come from: one
// over plain HTTP, where anyone on the path can swap the bytes and, without a
// signing key, the checksums with them. Loopback is allowed, for a mirror on
// this machine and for tests.
func CheckBase(base string) error {
u, err := url.Parse(base)
if err != nil || u.Host == "" {
return fmt.Errorf("%q is not a release address", base)
}
switch u.Scheme {
case "https":
return nil
case "http":
if ip := net.ParseIP(u.Hostname()); (ip != nil && ip.IsLoopback()) || u.Hostname() == "localhost" {
return nil
}
}
return fmt.Errorf("releases are fetched over https, and %s is not", base)
}
// maxBinary bounds a release binary read into memory.
const maxBinary = 512 << 20
// Fetch downloads a release binary for a platform and checks it exactly as
// Install does — signature when this build trusts a key, then checksum —
// returning its bytes: for a machine that installs it on another, which
// then runs nothing that was not checked here.
func Fetch(ctx context.Context, client *http.Client, base, version, goos, goarch string) ([]byte, error) {
if !Tag.MatchString(version) {
return nil, fmt.Errorf("%q is not a release version", version)
}
if err := CheckBase(base); err != nil {
return nil, err
}
if client == nil {
client = &http.Client{Timeout: 10 * time.Minute}
}
asset := AssetName(version, goos, goarch)
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
want, err := checksum(ctx, client, root, asset)
if err != nil {
return nil, err
}
var buf bytes.Buffer
got, err := download(ctx, client, root+asset, &limitWriter{w: &buf, n: maxBinary})
if err != nil {
return nil, err
}
if got != want {
return nil, fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
asset, got[:12], want[:12])
}
return buf.Bytes(), nil
}
// limitWriter refuses to grow past n bytes.
type limitWriter struct {
w io.Writer
n int64
}
func (l *limitWriter) Write(p []byte) (int, error) {
if int64(len(p)) > l.n {
return 0, fmt.Errorf("the release binary is larger than %d bytes", maxBinary)
}
l.n -= int64(len(p))
return l.w.Write(p)
}
// checksum reads the release's checksum for asset — after checking the
// checksums' signature, when this build trusts a signing key. A release with
// no signature is then refused rather than taken on its checksums: a missing
+45
View File
@@ -26,7 +26,17 @@ func releaseServer(t *testing.T, version, asset string, body []byte, sum string)
return srv
}
// unsigned has this build trust no key for the test: the checksum alone.
func unsigned(t *testing.T) {
t.Helper()
was := trustedPub
trustedPub = ""
t.Setenv("LLMGW_RELEASE_KEY", "")
t.Cleanup(func() { trustedPub = was })
}
func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
unsigned(t)
dir := t.TempDir()
target := filepath.Join(dir, "llmgw")
_ = os.WriteFile(target, []byte("old"), 0o755)
@@ -59,3 +69,38 @@ func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
t.Error("a version that is not one was fetched")
}
}
func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) {
for base, ok := range map[string]bool{
"https://gitea.example/releases/download": true,
"http://127.0.0.1:8080/r": true,
"http://localhost/r": true,
"http://gitea.example/releases/download": false,
"http://10.0.0.5/r": false,
"ftp://x/r": false,
"not a url": false,
} {
if err := CheckBase(base); (err == nil) != ok {
t.Errorf("%s: %v", base, err)
}
}
if _, err := Install(t.Context(), nil, "http://gitea.example/r", "v1.2.3", "linux", "amd64", "/nonexistent"); err == nil {
t.Error("a release over plain http was installed")
}
}
func TestFetchGivesOnlyACheckedBinary(t *testing.T) {
unsigned(t)
body := []byte("for another machine")
sum := sha256.Sum256(body)
asset := AssetName("v1.2.3", "linux", "arm64")
srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:]))
got, err := Fetch(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "arm64")
if err != nil || string(got) != string(body) {
t.Fatalf("%q %v", got, err)
}
bad := releaseServer(t, "v1.2.3", asset, []byte("swapped"), hex.EncodeToString(sum[:]))
if _, err := Fetch(t.Context(), bad.Client(), bad.URL, "v1.2.3", "linux", "arm64"); err == nil {
t.Error("a binary not matching its checksum was given")
}
}
+11 -4
View File
@@ -9,6 +9,8 @@ import (
"fmt"
"os"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// trustedPub is the public key releases are signed with, base64, built into
@@ -24,11 +26,16 @@ import (
//go:embed trusted.pub
var trustedPub string
// TrustedKey is the key releases are verified with: LLMGW_RELEASE_KEY when set
// (for rotating a key, or testing), the built-in one otherwise; empty is none.
// TrustedKey is the key releases are verified with: the built-in one; empty
// is none. LLMGW_RELEASE_KEY replaces it in a development build — for testing
// a release pipeline — and never in a release: a key taken from the
// environment would let whatever sets the environment of an upgrading
// process choose what it trusts.
func TrustedKey() string {
if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" {
return k
if !Tag.MatchString(version.Version) {
if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" {
return k
}
}
return strings.TrimSpace(trustedPub)
}
+19
View File
@@ -9,6 +9,8 @@ import (
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// signedServer publishes a release whose checksums are signed with seed,
@@ -97,3 +99,20 @@ func TestAKeyThatIsNotOneIsSaid(t *testing.T) {
t.Errorf("a malformed trusted key: %v", err)
}
}
func TestAReleaseBuildTakesNoKeyFromTheEnvironment(t *testing.T) {
pub, _, _ := GenerateKey()
t.Setenv("LLMGW_RELEASE_KEY", pub)
was, built := version.Version, trustedPub
t.Cleanup(func() { version.Version, trustedPub = was, built })
trustedPub = "the built-in key"
version.Version = "dev"
if TrustedKey() != pub {
t.Error("a development build ignored LLMGW_RELEASE_KEY")
}
version.Version = "v1.2.3"
if TrustedKey() != "the built-in key" {
t.Error("a release build took its trusted key from the environment")
}
}
+1
View File
@@ -0,0 +1 @@
DrmKRQmeI7w0xSnBThCpfMvyqBVrxYcbJGJT52fl6J8=