Sign releases, and install only signed ones once a key is trusted

ed25519 over checksums.txt: llmgw release keygen makes the pair, CI signs
with the RELEASE_SIGNING_KEY secret when it is set, and release.Install
refuses an unsigned or wrongly signed release when the build embeds a
trusted key (internal/release/trusted.pub, empty for now) or
LLMGW_RELEASE_KEY names one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 23:03:48 +02:00
1 parent 561667c9be
commit 623cdb5ba3
9 files changed
+319 -4

No files matched your search

+18
View File
@@ -93,6 +93,24 @@ jobs:
./dist/llmgw-${version}-linux-amd64 --version
cat dist/checksums.txt
# Signed when the repository has a signing key, so a machine that trusts
# it installs nothing the server alone could have put there. Without the
# secret nothing is signed and releases are checked on their checksums,
# as before; a build that trusts a key refuses those, so the secret has
# to be in place before trusted.pub is filled in.
- name: Sign the checksums
env:
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
run: |
set -euo pipefail
version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then
echo "no RELEASE_SIGNING_KEY secret; releases are not signed"
exit 0
fi
./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt
echo "signed: dist/checksums.txt.sig"
- name: Attach to the release
env:
TOKEN: ${{ secrets.GITEA_TOKEN }}
+8
View File
@@ -42,6 +42,14 @@ What v0.10.5 made settable from the command line is now on the pages too:
critical model's card no longer refuses a replica that preemption would make
room for.
### Releases can be signed
`llmgw release keygen` makes an ed25519 key pair. With the private key in the
CI secret `RELEASE_SIGNING_KEY`, releases carry `checksums.txt.sig`; with the
public key in `internal/release/trusted.pub`, every binary built after refuses
a release not signed with it, wherever it installs one. Off until then, and
docs/install.md says in which order to turn it on.
### Restarts borrow a lower class's room
A restart places the replacement beside the replica it replaces, and where
+27
View File
@@ -966,6 +966,33 @@ replica alone on purpose, because churning one costs more than the difference.
`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
the new one before stopping the old.
### Signed releases
Every install above checks a release against its published checksums, which
proves the download is what the server holds — not that the server holds what
was built. Signing closes that: CI signs the checksums, and a binary that
trusts a key installs only releases signed with it. Off until a key exists;
turning it on, once:
```
llmgw release keygen
```
1. Put the **private** key in the repository's Actions secret
`RELEASE_SIGNING_KEY`. From the next tag, releases carry
`checksums.txt.sig`.
2. Put the **public** key in `internal/release/trusted.pub` and release. Every
binary built from then on carries it and refuses releases that are not
signed with it — `ctl upgrade`, agents upgrading themselves, and
`llmgw upgrade` alike.
In that order: a build that trusts a key refuses unsigned releases, so the
secret has to be signing before the key is trusted. The first trusting release
is itself installed by a binary that trusts nothing; every one after is checked
by the one before. `LLMGW_RELEASE_KEY` overrides the built-in key, for rotating
it. The onboarding script fetches with `curl` and checks checksums only; a node
is checked from its first upgrade on.
## Backing up
`llmgw server backup` writes one archive holding everything a control plane
+67
View File
@@ -0,0 +1,67 @@
package cli
import (
"errors"
"fmt"
"os"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
func newReleaseCmd() *cobra.Command {
c := &cobra.Command{
Use: "release",
Short: "Sign releases, and make the key they are signed with",
}
c.AddCommand(&cobra.Command{
Use: "keygen",
Short: "Make a release signing key pair",
Long: "Prints a public key and a private key. The private key goes into the CI\n" +
"secret RELEASE_SIGNING_KEY and nowhere else; the public key into\n" +
"internal/release/trusted.pub, which every build from then on carries\n" +
"and checks releases against. Run once, on a machine you trust.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
pub, priv, err := release.GenerateKey()
if err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "public: %s\nprivate: %s\n", pub, priv)
fmt.Fprintln(cmd.ErrOrStderr(), "\nThe private key signs every release: put it in the CI secret "+
"RELEASE_SIGNING_KEY, then clear this terminal's scrollback.")
return nil
},
})
c.AddCommand(&cobra.Command{
Use: "sign FILE",
Short: "Sign a release's checksums, writing FILE.sig",
Long: "Reads the private key from RELEASE_SIGNING_KEY, never from an argument,\n" +
"so it does not land in a shell history or a process list.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
key := os.Getenv("RELEASE_SIGNING_KEY")
if key == "" {
return errors.New("RELEASE_SIGNING_KEY is not set")
}
data, err := os.ReadFile(args[0])
if err != nil {
return err
}
sig, err := release.Sign(key, data)
if err != nil {
return err
}
if pub := release.TrustedKey(); pub != "" {
// Signed with a key this build would not accept is a release
// every upgraded machine refuses: stopped here instead.
if err := release.Verify(pub, data, sig); err != nil {
return fmt.Errorf("the signing key does not match this build's trusted key: %w", err)
}
}
return os.WriteFile(args[0]+".sig", []byte(sig+"\n"), 0o644) //nolint:gosec // public
},
})
return c
}
+1
View File
@@ -48,6 +48,7 @@ func Root() *cobra.Command {
newDoctorCmd(),
newNodeCmd(),
newSelfUpgradeCmd(),
newReleaseCmd(),
)
return root
}
+27 -4
View File
@@ -9,6 +9,7 @@ package release
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
@@ -51,7 +52,7 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar
asset := AssetName(version, goos, goarch)
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
want, err := checksum(ctx, client, root+"checksums.txt", asset)
want, err := checksum(ctx, client, root, asset)
if err != nil {
return "", err
}
@@ -85,13 +86,35 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar
return backup, nil
}
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
body, err := get(ctx, client, url)
// checksum reads the release's checksum for asset — after checking the
// checksums' signature, when this build trusts a signing key. A release with
// no signature is then refused rather than taken on its checksums: a missing
// signature is exactly what replacing a release would look like.
func checksum(ctx context.Context, client *http.Client, root, asset string) (string, error) {
body, err := get(ctx, client, root+"checksums.txt")
if err != nil {
return "", err
}
defer body.Close()
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
sums, err := io.ReadAll(io.LimitReader(body, 1<<20))
if err != nil {
return "", err
}
if key := TrustedKey(); key != "" {
sigBody, err := get(ctx, client, root+SignatureName)
if err != nil {
return "", fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err)
}
sig, err := io.ReadAll(io.LimitReader(sigBody, 4096))
sigBody.Close()
if err != nil {
return "", err
}
if err := Verify(key, sums, string(sig)); err != nil {
return "", err
}
}
sc := bufio.NewScanner(bytes.NewReader(sums))
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
+72
View File
@@ -0,0 +1,72 @@
package release
import (
"crypto/ed25519"
"crypto/rand"
_ "embed"
"encoding/base64"
"errors"
"fmt"
"os"
"strings"
)
// trustedPub is the public key releases are signed with, base64, built into
// every binary from the repository. Empty — as it is until a key is made —
// trusts a release on its checksums alone, as every release before this did.
//
// Built in rather than fetched, because the point of a signature is that the
// server the release comes from cannot vouch for it: a machine that already
// runs llmgw checks the next release against the key the release it runs was
// built with, so replacing both the binaries and the checksums on the server
// is no longer enough.
//
//go:embed trusted.pub
var trustedPub string
// TrustedKey is the key releases are verified with: LLMGW_RELEASE_KEY when set
// (for rotating a key, or testing), the built-in one otherwise; empty is none.
func TrustedKey() string {
if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" {
return k
}
return strings.TrimSpace(trustedPub)
}
// SignatureName is the file a release's checksums are signed in.
const SignatureName = "checksums.txt.sig"
// GenerateKey makes a signing key pair, both base64: the private key is the
// 32-byte seed, which is what goes into the CI secret.
func GenerateKey() (public, private string, err error) {
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return "", "", err
}
return base64.StdEncoding.EncodeToString(pub), base64.StdEncoding.EncodeToString(priv.Seed()), nil
}
// Sign signs data with a base64 seed, returning the signature in base64.
func Sign(privateSeed string, data []byte) (string, error) {
seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(privateSeed))
if err != nil || len(seed) != ed25519.SeedSize {
return "", errors.New("the signing key is not a base64 ed25519 seed")
}
return base64.StdEncoding.EncodeToString(ed25519.Sign(ed25519.NewKeyFromSeed(seed), data)), nil
}
// Verify checks a base64 signature of data against a base64 public key.
func Verify(public string, data []byte, signature string) error {
pub, err := base64.StdEncoding.DecodeString(strings.TrimSpace(public))
if err != nil || len(pub) != ed25519.PublicKeySize {
return errors.New("the trusted release key is not a base64 ed25519 public key")
}
sig, err := base64.StdEncoding.DecodeString(strings.TrimSpace(signature))
if err != nil {
return fmt.Errorf("the release signature is not base64: %w", err)
}
if !ed25519.Verify(pub, data, sig) {
return errors.New("the release's checksums are not signed by the trusted key")
}
return nil
}
+99
View File
@@ -0,0 +1,99 @@
package release
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// signedServer publishes a release whose checksums are signed with seed,
// or not signed at all when seed is empty; tamper rewrites the checksums
// after signing.
func signedServer(t *testing.T, seed string, body []byte, tamper bool) (*httptest.Server, string) {
t.Helper()
asset := AssetName("v2.0.0", "linux", "amd64")
sum := sha256.Sum256(body)
sums := hex.EncodeToString(sum[:]) + " " + asset + "\n"
sig := ""
if seed != "" {
var err error
if sig, err = Sign(seed, []byte(sums)); err != nil {
t.Fatal(err)
}
}
if tamper {
sums += strings.Repeat("0", 64) + " extra-file\n"
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v2.0.0/checksums.txt":
_, _ = w.Write([]byte(sums))
case "/v2.0.0/" + SignatureName:
if sig == "" {
http.NotFound(w, r)
return
}
_, _ = w.Write([]byte(sig + "\n"))
case "/v2.0.0/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv, asset
}
func TestABuildThatTrustsAKeyInstallsOnlyReleasesSignedWithIt(t *testing.T) {
pub, priv, err := GenerateKey()
if err != nil {
t.Fatal(err)
}
_, otherPriv, _ := GenerateKey()
t.Setenv("LLMGW_RELEASE_KEY", pub)
install := func(srv *httptest.Server) error {
target := filepath.Join(t.TempDir(), "llmgw")
_ = os.WriteFile(target, []byte("old"), 0o755)
_, err := Install(t.Context(), srv.Client(), srv.URL, "v2.0.0", "linux", "amd64", target)
return err
}
body := []byte("new binary")
good, _ := signedServer(t, priv, body, false)
if err := install(good); err != nil {
t.Fatalf("a release signed with the trusted key was refused: %v", err)
}
for name, srv := range map[string]*httptest.Server{
"unsigned": func() *httptest.Server { s, _ := signedServer(t, "", body, false); return s }(),
"signed by another": func() *httptest.Server { s, _ := signedServer(t, otherPriv, body, false); return s }(),
"checksums rewritten": func() *httptest.Server { s, _ := signedServer(t, priv, body, true); return s }(),
} {
if err := install(srv); err == nil {
t.Errorf("%s: installed", name)
}
}
// A build that trusts no key installs on checksums, as before.
t.Setenv("LLMGW_RELEASE_KEY", "")
if TrustedKey() != "" {
t.Skip("this build embeds a trusted key; the unsigned path cannot be tested here")
}
unsigned, _ := signedServer(t, "", body, false)
if err := install(unsigned); err != nil {
t.Errorf("without a trusted key an unsigned release was refused: %v", err)
}
}
func TestAKeyThatIsNotOneIsSaid(t *testing.T) {
if _, err := Sign("not-a-key", []byte("x")); err == nil {
t.Error("signed with a malformed key")
}
if err := Verify("bm9wZQ==", []byte("x"), "c2ln"); err == nil || !strings.Contains(err.Error(), "public key") {
t.Errorf("a malformed trusted key: %v", err)
}
}
View File
Whitespace-only changes.