diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 4dc30fe..80c85ae 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -93,6 +93,24 @@ jobs: ./dist/llmgw-${version}-linux-amd64 --version cat dist/checksums.txt + # Signed when the repository has a signing key, so a machine that trusts + # it installs nothing the server alone could have put there. Without the + # secret nothing is signed and releases are checked on their checksums, + # as before; a build that trusts a key refuses those, so the secret has + # to be in place before trusted.pub is filled in. + - name: Sign the checksums + env: + RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }} + run: | + set -euo pipefail + version="${GITHUB_REF_NAME:-$(git describe --tags --always)}" + if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then + echo "no RELEASE_SIGNING_KEY secret; releases are not signed" + exit 0 + fi + ./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt + echo "signed: dist/checksums.txt.sig" + - name: Attach to the release env: TOKEN: ${{ secrets.GITEA_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index afc4971..5406d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,14 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### Releases can be signed + +`llmgw release keygen` makes an ed25519 key pair. With the private key in the +CI secret `RELEASE_SIGNING_KEY`, releases carry `checksums.txt.sig`; with the +public key in `internal/release/trusted.pub`, every binary built after refuses +a release not signed with it, wherever it installs one. Off until then, and +docs/install.md says in which order to turn it on. + ### Restarts borrow a lower class's room A restart places the replacement beside the replica it replaces, and where diff --git a/docs/install.md b/docs/install.md index 1d11953..f6d93a5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -966,6 +966,33 @@ replica alone on purpose, because churning one costs more than the difference. `llmgw ctl restart ` replaces it with one on the new profile, starting the new one before stopping the old. +### Signed releases + +Every install above checks a release against its published checksums, which +proves the download is what the server holds — not that the server holds what +was built. Signing closes that: CI signs the checksums, and a binary that +trusts a key installs only releases signed with it. Off until a key exists; +turning it on, once: + +``` +llmgw release keygen +``` + +1. Put the **private** key in the repository's Actions secret + `RELEASE_SIGNING_KEY`. From the next tag, releases carry + `checksums.txt.sig`. +2. Put the **public** key in `internal/release/trusted.pub` and release. Every + binary built from then on carries it and refuses releases that are not + signed with it — `ctl upgrade`, agents upgrading themselves, and + `llmgw upgrade` alike. + +In that order: a build that trusts a key refuses unsigned releases, so the +secret has to be signing before the key is trusted. The first trusting release +is itself installed by a binary that trusts nothing; every one after is checked +by the one before. `LLMGW_RELEASE_KEY` overrides the built-in key, for rotating +it. The onboarding script fetches with `curl` and checks checksums only; a node +is checked from its first upgrade on. + ## Backing up `llmgw server backup` writes one archive holding everything a control plane diff --git a/internal/cli/release.go b/internal/cli/release.go new file mode 100644 index 0000000..fa3eb12 --- /dev/null +++ b/internal/cli/release.go @@ -0,0 +1,67 @@ +package cli + +import ( + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/release" +) + +func newReleaseCmd() *cobra.Command { + c := &cobra.Command{ + Use: "release", + Short: "Sign releases, and make the key they are signed with", + } + c.AddCommand(&cobra.Command{ + Use: "keygen", + Short: "Make a release signing key pair", + Long: "Prints a public key and a private key. The private key goes into the CI\n" + + "secret RELEASE_SIGNING_KEY and nowhere else; the public key into\n" + + "internal/release/trusted.pub, which every build from then on carries\n" + + "and checks releases against. Run once, on a machine you trust.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + pub, priv, err := release.GenerateKey() + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "public: %s\nprivate: %s\n", pub, priv) + fmt.Fprintln(cmd.ErrOrStderr(), "\nThe private key signs every release: put it in the CI secret "+ + "RELEASE_SIGNING_KEY, then clear this terminal's scrollback.") + return nil + }, + }) + c.AddCommand(&cobra.Command{ + Use: "sign FILE", + Short: "Sign a release's checksums, writing FILE.sig", + Long: "Reads the private key from RELEASE_SIGNING_KEY, never from an argument,\n" + + "so it does not land in a shell history or a process list.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + key := os.Getenv("RELEASE_SIGNING_KEY") + if key == "" { + return errors.New("RELEASE_SIGNING_KEY is not set") + } + data, err := os.ReadFile(args[0]) + if err != nil { + return err + } + sig, err := release.Sign(key, data) + if err != nil { + return err + } + if pub := release.TrustedKey(); pub != "" { + // Signed with a key this build would not accept is a release + // every upgraded machine refuses: stopped here instead. + if err := release.Verify(pub, data, sig); err != nil { + return fmt.Errorf("the signing key does not match this build's trusted key: %w", err) + } + } + return os.WriteFile(args[0]+".sig", []byte(sig+"\n"), 0o644) //nolint:gosec // public + }, + }) + return c +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 573733e..1245bc8 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -48,6 +48,7 @@ func Root() *cobra.Command { newDoctorCmd(), newNodeCmd(), newSelfUpgradeCmd(), + newReleaseCmd(), ) return root } diff --git a/internal/release/release.go b/internal/release/release.go index d1c64f9..b1d1cab 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -9,6 +9,7 @@ package release import ( "bufio" + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -51,7 +52,7 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar asset := AssetName(version, goos, goarch) root := strings.TrimSuffix(base, "/") + "/" + version + "/" - want, err := checksum(ctx, client, root+"checksums.txt", asset) + want, err := checksum(ctx, client, root, asset) if err != nil { return "", err } @@ -85,13 +86,35 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar return backup, nil } -func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) { - body, err := get(ctx, client, url) +// checksum reads the release's checksum for asset — after checking the +// checksums' signature, when this build trusts a signing key. A release with +// no signature is then refused rather than taken on its checksums: a missing +// signature is exactly what replacing a release would look like. +func checksum(ctx context.Context, client *http.Client, root, asset string) (string, error) { + body, err := get(ctx, client, root+"checksums.txt") if err != nil { return "", err } defer body.Close() - sc := bufio.NewScanner(io.LimitReader(body, 1<<20)) + sums, err := io.ReadAll(io.LimitReader(body, 1<<20)) + if err != nil { + return "", err + } + if key := TrustedKey(); key != "" { + sigBody, err := get(ctx, client, root+SignatureName) + if err != nil { + return "", fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err) + } + sig, err := io.ReadAll(io.LimitReader(sigBody, 4096)) + sigBody.Close() + if err != nil { + return "", err + } + if err := Verify(key, sums, string(sig)); err != nil { + return "", err + } + } + sc := bufio.NewScanner(bytes.NewReader(sums)) for sc.Scan() { f := strings.Fields(sc.Text()) if len(f) == 2 && f[1] == asset && len(f[0]) == 64 { diff --git a/internal/release/sign.go b/internal/release/sign.go new file mode 100644 index 0000000..c89caf5 --- /dev/null +++ b/internal/release/sign.go @@ -0,0 +1,72 @@ +package release + +import ( + "crypto/ed25519" + "crypto/rand" + _ "embed" + "encoding/base64" + "errors" + "fmt" + "os" + "strings" +) + +// trustedPub is the public key releases are signed with, base64, built into +// every binary from the repository. Empty — as it is until a key is made — +// trusts a release on its checksums alone, as every release before this did. +// +// Built in rather than fetched, because the point of a signature is that the +// server the release comes from cannot vouch for it: a machine that already +// runs llmgw checks the next release against the key the release it runs was +// built with, so replacing both the binaries and the checksums on the server +// is no longer enough. +// +//go:embed trusted.pub +var trustedPub string + +// TrustedKey is the key releases are verified with: LLMGW_RELEASE_KEY when set +// (for rotating a key, or testing), the built-in one otherwise; empty is none. +func TrustedKey() string { + if k := strings.TrimSpace(os.Getenv("LLMGW_RELEASE_KEY")); k != "" { + return k + } + return strings.TrimSpace(trustedPub) +} + +// SignatureName is the file a release's checksums are signed in. +const SignatureName = "checksums.txt.sig" + +// GenerateKey makes a signing key pair, both base64: the private key is the +// 32-byte seed, which is what goes into the CI secret. +func GenerateKey() (public, private string, err error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return "", "", err + } + return base64.StdEncoding.EncodeToString(pub), base64.StdEncoding.EncodeToString(priv.Seed()), nil +} + +// Sign signs data with a base64 seed, returning the signature in base64. +func Sign(privateSeed string, data []byte) (string, error) { + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(privateSeed)) + if err != nil || len(seed) != ed25519.SeedSize { + return "", errors.New("the signing key is not a base64 ed25519 seed") + } + return base64.StdEncoding.EncodeToString(ed25519.Sign(ed25519.NewKeyFromSeed(seed), data)), nil +} + +// Verify checks a base64 signature of data against a base64 public key. +func Verify(public string, data []byte, signature string) error { + pub, err := base64.StdEncoding.DecodeString(strings.TrimSpace(public)) + if err != nil || len(pub) != ed25519.PublicKeySize { + return errors.New("the trusted release key is not a base64 ed25519 public key") + } + sig, err := base64.StdEncoding.DecodeString(strings.TrimSpace(signature)) + if err != nil { + return fmt.Errorf("the release signature is not base64: %w", err) + } + if !ed25519.Verify(pub, data, sig) { + return errors.New("the release's checksums are not signed by the trusted key") + } + return nil +} diff --git a/internal/release/sign_test.go b/internal/release/sign_test.go new file mode 100644 index 0000000..130da1f --- /dev/null +++ b/internal/release/sign_test.go @@ -0,0 +1,99 @@ +package release + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// signedServer publishes a release whose checksums are signed with seed, +// or not signed at all when seed is empty; tamper rewrites the checksums +// after signing. +func signedServer(t *testing.T, seed string, body []byte, tamper bool) (*httptest.Server, string) { + t.Helper() + asset := AssetName("v2.0.0", "linux", "amd64") + sum := sha256.Sum256(body) + sums := hex.EncodeToString(sum[:]) + " " + asset + "\n" + sig := "" + if seed != "" { + var err error + if sig, err = Sign(seed, []byte(sums)); err != nil { + t.Fatal(err) + } + } + if tamper { + sums += strings.Repeat("0", 64) + " extra-file\n" + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2.0.0/checksums.txt": + _, _ = w.Write([]byte(sums)) + case "/v2.0.0/" + SignatureName: + if sig == "" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(sig + "\n")) + case "/v2.0.0/" + asset: + _, _ = w.Write(body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, asset +} + +func TestABuildThatTrustsAKeyInstallsOnlyReleasesSignedWithIt(t *testing.T) { + pub, priv, err := GenerateKey() + if err != nil { + t.Fatal(err) + } + _, otherPriv, _ := GenerateKey() + t.Setenv("LLMGW_RELEASE_KEY", pub) + install := func(srv *httptest.Server) error { + target := filepath.Join(t.TempDir(), "llmgw") + _ = os.WriteFile(target, []byte("old"), 0o755) + _, err := Install(t.Context(), srv.Client(), srv.URL, "v2.0.0", "linux", "amd64", target) + return err + } + body := []byte("new binary") + + good, _ := signedServer(t, priv, body, false) + if err := install(good); err != nil { + t.Fatalf("a release signed with the trusted key was refused: %v", err) + } + for name, srv := range map[string]*httptest.Server{ + "unsigned": func() *httptest.Server { s, _ := signedServer(t, "", body, false); return s }(), + "signed by another": func() *httptest.Server { s, _ := signedServer(t, otherPriv, body, false); return s }(), + "checksums rewritten": func() *httptest.Server { s, _ := signedServer(t, priv, body, true); return s }(), + } { + if err := install(srv); err == nil { + t.Errorf("%s: installed", name) + } + } + + // A build that trusts no key installs on checksums, as before. + t.Setenv("LLMGW_RELEASE_KEY", "") + if TrustedKey() != "" { + t.Skip("this build embeds a trusted key; the unsigned path cannot be tested here") + } + unsigned, _ := signedServer(t, "", body, false) + if err := install(unsigned); err != nil { + t.Errorf("without a trusted key an unsigned release was refused: %v", err) + } +} + +func TestAKeyThatIsNotOneIsSaid(t *testing.T) { + if _, err := Sign("not-a-key", []byte("x")); err == nil { + t.Error("signed with a malformed key") + } + if err := Verify("bm9wZQ==", []byte("x"), "c2ln"); err == nil || !strings.Contains(err.Error(), "public key") { + t.Errorf("a malformed trusted key: %v", err) + } +} diff --git a/internal/release/trusted.pub b/internal/release/trusted.pub new file mode 100644 index 0000000..e69de29