Secrets in Vault: env: and vault: references wherever a secret file is named, and the store's own secrets sealed under a master key — Vault transit or a local key

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:21:44 +02:00
1 parent bce54f0941
commit 35657c1417
20 files changed
+837 -14

No files matched your search

+136
View File
@@ -0,0 +1,136 @@
package secrets
import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"strings"
"golang.org/x/crypto/hkdf"
)
// Sealer encrypts the store's own secrets — what engine keys are derived
// from — under a key the store does not hold, so a copy of the store or of
// its backup does not hold them either.
type Sealer interface {
Seal(ctx context.Context, plain []byte) (string, error)
Open(ctx context.Context, sealed string) ([]byte, error)
}
// Sealed values say how they were sealed, so a plain value from before is
// told apart, and read.
const (
localPrefix = "llmgw:local:v1:"
transitPrefix = "llmgw:transit:v1:"
)
// IsSealed says whether a stored value is sealed.
func IsSealed(v string) bool {
return strings.HasPrefix(v, localPrefix) || strings.HasPrefix(v, transitPrefix)
}
// NewSealer is the sealer secrets.master_key names: vault-transit:KEY seals
// with Vault's transit engine, which never lets the key out; anything else
// is a reference to 32 bytes (hex, base64 or raw) used here.
func NewSealer(ctx context.Context, ref string) (Sealer, error) {
if key, ok := strings.CutPrefix(ref, "vault-transit:"); ok {
mount, name := "transit", key
if m, n, ok := strings.Cut(key, "/"); ok {
mount, name = m, n
}
return transit{mount: mount, key: name}, nil
}
raw, err := Read(ctx, ref)
if err != nil {
return nil, fmt.Errorf("secrets.master_key: %w", err)
}
key := decodeKey(raw)
if len(key) < 32 {
return nil, errors.New("secrets.master_key is shorter than 32 bytes: make one with `openssl rand -hex 32`")
}
k := make([]byte, 32)
if _, err := io.ReadFull(hkdf.New(sha256.New, key, nil, []byte("llmgw cluster secrets v1")), k); err != nil {
return nil, err
}
block, err := aes.NewCipher(k)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
return local{gcm: gcm}, nil
}
func decodeKey(b []byte) []byte {
s := strings.TrimSpace(string(b))
if k, err := hex.DecodeString(s); err == nil {
return k
}
if k, err := base64.StdEncoding.DecodeString(s); err == nil {
return k
}
return []byte(s)
}
type local struct{ gcm cipher.AEAD }
func (l local) Seal(_ context.Context, plain []byte) (string, error) {
nonce := make([]byte, l.gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", err
}
return localPrefix + base64.StdEncoding.EncodeToString(l.gcm.Seal(nonce, nonce, plain, nil)), nil
}
func (l local) Open(_ context.Context, sealed string) ([]byte, error) {
b, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(sealed, localPrefix))
if err != nil || !strings.HasPrefix(sealed, localPrefix) || len(b) < l.gcm.NonceSize() {
return nil, errors.New("not a value this key sealed")
}
out, err := l.gcm.Open(nil, b[:l.gcm.NonceSize()], b[l.gcm.NonceSize():], nil)
if err != nil {
return nil, errors.New("the master key does not open this value: it was sealed under another")
}
return out, nil
}
type transit struct{ mount, key string }
func (t transit) Seal(ctx context.Context, plain []byte) (string, error) {
var out struct {
Data struct {
Ciphertext string `json:"ciphertext"`
} `json:"data"`
}
if err := call(ctx, http.MethodPost, "/v1/"+t.mount+"/encrypt/"+t.key,
map[string]string{"plaintext": base64.StdEncoding.EncodeToString(plain)}, &out); err != nil {
return "", fmt.Errorf("sealing with Vault's transit key %s: %w", t.key, err)
}
return transitPrefix + out.Data.Ciphertext, nil
}
func (t transit) Open(ctx context.Context, sealed string) ([]byte, error) {
if !strings.HasPrefix(sealed, transitPrefix) {
return nil, errors.New("not a value Vault's transit engine sealed")
}
var out struct {
Data struct {
Plaintext string `json:"plaintext"`
} `json:"data"`
}
if err := call(ctx, http.MethodPost, "/v1/"+t.mount+"/decrypt/"+t.key,
map[string]string{"ciphertext": strings.TrimPrefix(sealed, transitPrefix)}, &out); err != nil {
return nil, fmt.Errorf("opening with Vault's transit key %s: %w", t.key, err)
}
return base64.StdEncoding.DecodeString(out.Data.Plaintext)
}
+278
View File
@@ -0,0 +1,278 @@
// Package secrets reads the secrets llmgw is configured with from where
// they are kept: a file, as always, an environment variable, or HashiCorp
// Vault (or OpenBao) — and seals the store's own secrets under a key that
// stays outside it.
//
// A reference is one of:
//
// /etc/llmgw/secrets/store-dsn a file, trimmed
// env:LLMGW_STORE_DSN an environment variable
// vault:secret/llmgw/store#dsn a field of a KV secret in Vault
//
// A Vault path's first segment is its KV v2 mount; secret/data/… is taken
// as written. KV v1 mounts work too.
package secrets
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"sync"
"time"
)
// Vault is how Vault is reached.
type Vault struct {
// Addr is Vault's address, like https://vault.internal:8200; empty
// takes VAULT_ADDR.
Addr string `koanf:"addr"`
// TokenFile holds a token; empty takes VAULT_TOKEN, or logs in with
// AppRole when RoleIDFile and SecretIDFile are set.
TokenFile string `koanf:"token_file"`
RoleIDFile string `koanf:"role_id_file"`
SecretIDFile string `koanf:"secret_id_file"`
// CAFile verifies Vault's certificate; empty uses the system's.
CAFile string `koanf:"ca_file"`
// Namespace is Vault Enterprise's or OpenBao's namespace.
Namespace string `koanf:"namespace"`
}
var (
mu sync.Mutex
vault Vault
client = &http.Client{Timeout: 10 * time.Second}
token string
expiry time.Time
cache = map[string]cached{}
)
type cached struct {
b []byte
at time.Time
}
// cacheFor is how long a value read from Vault is used before it is read
// again: a secret rotated there reaches llmgw within it.
const cacheFor = time.Minute
// Configure says how to reach Vault. Called once configuration is read.
func Configure(v Vault) error {
mu.Lock()
defer mu.Unlock()
vault, token, expiry, cache = v, "", time.Time{}, map[string]cached{}
if v.CAFile != "" {
pem, err := os.ReadFile(v.CAFile)
if err != nil {
return fmt.Errorf("secrets.vault.ca_file: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return fmt.Errorf("secrets.vault.ca_file %s holds no certificate", v.CAFile)
}
client = &http.Client{Timeout: 10 * time.Second,
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
}
return nil
}
// IsReference says whether s names a secret somewhere other than a file.
func IsReference(s string) bool {
return strings.HasPrefix(s, "env:") || strings.HasPrefix(s, "vault:")
}
// Read reads a secret by reference, trimmed of surrounding space.
func Read(ctx context.Context, ref string) ([]byte, error) {
switch {
case strings.HasPrefix(ref, "env:"):
name := strings.TrimPrefix(ref, "env:")
v, ok := os.LookupEnv(name)
if !ok {
return nil, fmt.Errorf("the environment has no %s", name)
}
return []byte(strings.TrimSpace(v)), nil
case strings.HasPrefix(ref, "vault:"):
return readVault(ctx, strings.TrimPrefix(ref, "vault:"))
}
b, err := os.ReadFile(ref) //nolint:gosec // a path the operator configured
if err != nil {
return nil, err
}
return bytes.TrimSpace(b), nil
}
// ReadString is Read as a string.
func ReadString(ctx context.Context, ref string) (string, error) {
b, err := Read(ctx, ref)
return string(b), err
}
func readVault(ctx context.Context, ref string) ([]byte, error) {
path, field, ok := strings.Cut(ref, "#")
if !ok || path == "" || field == "" {
return nil, fmt.Errorf("vault:%s: write it as vault:mount/path#field", ref)
}
mu.Lock()
if c, ok := cache[ref]; ok && time.Since(c.at) < cacheFor {
mu.Unlock()
return c.b, nil
}
mu.Unlock()
api := path
if mount, rest, ok := strings.Cut(path, "/"); ok && !strings.HasPrefix(rest, "data/") {
api = mount + "/data/" + rest
}
var out struct {
Data map[string]any `json:"data"`
}
if err := call(ctx, http.MethodGet, "/v1/"+api, nil, &out); err != nil {
// A KV v1 mount has no /data/ in its path.
if api != path && errors.Is(err, errNotFound) {
err = call(ctx, http.MethodGet, "/v1/"+path, nil, &out)
}
if err != nil {
return nil, fmt.Errorf("vault:%s: %w", ref, err)
}
}
data := out.Data
if inner, ok := data["data"].(map[string]any); ok {
data = inner // KV v2
}
v, ok := data[field]
if !ok {
return nil, fmt.Errorf("vault:%s: the secret has no field %q", ref, field)
}
s, ok := v.(string)
if !ok {
return nil, fmt.Errorf("vault:%s: field %q is not a string", ref, field)
}
b := []byte(strings.TrimSpace(s))
mu.Lock()
cache[ref] = cached{b: b, at: time.Now()}
mu.Unlock()
return b, nil
}
var errNotFound = errors.New("not found")
// call makes one request of Vault's API with the token, logging in first
// when there is none.
func call(ctx context.Context, method, path string, body, out any) error {
tok, addr, err := currentToken(ctx)
if err != nil {
return err
}
return do(ctx, addr, tok, method, path, body, out)
}
func do(ctx context.Context, addr, tok, method, path string, body, out any) error {
var rd io.Reader
if body != nil {
b, _ := json.Marshal(body)
rd = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(addr, "/")+path, rd)
if err != nil {
return err
}
if tok != "" {
req.Header.Set("X-Vault-Token", tok)
}
mu.Lock()
ns, c := vault.Namespace, client
mu.Unlock()
if ns != "" {
req.Header.Set("X-Vault-Namespace", ns)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
switch {
case resp.StatusCode == http.StatusNotFound:
return errNotFound
case resp.StatusCode >= 300:
var e struct {
Errors []string `json:"errors"`
}
_ = json.Unmarshal(b, &e)
return fmt.Errorf("vault answered %s: %s", resp.Status, strings.Join(e.Errors, "; "))
}
if out != nil {
return json.Unmarshal(b, out)
}
return nil
}
// currentToken is the token to send, logging in with AppRole when that is
// how it is configured and the last login has expired.
func currentToken(ctx context.Context) (string, string, error) {
mu.Lock()
v := vault
tok, exp := token, expiry
mu.Unlock()
addr := v.Addr
if addr == "" {
addr = os.Getenv("VAULT_ADDR")
}
if addr == "" {
return "", "", errors.New("no Vault to ask: set secrets.vault.addr or VAULT_ADDR")
}
if tok != "" && (exp.IsZero() || time.Now().Before(exp)) {
return tok, addr, nil
}
switch {
case v.TokenFile != "":
b, err := os.ReadFile(v.TokenFile)
if err != nil {
return "", "", fmt.Errorf("secrets.vault.token_file: %w", err)
}
tok, exp = strings.TrimSpace(string(b)), time.Time{}
case v.RoleIDFile != "" && v.SecretIDFile != "":
role, err := os.ReadFile(v.RoleIDFile)
if err != nil {
return "", "", err
}
sec, err := os.ReadFile(v.SecretIDFile)
if err != nil {
return "", "", err
}
var out struct {
Auth struct {
ClientToken string `json:"client_token"`
LeaseDuration int `json:"lease_duration"`
} `json:"auth"`
}
if err := do(ctx, addr, "", http.MethodPost, "/v1/auth/approle/login", map[string]string{
"role_id": strings.TrimSpace(string(role)), "secret_id": strings.TrimSpace(string(sec))}, &out); err != nil {
return "", "", fmt.Errorf("logging in to Vault with AppRole: %w", err)
}
tok = out.Auth.ClientToken
if d := out.Auth.LeaseDuration; d > 0 {
// Renewed by logging in again, a little before it runs out.
exp = time.Now().Add(time.Duration(d) * time.Second * 9 / 10)
}
default:
tok, exp = os.Getenv("VAULT_TOKEN"), time.Time{}
}
if tok == "" {
return "", "", errors.New("no Vault token: set secrets.vault.token_file, AppRole's files, or VAULT_TOKEN")
}
mu.Lock()
token, expiry = tok, exp
mu.Unlock()
return tok, addr, nil
}
+142
View File
@@ -0,0 +1,142 @@
package secrets
import (
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// fakeVault answers KV v2 reads, AppRole logins and transit, and counts
// what it was asked.
func fakeVault(t *testing.T) (*httptest.Server, *int) {
t.Helper()
reads := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/auth/approle/login" {
var in map[string]string
_ = json.NewDecoder(r.Body).Decode(&in)
if in["role_id"] != "role" || in["secret_id"] != "sid" {
http.Error(w, `{"errors":["denied"]}`, 400)
return
}
_, _ = w.Write([]byte(`{"auth":{"client_token":"approle-token","lease_duration":3600}}`))
return
}
if tok := r.Header.Get("X-Vault-Token"); tok != "root" && tok != "approle-token" {
http.Error(w, `{"errors":["permission denied"]}`, 403)
return
}
switch {
case r.URL.Path == "/v1/secret/data/llmgw/store":
reads++
_, _ = w.Write([]byte(`{"data":{"data":{"dsn":"postgres://u:p@h/db"}}}`))
case strings.HasPrefix(r.URL.Path, "/v1/transit/encrypt/llmgw"):
var in map[string]string
_ = json.NewDecoder(r.Body).Decode(&in)
_, _ = w.Write([]byte(`{"data":{"ciphertext":"vault:v1:` + in["plaintext"] + `"}}`))
case strings.HasPrefix(r.URL.Path, "/v1/transit/decrypt/llmgw"):
var in map[string]string
_ = json.NewDecoder(r.Body).Decode(&in)
_, _ = w.Write([]byte(`{"data":{"plaintext":"` + strings.TrimPrefix(in["ciphertext"], "vault:v1:") + `"}}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv, &reads
}
func TestReferencesAreRead(t *testing.T) {
ctx := t.Context()
f := filepath.Join(t.TempDir(), "s")
_ = os.WriteFile(f, []byte(" from-file\n"), 0o600)
if v, err := ReadString(ctx, f); err != nil || v != "from-file" {
t.Errorf("file: %q %v", v, err)
}
t.Setenv("LLMGW_TEST_SECRET", "from-env")
if v, err := ReadString(ctx, "env:LLMGW_TEST_SECRET"); err != nil || v != "from-env" {
t.Errorf("env: %q %v", v, err)
}
if _, err := Read(ctx, "env:LLMGW_NOT_SET_ANYWHERE"); err == nil {
t.Error("an unset variable read as empty")
}
srv, reads := fakeVault(t)
t.Setenv("VAULT_TOKEN", "root")
if err := Configure(Vault{Addr: srv.URL}); err != nil {
t.Fatal(err)
}
for range 3 {
if v, err := ReadString(ctx, "vault:secret/llmgw/store#dsn"); err != nil || v != "postgres://u:p@h/db" {
t.Fatalf("vault: %q %v", v, err)
}
}
if *reads != 1 {
t.Errorf("read from Vault %d times; cached for a minute", *reads)
}
if _, err := Read(ctx, "vault:secret/llmgw/store#nope"); err == nil {
t.Error("a missing field read")
}
if _, err := Read(ctx, "vault:secret/llmgw/store"); err == nil {
t.Error("a reference without a field read")
}
// AppRole, with no token given.
t.Setenv("VAULT_TOKEN", "")
dir := t.TempDir()
_ = os.WriteFile(filepath.Join(dir, "role"), []byte("role\n"), 0o600)
_ = os.WriteFile(filepath.Join(dir, "sid"), []byte("sid\n"), 0o600)
_ = Configure(Vault{Addr: srv.URL, RoleIDFile: filepath.Join(dir, "role"), SecretIDFile: filepath.Join(dir, "sid")})
if v, err := ReadString(ctx, "vault:secret/llmgw/store#dsn"); err != nil || v == "" {
t.Errorf("approle: %q %v", v, err)
}
}
func TestSealersSealAndOpen(t *testing.T) {
ctx := t.Context()
keyFile := filepath.Join(t.TempDir(), "master")
_ = os.WriteFile(keyFile, []byte(strings.Repeat("ab", 32)), 0o600)
l, err := NewSealer(ctx, keyFile)
if err != nil {
t.Fatal(err)
}
sealed, err := l.Seal(ctx, []byte("the engine key"))
if err != nil || !IsSealed(sealed) || strings.Contains(sealed, "engine key") {
t.Fatalf("%q %v", sealed, err)
}
if got, err := l.Open(ctx, sealed); err != nil || string(got) != "the engine key" {
t.Errorf("open: %q %v", got, err)
}
other := filepath.Join(t.TempDir(), "other")
_ = os.WriteFile(other, []byte(strings.Repeat("cd", 32)), 0o600)
l2, _ := NewSealer(ctx, other)
if _, err := l2.Open(ctx, sealed); err == nil {
t.Error("another key opened it")
}
short := filepath.Join(t.TempDir(), "short")
_ = os.WriteFile(short, []byte("abc"), 0o600)
if _, err := NewSealer(ctx, short); err == nil {
t.Error("a short key was taken")
}
srv, _ := fakeVault(t)
t.Setenv("VAULT_TOKEN", "root")
_ = Configure(Vault{Addr: srv.URL})
tr, err := NewSealer(ctx, "vault-transit:llmgw")
if err != nil {
t.Fatal(err)
}
sealed, err = tr.Seal(ctx, []byte("secret"))
if err != nil || !strings.HasPrefix(sealed, transitPrefix) ||
!strings.Contains(sealed, base64.StdEncoding.EncodeToString([]byte("secret"))) {
t.Fatalf("%q %v", sealed, err)
}
if got, err := tr.Open(ctx, sealed); err != nil || string(got) != "secret" {
t.Errorf("transit open: %q %v", got, err)
}
}