From d3fb217b31520b5adbc02b9519636bd120ff400c Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 10:19:38 +0200 Subject: [PATCH] Bound the key cache and ration lookups of unknown keys per client address Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 ++++ internal/auth/auth.go | 49 ++++++++++++++++++++++++++++++++++--- internal/auth/auth_test.go | 40 ++++++++++++++++++++++++++++++ internal/auth/buckets.go | 49 +++++++++++++++++++++++++++++++++++++ internal/auth/middleware.go | 15 +++++++++++- 5 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 internal/auth/buckets.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 01cdb7e4..4aef7f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,11 @@ APIs change between minors without a deprecation period. grant now schedules its own removal within the hour as it is written, and the password is dropped once the grant stands in for it. An address given to the system ssh can no longer be read as one of its options. +- **Made-up keys cannot fill memory or the database.** Every well-formed key + presented, real or not, was cached for good and cost a database read the + first time. The cache is now bounded, and keys not in it are looked up at + most 20 a second per client address (50 at once); beyond that the request + is answered 429 without a read. ### Everything that can be changed on a running cluster, from the console diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 6d25be00..01335512 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -96,6 +96,9 @@ var ( ErrInvalid = errors.New("invalid api key") ErrExpired = errors.New("api key expired") ErrRevoked = errors.New("api key revoked") + // ErrThrottled is a client presenting more unknown keys than it is + // allowed to have looked up. + ErrThrottled = errors.New("too many unrecognised api keys from this address") ) // Authenticator resolves keys and caches the result. @@ -111,6 +114,10 @@ type Authenticator struct { // and writing it per request would put a database write back on the path. touchMu sync.Mutex touch map[string]time.Time + + // lookups rations the database reads of keys not in the cache, per + // client address. + lookups *buckets } type entry struct { @@ -124,13 +131,22 @@ func New(s store.Store, cfg Config) *Authenticator { cfg.normalize() return &Authenticator{ store: s, cfg: cfg, now: time.Now, - cache: map[string]entry{}, - touch: map[string]time.Time{}, + cache: map[string]entry{}, + touch: map[string]time.Time{}, + lookups: newBuckets(20, 50), } } // Authenticate resolves a presented credential. func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Principal, error) { + return a.AuthenticateFrom(ctx, presented, "") +} + +// AuthenticateFrom resolves a credential presented by client, an address: +// a key not in the cache costs a database read, so those are rationed per +// client, and one that has made too many is told to slow down rather than +// read for. Empty client is not rationed. +func (a *Authenticator) AuthenticateFrom(ctx context.Context, presented, client string) (Principal, error) { if presented == "" { return Principal{}, ErrMissing } @@ -150,6 +166,9 @@ func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Pri return e.principal, e.err } + if client != "" && !a.lookups.allow(client, a.now()) { + return Principal{}, ErrThrottled + } p, err := a.resolve(ctx, hash) if err != nil && !isRejection(err) { // A database problem is not an authentication answer, so it must not be @@ -211,10 +230,34 @@ func (a *Authenticator) lookup(id string) (entry, bool) { return e, true } +// maxCached bounds the cache. Every well-formed key anyone presents is an +// entry, a rejected one included, so without a bound a stream of made-up keys +// grows it until the process runs out of memory. +const maxCached = 100_000 + func (a *Authenticator) store_(id string, e entry) { a.mu.Lock() + defer a.mu.Unlock() + if len(a.cache) >= maxCached { + now := a.now() + for k, v := range a.cache { + if now.After(v.expires) { + delete(a.cache, k) + } + } + // Still full of live entries: half of them go, which costs only a + // lookup each for the keys in use. + if len(a.cache) >= maxCached { + n := 0 + for k := range a.cache { + if n%2 == 0 { + delete(a.cache, k) + } + n++ + } + } + } a.cache[id] = e - a.mu.Unlock() } // Invalidate drops every cached decision. diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 65e3f6f4..f3351583 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -438,3 +438,43 @@ func TestAlterAlwaysChangesTheKey(t *testing.T) { t.Error("alter is a no-op when the key already ends in the replacement") } } + +func TestAFloodOfMadeUpKeysFromOneAddressIsSlowedBeforeTheDatabase(t *testing.T) { + s, secret, _ := fixture(t) + a := New(s, DefaultConfig()) + now := time.Now() + a.now = func() time.Time { return now } + ctx := context.Background() + + throttled := 0 + for i := 0; i < 200; i++ { + madeUp, _, _ := store.NewKey("x", "x", "proj1") + if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) { + throttled++ + } + } + if throttled < 100 { + t.Errorf("only %d of 200 made-up keys were held back", throttled) + } + // Another address, and the real key, are not caught by it. + if _, err := a.AuthenticateFrom(ctx, secret, "198.51.100.1"); err != nil { + t.Errorf("a real key from another address: %v", err) + } + // Time restores the allowance. + now = now.Add(5 * time.Second) + madeUp, _, _ := store.NewKey("y", "y", "proj1") + if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) { + t.Error("still held back after waiting") + } +} + +func TestTheCacheDoesNotGrowWithoutBound(t *testing.T) { + s, _, _ := fixture(t) + a := New(s, DefaultConfig()) + for i := 0; i < maxCached+10; i++ { + a.store_(string(rune(i))+"-"+time.Now().String(), entry{expires: time.Now().Add(time.Hour)}) + } + if n := len(a.cache); n > maxCached { + t.Errorf("%d entries", n) + } +} diff --git a/internal/auth/buckets.go b/internal/auth/buckets.go new file mode 100644 index 00000000..7604001b --- /dev/null +++ b/internal/auth/buckets.go @@ -0,0 +1,49 @@ +package auth + +import ( + "sync" + "time" +) + +// buckets are token buckets by key: rate a second, up to burst at once. Ones +// full and unused are dropped as more arrive, so a key is not kept for ever. +type buckets struct { + mu sync.Mutex + rate float64 + burst float64 + by map[string]*bucket +} + +type bucket struct { + tokens float64 + at time.Time +} + +func newBuckets(rate, burst float64) *buckets { + return &buckets{rate: rate, burst: burst, by: map[string]*bucket{}} +} + +// allow takes a token for key at now, reporting whether there was one. +func (b *buckets) allow(key string, now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.by) > 50_000 { + for k, v := range b.by { + if now.Sub(v.at).Seconds()*b.rate >= b.burst { + delete(b.by, k) + } + } + } + k, ok := b.by[key] + if !ok { + k = &bucket{tokens: b.burst, at: now} + b.by[key] = k + } + k.tokens = min(b.burst, k.tokens+now.Sub(k.at).Seconds()*b.rate) + k.at = now + if k.tokens < 1 { + return false + } + k.tokens-- + return true +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go index 3b748820..443b5623 100644 --- a/internal/auth/middleware.go +++ b/internal/auth/middleware.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "log/slog" + "net" "net/http" "strings" ) @@ -24,7 +25,7 @@ func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http. return } - p, err := a.Authenticate(r.Context(), BearerToken(r)) + p, err := a.AuthenticateFrom(r.Context(), BearerToken(r), clientAddr(r)) if err != nil { writeAuthError(w, err) // The key prefix, never the key: a rejected credential in a log @@ -66,6 +67,9 @@ func writeAuthError(w http.ResponseWriter, err error) { switch { case errors.Is(err, ErrMissing): msg = "no api key was provided" + case errors.Is(err, ErrThrottled): + w.Header().Set("Retry-After", "1") + status, kind, msg = http.StatusTooManyRequests, "rate_limit_exceeded", "too many unrecognised api keys; slow down" case errors.Is(err, ErrExpired): msg = "this api key has expired" case errors.Is(err, ErrRevoked): @@ -104,3 +108,12 @@ func WriteForbiddenModel(w http.ResponseWriter, model string) { "type": "invalid_request_error", "code": nil, }}) } + +// clientAddr is the address a request came from, without its port. +func clientAddr(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +}