Bound the key cache and ration lookups of unknown keys per client address

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:19:38 +02:00
1 parent ebf437a36a
commit d3fb217b31
5 files changed
+154 -4

No files matched your search

+5
View File
@@ -114,6 +114,11 @@ APIs change between minors without a deprecation period.
grant now schedules its own removal within the hour as it is written, and
the password is dropped once the grant stands in for it. An address given
to the system ssh can no longer be read as one of its options.
- **Made-up keys cannot fill memory or the database.** Every well-formed key
presented, real or not, was cached for good and cost a database read the
first time. The cache is now bounded, and keys not in it are looked up at
most 20 a second per client address (50 at once); beyond that the request
is answered 429 without a read.
### Everything that can be changed on a running cluster, from the console
+46 -3
View File
@@ -96,6 +96,9 @@ var (
ErrInvalid = errors.New("invalid api key")
ErrExpired = errors.New("api key expired")
ErrRevoked = errors.New("api key revoked")
// ErrThrottled is a client presenting more unknown keys than it is
// allowed to have looked up.
ErrThrottled = errors.New("too many unrecognised api keys from this address")
)
// Authenticator resolves keys and caches the result.
@@ -111,6 +114,10 @@ type Authenticator struct {
// and writing it per request would put a database write back on the path.
touchMu sync.Mutex
touch map[string]time.Time
// lookups rations the database reads of keys not in the cache, per
// client address.
lookups *buckets
}
type entry struct {
@@ -124,13 +131,22 @@ func New(s store.Store, cfg Config) *Authenticator {
cfg.normalize()
return &Authenticator{
store: s, cfg: cfg, now: time.Now,
cache: map[string]entry{},
touch: map[string]time.Time{},
cache: map[string]entry{},
touch: map[string]time.Time{},
lookups: newBuckets(20, 50),
}
}
// Authenticate resolves a presented credential.
func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Principal, error) {
return a.AuthenticateFrom(ctx, presented, "")
}
// AuthenticateFrom resolves a credential presented by client, an address:
// a key not in the cache costs a database read, so those are rationed per
// client, and one that has made too many is told to slow down rather than
// read for. Empty client is not rationed.
func (a *Authenticator) AuthenticateFrom(ctx context.Context, presented, client string) (Principal, error) {
if presented == "" {
return Principal{}, ErrMissing
}
@@ -150,6 +166,9 @@ func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Pri
return e.principal, e.err
}
if client != "" && !a.lookups.allow(client, a.now()) {
return Principal{}, ErrThrottled
}
p, err := a.resolve(ctx, hash)
if err != nil && !isRejection(err) {
// A database problem is not an authentication answer, so it must not be
@@ -211,10 +230,34 @@ func (a *Authenticator) lookup(id string) (entry, bool) {
return e, true
}
// maxCached bounds the cache. Every well-formed key anyone presents is an
// entry, a rejected one included, so without a bound a stream of made-up keys
// grows it until the process runs out of memory.
const maxCached = 100_000
func (a *Authenticator) store_(id string, e entry) {
a.mu.Lock()
defer a.mu.Unlock()
if len(a.cache) >= maxCached {
now := a.now()
for k, v := range a.cache {
if now.After(v.expires) {
delete(a.cache, k)
}
}
// Still full of live entries: half of them go, which costs only a
// lookup each for the keys in use.
if len(a.cache) >= maxCached {
n := 0
for k := range a.cache {
if n%2 == 0 {
delete(a.cache, k)
}
n++
}
}
}
a.cache[id] = e
a.mu.Unlock()
}
// Invalidate drops every cached decision.
+40
View File
@@ -438,3 +438,43 @@ func TestAlterAlwaysChangesTheKey(t *testing.T) {
t.Error("alter is a no-op when the key already ends in the replacement")
}
}
func TestAFloodOfMadeUpKeysFromOneAddressIsSlowedBeforeTheDatabase(t *testing.T) {
s, secret, _ := fixture(t)
a := New(s, DefaultConfig())
now := time.Now()
a.now = func() time.Time { return now }
ctx := context.Background()
throttled := 0
for i := 0; i < 200; i++ {
madeUp, _, _ := store.NewKey("x", "x", "proj1")
if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) {
throttled++
}
}
if throttled < 100 {
t.Errorf("only %d of 200 made-up keys were held back", throttled)
}
// Another address, and the real key, are not caught by it.
if _, err := a.AuthenticateFrom(ctx, secret, "198.51.100.1"); err != nil {
t.Errorf("a real key from another address: %v", err)
}
// Time restores the allowance.
now = now.Add(5 * time.Second)
madeUp, _, _ := store.NewKey("y", "y", "proj1")
if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) {
t.Error("still held back after waiting")
}
}
func TestTheCacheDoesNotGrowWithoutBound(t *testing.T) {
s, _, _ := fixture(t)
a := New(s, DefaultConfig())
for i := 0; i < maxCached+10; i++ {
a.store_(string(rune(i))+"-"+time.Now().String(), entry{expires: time.Now().Add(time.Hour)})
}
if n := len(a.cache); n > maxCached {
t.Errorf("%d entries", n)
}
}
+49
View File
@@ -0,0 +1,49 @@
package auth
import (
"sync"
"time"
)
// buckets are token buckets by key: rate a second, up to burst at once. Ones
// full and unused are dropped as more arrive, so a key is not kept for ever.
type buckets struct {
mu sync.Mutex
rate float64
burst float64
by map[string]*bucket
}
type bucket struct {
tokens float64
at time.Time
}
func newBuckets(rate, burst float64) *buckets {
return &buckets{rate: rate, burst: burst, by: map[string]*bucket{}}
}
// allow takes a token for key at now, reporting whether there was one.
func (b *buckets) allow(key string, now time.Time) bool {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.by) > 50_000 {
for k, v := range b.by {
if now.Sub(v.at).Seconds()*b.rate >= b.burst {
delete(b.by, k)
}
}
}
k, ok := b.by[key]
if !ok {
k = &bucket{tokens: b.burst, at: now}
b.by[key] = k
}
k.tokens = min(b.burst, k.tokens+now.Sub(k.at).Seconds()*b.rate)
k.at = now
if k.tokens < 1 {
return false
}
k.tokens--
return true
}
+14 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"log/slog"
"net"
"net/http"
"strings"
)
@@ -24,7 +25,7 @@ func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http.
return
}
p, err := a.Authenticate(r.Context(), BearerToken(r))
p, err := a.AuthenticateFrom(r.Context(), BearerToken(r), clientAddr(r))
if err != nil {
writeAuthError(w, err)
// The key prefix, never the key: a rejected credential in a log
@@ -66,6 +67,9 @@ func writeAuthError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, ErrMissing):
msg = "no api key was provided"
case errors.Is(err, ErrThrottled):
w.Header().Set("Retry-After", "1")
status, kind, msg = http.StatusTooManyRequests, "rate_limit_exceeded", "too many unrecognised api keys; slow down"
case errors.Is(err, ErrExpired):
msg = "this api key has expired"
case errors.Is(err, ErrRevoked):
@@ -104,3 +108,12 @@ func WriteForbiddenModel(w http.ResponseWriter, model string) {
"type": "invalid_request_error", "code": nil,
}})
}
// clientAddr is the address a request came from, without its port.
func clientAddr(r *http.Request) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return r.RemoteAddr
}
return host
}