Bound the key cache and ration lookups of unknown keys per client address
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ebf437a36a
commit
d3fb217b31
5 files changed
+154
-4
No files matched your search
@@ -114,6 +114,11 @@ APIs change between minors without a deprecation period.
|
||||
grant now schedules its own removal within the hour as it is written, and
|
||||
the password is dropped once the grant stands in for it. An address given
|
||||
to the system ssh can no longer be read as one of its options.
|
||||
- **Made-up keys cannot fill memory or the database.** Every well-formed key
|
||||
presented, real or not, was cached for good and cost a database read the
|
||||
first time. The cache is now bounded, and keys not in it are looked up at
|
||||
most 20 a second per client address (50 at once); beyond that the request
|
||||
is answered 429 without a read.
|
||||
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
|
||||
+46
-3
@@ -96,6 +96,9 @@ var (
|
||||
ErrInvalid = errors.New("invalid api key")
|
||||
ErrExpired = errors.New("api key expired")
|
||||
ErrRevoked = errors.New("api key revoked")
|
||||
// ErrThrottled is a client presenting more unknown keys than it is
|
||||
// allowed to have looked up.
|
||||
ErrThrottled = errors.New("too many unrecognised api keys from this address")
|
||||
)
|
||||
|
||||
// Authenticator resolves keys and caches the result.
|
||||
@@ -111,6 +114,10 @@ type Authenticator struct {
|
||||
// and writing it per request would put a database write back on the path.
|
||||
touchMu sync.Mutex
|
||||
touch map[string]time.Time
|
||||
|
||||
// lookups rations the database reads of keys not in the cache, per
|
||||
// client address.
|
||||
lookups *buckets
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
@@ -124,13 +131,22 @@ func New(s store.Store, cfg Config) *Authenticator {
|
||||
cfg.normalize()
|
||||
return &Authenticator{
|
||||
store: s, cfg: cfg, now: time.Now,
|
||||
cache: map[string]entry{},
|
||||
touch: map[string]time.Time{},
|
||||
cache: map[string]entry{},
|
||||
touch: map[string]time.Time{},
|
||||
lookups: newBuckets(20, 50),
|
||||
}
|
||||
}
|
||||
|
||||
// Authenticate resolves a presented credential.
|
||||
func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Principal, error) {
|
||||
return a.AuthenticateFrom(ctx, presented, "")
|
||||
}
|
||||
|
||||
// AuthenticateFrom resolves a credential presented by client, an address:
|
||||
// a key not in the cache costs a database read, so those are rationed per
|
||||
// client, and one that has made too many is told to slow down rather than
|
||||
// read for. Empty client is not rationed.
|
||||
func (a *Authenticator) AuthenticateFrom(ctx context.Context, presented, client string) (Principal, error) {
|
||||
if presented == "" {
|
||||
return Principal{}, ErrMissing
|
||||
}
|
||||
@@ -150,6 +166,9 @@ func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Pri
|
||||
return e.principal, e.err
|
||||
}
|
||||
|
||||
if client != "" && !a.lookups.allow(client, a.now()) {
|
||||
return Principal{}, ErrThrottled
|
||||
}
|
||||
p, err := a.resolve(ctx, hash)
|
||||
if err != nil && !isRejection(err) {
|
||||
// A database problem is not an authentication answer, so it must not be
|
||||
@@ -211,10 +230,34 @@ func (a *Authenticator) lookup(id string) (entry, bool) {
|
||||
return e, true
|
||||
}
|
||||
|
||||
// maxCached bounds the cache. Every well-formed key anyone presents is an
|
||||
// entry, a rejected one included, so without a bound a stream of made-up keys
|
||||
// grows it until the process runs out of memory.
|
||||
const maxCached = 100_000
|
||||
|
||||
func (a *Authenticator) store_(id string, e entry) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if len(a.cache) >= maxCached {
|
||||
now := a.now()
|
||||
for k, v := range a.cache {
|
||||
if now.After(v.expires) {
|
||||
delete(a.cache, k)
|
||||
}
|
||||
}
|
||||
// Still full of live entries: half of them go, which costs only a
|
||||
// lookup each for the keys in use.
|
||||
if len(a.cache) >= maxCached {
|
||||
n := 0
|
||||
for k := range a.cache {
|
||||
if n%2 == 0 {
|
||||
delete(a.cache, k)
|
||||
}
|
||||
n++
|
||||
}
|
||||
}
|
||||
}
|
||||
a.cache[id] = e
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
// Invalidate drops every cached decision.
|
||||
|
||||
@@ -438,3 +438,43 @@ func TestAlterAlwaysChangesTheKey(t *testing.T) {
|
||||
t.Error("alter is a no-op when the key already ends in the replacement")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFloodOfMadeUpKeysFromOneAddressIsSlowedBeforeTheDatabase(t *testing.T) {
|
||||
s, secret, _ := fixture(t)
|
||||
a := New(s, DefaultConfig())
|
||||
now := time.Now()
|
||||
a.now = func() time.Time { return now }
|
||||
ctx := context.Background()
|
||||
|
||||
throttled := 0
|
||||
for i := 0; i < 200; i++ {
|
||||
madeUp, _, _ := store.NewKey("x", "x", "proj1")
|
||||
if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) {
|
||||
throttled++
|
||||
}
|
||||
}
|
||||
if throttled < 100 {
|
||||
t.Errorf("only %d of 200 made-up keys were held back", throttled)
|
||||
}
|
||||
// Another address, and the real key, are not caught by it.
|
||||
if _, err := a.AuthenticateFrom(ctx, secret, "198.51.100.1"); err != nil {
|
||||
t.Errorf("a real key from another address: %v", err)
|
||||
}
|
||||
// Time restores the allowance.
|
||||
now = now.Add(5 * time.Second)
|
||||
madeUp, _, _ := store.NewKey("y", "y", "proj1")
|
||||
if _, err := a.AuthenticateFrom(ctx, madeUp, "203.0.113.9"); errors.Is(err, ErrThrottled) {
|
||||
t.Error("still held back after waiting")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCacheDoesNotGrowWithoutBound(t *testing.T) {
|
||||
s, _, _ := fixture(t)
|
||||
a := New(s, DefaultConfig())
|
||||
for i := 0; i < maxCached+10; i++ {
|
||||
a.store_(string(rune(i))+"-"+time.Now().String(), entry{expires: time.Now().Add(time.Hour)})
|
||||
}
|
||||
if n := len(a.cache); n > maxCached {
|
||||
t.Errorf("%d entries", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// buckets are token buckets by key: rate a second, up to burst at once. Ones
|
||||
// full and unused are dropped as more arrive, so a key is not kept for ever.
|
||||
type buckets struct {
|
||||
mu sync.Mutex
|
||||
rate float64
|
||||
burst float64
|
||||
by map[string]*bucket
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
tokens float64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newBuckets(rate, burst float64) *buckets {
|
||||
return &buckets{rate: rate, burst: burst, by: map[string]*bucket{}}
|
||||
}
|
||||
|
||||
// allow takes a token for key at now, reporting whether there was one.
|
||||
func (b *buckets) allow(key string, now time.Time) bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if len(b.by) > 50_000 {
|
||||
for k, v := range b.by {
|
||||
if now.Sub(v.at).Seconds()*b.rate >= b.burst {
|
||||
delete(b.by, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
k, ok := b.by[key]
|
||||
if !ok {
|
||||
k = &bucket{tokens: b.burst, at: now}
|
||||
b.by[key] = k
|
||||
}
|
||||
k.tokens = min(b.burst, k.tokens+now.Sub(k.at).Seconds()*b.rate)
|
||||
k.at = now
|
||||
if k.tokens < 1 {
|
||||
return false
|
||||
}
|
||||
k.tokens--
|
||||
return true
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
@@ -24,7 +25,7 @@ func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http.
|
||||
return
|
||||
}
|
||||
|
||||
p, err := a.Authenticate(r.Context(), BearerToken(r))
|
||||
p, err := a.AuthenticateFrom(r.Context(), BearerToken(r), clientAddr(r))
|
||||
if err != nil {
|
||||
writeAuthError(w, err)
|
||||
// The key prefix, never the key: a rejected credential in a log
|
||||
@@ -66,6 +67,9 @@ func writeAuthError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, ErrMissing):
|
||||
msg = "no api key was provided"
|
||||
case errors.Is(err, ErrThrottled):
|
||||
w.Header().Set("Retry-After", "1")
|
||||
status, kind, msg = http.StatusTooManyRequests, "rate_limit_exceeded", "too many unrecognised api keys; slow down"
|
||||
case errors.Is(err, ErrExpired):
|
||||
msg = "this api key has expired"
|
||||
case errors.Is(err, ErrRevoked):
|
||||
@@ -104,3 +108,12 @@ func WriteForbiddenModel(w http.ResponseWriter, model string) {
|
||||
"type": "invalid_request_error", "code": nil,
|
||||
}})
|
||||
}
|
||||
|
||||
// clientAddr is the address a request came from, without its port.
|
||||
func clientAddr(r *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
Reference in new issue
Block a user