Give every engine a key only the router holds
Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d167686cf1
commit
9853936e0b
34 files changed
+356
-29
No files matched your search
@@ -238,9 +238,13 @@ type ModelView struct {
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
type ReplicaView struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
ID string `json:"id"`
|
||||
// EngineKey is the key this replica's engine demands, for `ctl bench`
|
||||
// and anyone else holding the admin listener who needs to reach an engine
|
||||
// directly. The client-facing pages never render it.
|
||||
EngineKey string `json:"engine_key,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
// Stale is true when the profile has been edited since this replica was
|
||||
// placed, so it is running a definition the catalog no longer holds.
|
||||
//
|
||||
@@ -471,9 +475,10 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
|
||||
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
|
||||
Endpoint: rep.Endpoint, Err: rep.Err,
|
||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||
IdleSec: -1,
|
||||
EngineKey: a.ctl.EngineKey(rep.ID),
|
||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||
IdleSec: -1,
|
||||
}
|
||||
if rep.LastUsedAt.IsZero() {
|
||||
v.IdleSec = -1
|
||||
|
||||
@@ -191,6 +191,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
Pool: cmd.Pool,
|
||||
Fabric: cmd.Fabric,
|
||||
ProfileFp: cmd.ProfileFP,
|
||||
ApiKey: cmd.Spec.APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,6 +212,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
Pool: m.GetPool(),
|
||||
Fabric: m.GetFabric(),
|
||||
ProfileFP: m.GetProfileFp(),
|
||||
APIKey: m.GetApiKey(),
|
||||
Profile: engines.Profile{
|
||||
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
||||
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
||||
|
||||
@@ -4,6 +4,9 @@ package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
@@ -304,6 +307,7 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
ID: rep.ID,
|
||||
URL: strings.TrimRight(rep.Endpoint, "/"),
|
||||
Models: names,
|
||||
APIKey: c.engineKey(rep.ID),
|
||||
Engine: prof.Engine,
|
||||
Pool: rep.Pool,
|
||||
Weight: rep.Weight,
|
||||
@@ -321,3 +325,29 @@ func fingerprint(infos []endpoint.Info) string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// EngineKey is the key a replica's engine demands, empty when engines run
|
||||
// without one.
|
||||
func (c *Controller) EngineKey(replicaID string) string { return c.engineKey(replicaID) }
|
||||
|
||||
func (c *Controller) engineKey(replicaID string) string {
|
||||
if c.cfg.Reconciler.EngineKey == nil {
|
||||
return ""
|
||||
}
|
||||
return c.cfg.Reconciler.EngineKey(replicaID)
|
||||
}
|
||||
|
||||
// EngineKeys derives each replica's engine key from one secret.
|
||||
//
|
||||
// HMAC rather than a stored random key per replica: the router needs the key
|
||||
// for every replica it sends to, including ones adopted after a restart that
|
||||
// this process never started, and a second control plane has to agree. A
|
||||
// function of a shared secret and the replica id gives all three without a
|
||||
// table to keep in step. A key taken from one node opens that replica only.
|
||||
func EngineKeys(secret []byte) func(replicaID string) string {
|
||||
return func(replicaID string) string {
|
||||
m := hmac.New(sha256.New, secret)
|
||||
m.Write([]byte("llmgw engine key\x00" + replicaID))
|
||||
return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package control
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEngineKeysAreTheSameForTheSameReplicaAndNoOther(t *testing.T) {
|
||||
// A control plane that restarts, or a second one, must compute the key a
|
||||
// running engine was started with; a key taken from one node must open
|
||||
// that replica only.
|
||||
a, b := EngineKeys([]byte("s")), EngineKeys([]byte("s"))
|
||||
if a("r1") != b("r1") {
|
||||
t.Error("the same secret gave two keys for one replica")
|
||||
}
|
||||
if a("r1") == a("r2") {
|
||||
t.Error("two replicas share a key")
|
||||
}
|
||||
if a("r1") == EngineKeys([]byte("other"))("r1") {
|
||||
t.Error("the key does not depend on the secret")
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,11 @@ type Config struct {
|
||||
PortRange [2]int
|
||||
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
||||
MgmtInterface string
|
||||
// EngineKey gives each replica the key its engine demands. Derived from
|
||||
// the replica id rather than stored, so a control plane that restarts —
|
||||
// or a second one — computes the same key for a replica it adopts. Nil
|
||||
// starts engines without one.
|
||||
EngineKey func(replicaID string) string
|
||||
// EvictionGrace is how long a replica must have gone unasked-for before it
|
||||
// may be stopped to make room for another model.
|
||||
//
|
||||
@@ -883,6 +888,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
MgmtInterface: r.cfg.MgmtInterface,
|
||||
HTTPPort: s.HTTPPort,
|
||||
Profile: espec,
|
||||
APIKey: r.engineKey(id),
|
||||
},
|
||||
StartupTimeout: prof.StartupTimeout,
|
||||
Pool: pl.Pool,
|
||||
@@ -1380,3 +1386,10 @@ func (r *Reconciler) profileMemory(model, profile string) int {
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *Reconciler) engineKey(replicaID string) string {
|
||||
if r.cfg.EngineKey == nil {
|
||||
return ""
|
||||
}
|
||||
return r.cfg.EngineKey(replicaID)
|
||||
}
|
||||
@@ -1721,3 +1721,23 @@ func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
|
||||
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) {
|
||||
// Engines listen on every address a node has. Started without a key, any
|
||||
// machine that could reach the port reached the model past the gateway's
|
||||
// authentication, quotas and accounting — checked from a neighbouring node
|
||||
// on the live cluster.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id }
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
started := h.disp.startedShards()
|
||||
if len(started) == 0 {
|
||||
t.Fatal("nothing was started")
|
||||
}
|
||||
for _, c := range started {
|
||||
if c.Spec.APIKey != "key-for-"+c.ReplicaID {
|
||||
t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user