Give every engine a key only the router holds

Engines listen on every address their node has, with no key. Checked on the
live cluster: from a neighbouring node, a request straight to an engine's
port was answered — past the gateway's authentication, quotas and
accounting, by anyone who could reach the node.

Each replica's engine is now started with a key and refuses requests without
it. The key is derived from the replica id with an HMAC over a secret kept
in the store, rather than stored per replica: the router has to send the
right key to replicas this process never started — adopted after a restart,
or placed by the other control plane — and a function of a shared secret
gives that without a table to keep in step. A key taken from one node opens
that replica only. The secret is made on first use and copied with the rest
of the store when moving to PostgreSQL; a new one would lock the router out
of every running engine at the moment of the move.

The key reaches llama.cpp and vLLM through their environment
(LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every
user on the node can read; SGLang only takes it as an argument. The agent's
readiness check, the router's proxy and its metrics scrape send it, the
client's own Authorization still never reaches the engine, and `ctl bench`
fetches the key of the replica it is pointed at from the admin API.

Engines started before this keep running without a key until their replica
is next started. With no store there is nowhere to keep the secret, and
engines start without one, with a warning.

Also formats two files that went in unformatted.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 00:38:29 +02:00
1 parent d167686cf1
commit 9853936e0b
34 files changed
+356 -29

No files matched your search

+15 -3
View File
@@ -2525,7 +2525,11 @@ type StartShard struct {
Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"`
// profile_fp identifies the version of the profile, so the agent can say
// which one a running shard was started from. The agent never interprets it.
ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"`
ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"`
// api_key is what the engine demands of every request, derived by the
// control plane from the replica id. The router sends it; nothing else
// that can reach the node's ports has it.
ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -2665,6 +2669,13 @@ func (x *StartShard) GetProfileFp() string {
return ""
}
func (x *StartShard) GetApiKey() string {
if x != nil {
return x.ApiKey
}
return ""
}
type EngineProfile struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
@@ -3693,7 +3704,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" +
"\vbytes_total\x18\x03 \x01(\x03R\n" +
"bytesTotal\"\xd9\x03\n" +
"bytesTotal\"\xf2\x03\n" +
"\n" +
"StartShard\x12\x1d\n" +
"\n" +
@@ -3716,7 +3727,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04pool\x18\r \x01(\tR\x04pool\x12\x16\n" +
"\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" +
"\n" +
"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xfd\x02\n" +
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" +
"\rEngineProfile\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
+4
View File
@@ -524,6 +524,10 @@ message StartShard {
// profile_fp identifies the version of the profile, so the agent can say
// which one a running shard was started from. The agent never interprets it.
string profile_fp = 15;
// api_key is what the engine demands of every request, derived by the
// control plane from the replica id. The router sends it; nothing else
// that can reach the node's ports has it.
string api_key = 16;
}
message EngineProfile {
+8
View File
@@ -71,6 +71,7 @@ rather than repeated.
| `lease_ttl` | `time.Duration` | [Control](#control) |
| `id` | `string` | [Endpoint](#endpoint) |
| `url` | `string` | [Endpoint](#endpoint) |
| `api_key` | `string` | [Endpoint](#endpoint) |
| `models` | `[]string` | [Endpoint](#endpoint) |
| `engine` | `string` | [Endpoint](#endpoint) |
| `pool` | `string` | [Endpoint](#endpoint) |
@@ -566,6 +567,13 @@ reading the alert. Zero takes the default.
`string`
### `api_key`
`string`
APIKey is sent to the engine as a bearer token, for an engine started
with one.
### `models`
`[]string`
+7
View File
@@ -70,6 +70,13 @@ type ShardSpec struct {
HTTPPort int
Profile Profile
// APIKey is what the engine demands of every request. Engines listen on
// all of a node's addresses; without a key anybody who can reach the port
// talks to the model directly, past the gateway's authentication, quotas
// and accounting. Given to the engine in its environment rather than on
// its command line, which every user on the node can read.
APIKey string `json:"api_key,omitempty"`
// Pool and Fabric are control plane concepts the agent cannot derive. They
// are carried so the agent can report them back after a restart; Fabric
// also says, with an "nvl:" prefix, that the replica is inside a
+5 -1
View File
@@ -120,8 +120,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
}
args = append(args, p.Args...)
env := baseEnv(s)
if s.APIKey != "" {
env["LLAMA_API_KEY"] = s.APIKey
}
return []Command{{
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
}}, nil
}
+16
View File
@@ -245,3 +245,19 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
t.Errorf("expected it passed through, got %q %v", got, err)
}
}
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
// The command line is readable by every user on the node; the
// environment of a root process is not.
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
t.Errorf("the key is on the command line: %v", cmds[0].Args)
}
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
t.Error("llama-server was not given its key")
}
}
+5
View File
@@ -96,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
)
}
args = append(args, p.Args...)
// SGLang reads its key only from the command line, so on a node with
// other users it is visible to them in the process list.
if s.APIKey != "" && s.Leader() {
args = append(args, "--api-key", s.APIKey)
}
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
}
+3
View File
@@ -150,6 +150,9 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
// Profile arguments last, so an operator can override anything above.
args = append(args, p.Args...)
if s.APIKey != "" {
env["VLLM_API_KEY"] = s.APIKey
}
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
}
+7 -4
View File
@@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth
}
// Ready reports whether the engine at baseURL is serving model.
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
return fmt.Errorf("health: %w", err)
}
if !h.requireName || model == "" {
@@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
ID string `json:"id"`
} `json:"data"`
}
if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
return fmt.Errorf("models: %w", err)
}
for _, d := range list.Data {
@@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
return fmt.Errorf("engine is up but does not serve %q yet", model)
}
func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
if apiKey != "" {
req.Header.Set("Authorization", "Bearer "+apiKey)
}
resp, err := h.client.Do(req)
if err != nil {
return err
+24 -5
View File
@@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
func TestReadyWhenServingModel(t *testing.T) {
srv := engineStub(t, true, "llama-3.3-70b")
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
t.Errorf("a serving engine was reported unready: %v", err)
}
}
@@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) {
// The case that matters: the process is up but the weights are not loaded.
// Routing on /health alone would send traffic into a loading model.
srv := engineStub(t, false, "llama-3.3-70b")
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
if err == nil {
t.Fatal("an unhealthy engine was reported ready")
}
@@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
// /health passes but the model is not listed: the engine is up and serving
// something else, or still registering.
srv := engineStub(t, true, "some-other-model")
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
if err == nil {
t.Fatal("a model that is not served was reported ready")
}
@@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
func TestModelCheckCanBeSkipped(t *testing.T) {
srv := engineStub(t, true)
h := NewHTTPHealth(time.Second, WithoutModelCheck())
if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
t.Errorf("the model check was not skipped: %v", err)
}
}
func TestUnreachableEngine(t *testing.T) {
h := NewHTTPHealth(200 * time.Millisecond)
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
t.Error("an unreachable engine was reported ready")
}
}
func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
// An engine started with a key refuses /v1/models without it, and a
// shard that is serving would read as never ready.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer k1" {
w.WriteHeader(http.StatusUnauthorized)
return
}
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
}))
defer srv.Close()
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
t.Errorf("with the key: %v", err)
}
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
t.Error("without the key the engine refused, and that read as ready")
}
}
+2 -2
View File
@@ -79,7 +79,7 @@ func (c *Config) normalize() {
// supervisor can be tested without an HTTP engine.
type HealthChecker interface {
// Ready reports whether the shard at baseURL serves model.
Ready(ctx context.Context, baseURL, model string) error
Ready(ctx context.Context, baseURL, model, apiKey string) error
}
// Shard is one supervised engine process.
@@ -411,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
last := strings.Join(sh.Logs(10), "\n")
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
case <-t.C:
if err := s.health.Ready(ctx, baseURL, model); err != nil {
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
continue
}
sh.setState(Ready, nil)
+1 -1
View File
@@ -21,7 +21,7 @@ import (
// scriptedHealth reports ready once the flag is set.
type scriptedHealth struct{ ready atomic.Bool }
func (h *scriptedHealth) Ready(context.Context, string, string) error {
func (h *scriptedHealth) Ready(context.Context, string, string, string) error {
if h.ready.Load() {
return nil
}
+1
View File
@@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i
)
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
sampler.apiKey = cfg.APIKey
sampler.start(ctx)
start := time.Now()
+4
View File
@@ -19,6 +19,7 @@ import (
// Optional, because a run pointed at the gateway has no single engine to read.
type sampler struct {
url string
apiKey string
engine string
client *http.Client
cancel context.CancelFunc
@@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) {
if err != nil {
return
}
if s.apiKey != "" {
req.Header.Set("Authorization", "Bearer "+s.apiKey)
}
resp, err := s.client.Do(req)
if err != nil {
return
+20
View File
@@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
cfg.MetricsURL = metrics
cfg.Engine = engine
cfg.APIKey = key
// A replica's engine demands its own key. Asked for rather than
// typed: the admin API knows which replica answers at the target.
if cfg.APIKey == "" {
cfg.APIKey = replicaKey(cmd, client, target)
}
if len(steps) > 0 {
cfg.Concurrency = steps
}
@@ -227,3 +232,18 @@ func ms(d time.Duration) string {
return d.Round(10 * time.Millisecond).String()
}
}
// replicaKey is the engine key of the replica at target, or "" when target is
// not a replica (the gateway, say) or the admin API cannot be asked.
func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
var reps []adminapi.ReplicaView
if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
return ""
}
for _, rep := range reps {
if sameEndpoint(rep.Endpoint, target) {
return rep.EngineKey
}
}
return ""
}
+14
View File
@@ -385,6 +385,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
}
// Each replica's engine demands a key only the router has. The secret it
// is derived from lives in the store, so every control plane against it —
// and this one after a restart — computes the same key for a replica.
if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
secret, err := h.Store().ClusterSecret(ctx, "engine-key")
if err != nil {
return nil, nil, fmt.Errorf("reading the engine key secret: %w", err)
}
ccfg.Reconciler.EngineKey = control.EngineKeys([]byte(secret))
} else {
log.Warn("engines start without a key: there is no store to keep the secret " +
"they are derived from, so anyone who can reach a node's ports reaches its models")
}
// The service needs the inventory and the status sink, both of which the
// controller owns; the controller needs the service as its dispatcher. The
// cycle is broken by giving the controller a dispatcher that forwards to
+5 -2
View File
@@ -404,8 +404,11 @@ type Scoring struct {
// Endpoint is one engine replica. Static in phase 1.
type Endpoint struct {
ID string `koanf:"id"`
URL string `koanf:"url"`
ID string `koanf:"id"`
URL string `koanf:"url"`
// APIKey is sent to the engine as a bearer token, for an engine started
// with one.
APIKey string `koanf:"api_key"`
Models []string `koanf:"models"`
// Engine names the inference engine, which decides what its metrics are
// called. Empty means vLLM's names.
+11 -6
View File
@@ -238,9 +238,13 @@ type ModelView struct {
// ReplicaView is a replica as `ctl replicas` shows it.
type ReplicaView struct {
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
ID string `json:"id"`
// EngineKey is the key this replica's engine demands, for `ctl bench`
// and anyone else holding the admin listener who needs to reach an engine
// directly. The client-facing pages never render it.
EngineKey string `json:"engine_key,omitempty"`
Model string `json:"model"`
Profile string `json:"profile"`
// Stale is true when the profile has been edited since this replica was
// placed, so it is running a definition the catalog no longer holds.
//
@@ -471,9 +475,10 @@ func (a *API) Replicas(model string) []ReplicaView {
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
Endpoint: rep.Endpoint, Err: rep.Err,
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
IdleSec: -1,
EngineKey: a.ctl.EngineKey(rep.ID),
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
IdleSec: -1,
}
if rep.LastUsedAt.IsZero() {
v.IdleSec = -1
+2
View File
@@ -191,6 +191,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
Pool: cmd.Pool,
Fabric: cmd.Fabric,
ProfileFp: cmd.ProfileFP,
ApiKey: cmd.Spec.APIKey,
}
}
@@ -211,6 +212,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
Pool: m.GetPool(),
Fabric: m.GetFabric(),
ProfileFP: m.GetProfileFp(),
APIKey: m.GetApiKey(),
Profile: engines.Profile{
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
TP: int(p.GetTp()), PP: int(p.GetPp()),
+30
View File
@@ -4,6 +4,9 @@ package control
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"log/slog"
"strings"
@@ -304,6 +307,7 @@ func (c *Controller) endpoints() []endpoint.Info {
ID: rep.ID,
URL: strings.TrimRight(rep.Endpoint, "/"),
Models: names,
APIKey: c.engineKey(rep.ID),
Engine: prof.Engine,
Pool: rep.Pool,
Weight: rep.Weight,
@@ -321,3 +325,29 @@ func fingerprint(infos []endpoint.Info) string {
}
return b.String()
}
// EngineKey is the key a replica's engine demands, empty when engines run
// without one.
func (c *Controller) EngineKey(replicaID string) string { return c.engineKey(replicaID) }
func (c *Controller) engineKey(replicaID string) string {
if c.cfg.Reconciler.EngineKey == nil {
return ""
}
return c.cfg.Reconciler.EngineKey(replicaID)
}
// EngineKeys derives each replica's engine key from one secret.
//
// HMAC rather than a stored random key per replica: the router needs the key
// for every replica it sends to, including ones adopted after a restart that
// this process never started, and a second control plane has to agree. A
// function of a shared secret and the replica id gives all three without a
// table to keep in step. A key taken from one node opens that replica only.
func EngineKeys(secret []byte) func(replicaID string) string {
return func(replicaID string) string {
m := hmac.New(sha256.New, secret)
m.Write([]byte("llmgw engine key\x00" + replicaID))
return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40]
}
}
+19
View File
@@ -0,0 +1,19 @@
package control
import "testing"
func TestEngineKeysAreTheSameForTheSameReplicaAndNoOther(t *testing.T) {
// A control plane that restarts, or a second one, must compute the key a
// running engine was started with; a key taken from one node must open
// that replica only.
a, b := EngineKeys([]byte("s")), EngineKeys([]byte("s"))
if a("r1") != b("r1") {
t.Error("the same secret gave two keys for one replica")
}
if a("r1") == a("r2") {
t.Error("two replicas share a key")
}
if a("r1") == EngineKeys([]byte("other"))("r1") {
t.Error("the key does not depend on the secret")
}
}
+13
View File
@@ -69,6 +69,11 @@ type Config struct {
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
MgmtInterface string
// EngineKey gives each replica the key its engine demands. Derived from
// the replica id rather than stored, so a control plane that restarts —
// or a second one — computes the same key for a replica it adopts. Nil
// starts engines without one.
EngineKey func(replicaID string) string
// EvictionGrace is how long a replica must have gone unasked-for before it
// may be stopped to make room for another model.
//
@@ -883,6 +888,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
MgmtInterface: r.cfg.MgmtInterface,
HTTPPort: s.HTTPPort,
Profile: espec,
APIKey: r.engineKey(id),
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
@@ -1380,3 +1386,10 @@ func (r *Reconciler) profileMemory(model, profile string) int {
}
return 0
}
func (r *Reconciler) engineKey(replicaID string) string {
if r.cfg.EngineKey == nil {
return ""
}
return r.cfg.EngineKey(replicaID)
}
@@ -1721,3 +1721,23 @@ func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
}
}
func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) {
// Engines listen on every address a node has. Started without a key, any
// machine that could reach the port reached the model past the gateway's
// authentication, quotas and accounting — checked from a neighbouring node
// on the live cluster.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id }
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) == 0 {
t.Fatal("nothing was started")
}
for _, c := range started {
if c.Spec.APIKey != "key-for-"+c.ReplicaID {
t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey)
}
}
}
-1
View File
@@ -84,4 +84,3 @@ func validSource(source string) error {
}
return checkScheme(u)
}
+3
View File
@@ -22,6 +22,9 @@ type Info struct {
ID string
// URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash.
URL string
// APIKey is what the engine demands of every request, sent as a bearer
// token. Empty for an engine that asks for none.
APIKey string
// Models lists the model names this replica serves.
Models []string
// Engine names the inference engine serving here, which decides what its
+6 -2
View File
@@ -148,6 +148,8 @@ type Request struct {
Body []byte
// Header carries headers to forward (authorization is deliberately not).
Header http.Header
// APIKey is the engine's own key, sent in place of the client's.
APIKey string
// Stream says whether an SSE response is expected.
Stream bool
// SuppressUsageChunk drops the final usage-only chunk from the client while
@@ -218,6 +220,9 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
}
req.Header = cloneForUpstream(r.Header)
req.Header.Set("Content-Type", "application/json")
if r.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+r.APIKey)
}
req.ContentLength = int64(len(r.Body))
// A non-streaming request waits for the whole completion before a single
@@ -518,8 +523,7 @@ func copyHeaders(dst, src http.Header) {
// cloneForUpstream forwards only what the engine needs. The client's
// Authorization header is deliberately dropped: it authenticates the caller to
// llmgw, not llmgw to the engine, and engines are reached over a trusted
// network (plan §14).
// llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do.
func cloneForUpstream(h http.Header) http.Header {
out := make(http.Header, 4)
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} {
+19
View File
@@ -355,3 +355,22 @@ func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) {
t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed)
}
}
func TestTheEngineGetsLlmgwsKeyNotTheClients(t *testing.T) {
var got string
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Get("Authorization")
io.WriteString(w, `{}`)
}))
defer up.Close()
client := http.Header{"Authorization": {"Bearer sk-the-callers-own"}}
_, err := testProxy().Do(t.Context(), httptest.NewRecorder(),
Request{URL: up.URL, Body: []byte(`{}`), Header: client, APIKey: "llmgw-engine"})
if err != nil {
t.Fatal(err)
}
if got != "Bearer llmgw-engine" {
t.Errorf("the engine was sent %q", got)
}
}
+2
View File
@@ -373,6 +373,7 @@ func toInfos(eps []config.Endpoint) []endpoint.Info {
out = append(out, endpoint.Info{
ID: e.ID,
URL: strings.TrimRight(e.URL, "/"),
APIKey: e.APIKey,
Models: e.Models,
Engine: e.Engine,
Pool: e.Pool,
@@ -673,6 +674,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
out, perr := r.px.Do(ctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
APIKey: dec.Endpoint.APIKey,
Body: req.UpstreamBody,
Header: req.Header,
Stream: req.Stream,
+6 -2
View File
@@ -157,7 +157,7 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
}
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
stats, err := p.fetch(ctx, in.URL, in.APIKey, p.mappingFor(in))
if err != nil {
if ctx.Err() != nil {
return
@@ -201,7 +201,7 @@ func (p *Poller) report(in endpoint.Info, err error, reported *string) {
*reported = "unreachable"
}
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
func (p *Poller) fetch(ctx context.Context, base, apiKey string, mapping Mapping) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()
@@ -209,6 +209,10 @@ func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpo
if err != nil {
return endpoint.Stats{}, err
}
// llama.cpp guards /metrics with its key; vLLM does not, and ignores one.
if apiKey != "" {
req.Header.Set("Authorization", "Bearer "+apiKey)
}
resp, err := p.http.Do(req)
if err != nil {
return endpoint.Stats{}, err
+4
View File
@@ -38,6 +38,10 @@ var copyOrder = []string{
"threads",
"thread_messages",
"memory_notes",
// Copied, not regenerated: every running engine was started with a key
// derived from this, and a new secret would lock the router out of all
// of them at the moment of the move.
"cluster_secrets",
}
// notCopied are tables deliberately left behind.
@@ -0,0 +1,12 @@
-- Secrets the control planes share and nobody types: generated on first use,
-- read by every control plane against this store afterwards.
--
-- The first is the one each replica's engine key is derived from. It has to be
-- the same for every control plane — the router sends a replica the key it was
-- started with, whichever control plane started it — and the same after a
-- restart, when running replicas are adopted rather than restarted.
CREATE TABLE cluster_secrets (
name TEXT PRIMARY KEY,
value TEXT NOT NULL,
created_at BIGINT NOT NULL
);
@@ -0,0 +1,12 @@
-- Secrets the control planes share and nobody types: generated on first use,
-- read by every control plane against this store afterwards.
--
-- The first is the one each replica's engine key is derived from. It has to be
-- the same for every control plane — the router sends a replica the key it was
-- started with, whichever control plane started it — and the same after a
-- restart, when running replicas are adopted rather than restarted.
CREATE TABLE cluster_secrets (
name TEXT PRIMARY KEY,
value TEXT NOT NULL,
created_at BIGINT NOT NULL
);
+31
View File
@@ -0,0 +1,31 @@
package store
import (
"context"
"crypto/rand"
"encoding/hex"
"time"
)
// ClusterSecret returns the named secret, creating a random one the first time
// it is asked for.
//
// Insert-if-absent and then read, so two control planes asking at once end up
// with the same value: the loser's insert changes nothing and it reads the
// winner's.
func (s *DB) ClusterSecret(ctx context.Context, name string) (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return "", err
}
if _, err := s.db.ExecContext(ctx,
`INSERT INTO cluster_secrets (name, value, created_at) VALUES (?, ?, ?)
ON CONFLICT (name) DO NOTHING`,
name, hex.EncodeToString(b), time.Now().UnixMilli()); err != nil {
return "", err
}
var v string
err := s.db.QueryRowContext(ctx,
`SELECT value FROM cluster_secrets WHERE name = ?`, name).Scan(&v)
return v, err
}
+25
View File
@@ -0,0 +1,25 @@
package store
import (
"path/filepath"
"testing"
)
func TestAClusterSecretIsMadeOnceAndKept(t *testing.T) {
db, err := Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
a, err := db.ClusterSecret(t.Context(), "engine-key")
if err != nil || len(a) != 64 {
t.Fatalf("%q %v", a, err)
}
b, err := db.ClusterSecret(t.Context(), "engine-key")
if err != nil || a != b {
t.Errorf("a second read made a new secret: %q then %q", a, b)
}
if c, _ := db.ClusterSecret(t.Context(), "other"); c == a {
t.Error("two names share a secret")
}
}