From 9853936e0b66b16c5df3df2caf41d70415529103 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 00:38:29 +0200 Subject: [PATCH] Give every engine a key only the router holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 18 +++++++++-- api/proto/llmgw/v1/agent.proto | 4 +++ docs/reference.md | 8 +++++ internal/agent/engines/engine.go | 7 +++++ internal/agent/engines/llamacpp.go | 6 +++- internal/agent/engines/llamacpp_test.go | 16 ++++++++++ internal/agent/engines/sglang.go | 5 +++ internal/agent/engines/vllm.go | 3 ++ internal/agent/supervisor/health.go | 11 ++++--- internal/agent/supervisor/health_test.go | 29 ++++++++++++++--- internal/agent/supervisor/supervisor.go | 4 +-- internal/agent/supervisor/supervisor_test.go | 2 +- internal/bench/bench.go | 1 + internal/bench/sampler.go | 4 +++ internal/cli/bench.go | 20 ++++++++++++ internal/cli/server.go | 14 +++++++++ internal/config/config.go | 7 +++-- internal/control/adminapi/api.go | 17 ++++++---- internal/control/agentsvc/convert.go | 2 ++ internal/control/control.go | 30 ++++++++++++++++++ internal/control/control_test.go | 19 ++++++++++++ internal/control/reconciler/reconciler.go | 13 ++++++++ .../control/reconciler/reconciler_test.go | 20 ++++++++++++ internal/mirror/safefetch.go | 1 - internal/router/endpoint/endpoint.go | 3 ++ internal/router/proxy/proxy.go | 8 +++-- internal/router/proxy/proxy_test.go | 19 ++++++++++++ internal/router/router.go | 2 ++ internal/router/scrape/poller.go | 8 +++-- internal/store/copy.go | 4 +++ .../postgres/0011_cluster_secrets.sql | 12 +++++++ .../sqlite/0011_cluster_secrets.sql | 12 +++++++ internal/store/secrets.go | 31 +++++++++++++++++++ internal/store/secrets_test.go | 25 +++++++++++++++ 34 files changed, 356 insertions(+), 29 deletions(-) create mode 100644 internal/control/control_test.go create mode 100644 internal/store/migrations/postgres/0011_cluster_secrets.sql create mode 100644 internal/store/migrations/sqlite/0011_cluster_secrets.sql create mode 100644 internal/store/secrets.go create mode 100644 internal/store/secrets_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 51719410..57a6fff1 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2525,7 +2525,11 @@ type StartShard struct { Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"` // profile_fp identifies the version of the profile, so the agent can say // which one a running shard was started from. The agent never interprets it. - ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` + ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` + // api_key is what the engine demands of every request, derived by the + // control plane from the replica id. The router sends it; nothing else + // that can reach the node's ports has it. + ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2665,6 +2669,13 @@ func (x *StartShard) GetProfileFp() string { return "" } +func (x *StartShard) GetApiKey() string { + if x != nil { + return x.ApiKey + } + return "" +} + type EngineProfile struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -3693,7 +3704,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + "\vbytes_total\x18\x03 \x01(\x03R\n" + - "bytesTotal\"\xd9\x03\n" + + "bytesTotal\"\xf2\x03\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -3716,7 +3727,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04pool\x18\r \x01(\tR\x04pool\x12\x16\n" + "\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" + "\n" + - "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xfd\x02\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" + + "\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 9894a2a5..36d620ba 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -524,6 +524,10 @@ message StartShard { // profile_fp identifies the version of the profile, so the agent can say // which one a running shard was started from. The agent never interprets it. string profile_fp = 15; + // api_key is what the engine demands of every request, derived by the + // control plane from the replica id. The router sends it; nothing else + // that can reach the node's ports has it. + string api_key = 16; } message EngineProfile { diff --git a/docs/reference.md b/docs/reference.md index 910ef09e..d3ac0c13 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -71,6 +71,7 @@ rather than repeated. | `lease_ttl` | `time.Duration` | [Control](#control) | | `id` | `string` | [Endpoint](#endpoint) | | `url` | `string` | [Endpoint](#endpoint) | +| `api_key` | `string` | [Endpoint](#endpoint) | | `models` | `[]string` | [Endpoint](#endpoint) | | `engine` | `string` | [Endpoint](#endpoint) | | `pool` | `string` | [Endpoint](#endpoint) | @@ -566,6 +567,13 @@ reading the alert. Zero takes the default. `string` +### `api_key` + +`string` + +APIKey is sent to the engine as a bearer token, for an engine started +with one. + ### `models` `[]string` diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 6bc5f712..4f7862aa 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -70,6 +70,13 @@ type ShardSpec struct { HTTPPort int Profile Profile + // APIKey is what the engine demands of every request. Engines listen on + // all of a node's addresses; without a key anybody who can reach the port + // talks to the model directly, past the gateway's authentication, quotas + // and accounting. Given to the engine in its environment rather than on + // its command line, which every user on the node can read. + APIKey string `json:"api_key,omitempty"` + // Pool and Fabric are control plane concepts the agent cannot derive. They // are carried so the agent can report them back after a restart; Fabric // also says, with an "nvl:" prefix, that the replica is inside a diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 26f7676c..9424cb66 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -120,8 +120,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } args = append(args, p.Args...) + env := baseEnv(s) + if s.APIKey != "" { + env["LLAMA_API_KEY"] = s.APIKey + } return []Command{{ - Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), + Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env), }}, nil } diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 7e93354a..e97c9337 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -245,3 +245,19 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) { t.Errorf("expected it passed through, got %q %v", got, err) } } + +func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) { + // The command line is readable by every user on the node; the + // environment of a root process is not. + cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1", + Profile: Profile{ModelPath: "/m.gguf"}}) + if err != nil { + t.Fatal(err) + } + if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") { + t.Errorf("the key is on the command line: %v", cmds[0].Args) + } + if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") { + t.Error("llama-server was not given its key") + } +} diff --git a/internal/agent/engines/sglang.go b/internal/agent/engines/sglang.go index db0034dc..501c33db 100644 --- a/internal/agent/engines/sglang.go +++ b/internal/agent/engines/sglang.go @@ -96,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) { ) } args = append(args, p.Args...) + // SGLang reads its key only from the command line, so on a node with + // other users it is visible to them in the process list. + if s.APIKey != "" && s.Leader() { + args = append(args, "--api-key", s.APIKey) + } return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil } diff --git a/internal/agent/engines/vllm.go b/internal/agent/engines/vllm.go index beaf5069..81a63d0c 100644 --- a/internal/agent/engines/vllm.go +++ b/internal/agent/engines/vllm.go @@ -150,6 +150,9 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { // Profile arguments last, so an operator can override anything above. args = append(args, p.Args...) + if s.APIKey != "" { + env["VLLM_API_KEY"] = s.APIKey + } return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil } diff --git a/internal/agent/supervisor/health.go b/internal/agent/supervisor/health.go index 96985d52..46f62888 100644 --- a/internal/agent/supervisor/health.go +++ b/internal/agent/supervisor/health.go @@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth } // Ready reports whether the engine at baseURL is serving model. -func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error { - if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil { +func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error { + if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil { return fmt.Errorf("health: %w", err) } if !h.requireName || model == "" { @@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error { ID string `json:"id"` } `json:"data"` } - if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil { + if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil { return fmt.Errorf("models: %w", err) } for _, d := range list.Data { @@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error { return fmt.Errorf("engine is up but does not serve %q yet", model) } -func (h *HTTPHealth) get(ctx context.Context, url string, out any) error { +func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return err } + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } resp, err := h.client.Do(req) if err != nil { return err diff --git a/internal/agent/supervisor/health_test.go b/internal/agent/supervisor/health_test.go index 4a9b9d7d..7186fe16 100644 --- a/internal/agent/supervisor/health_test.go +++ b/internal/agent/supervisor/health_test.go @@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server { func TestReadyWhenServingModel(t *testing.T) { srv := engineStub(t, true, "llama-3.3-70b") - if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil { + if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil { t.Errorf("a serving engine was reported unready: %v", err) } } @@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) { // The case that matters: the process is up but the weights are not loaded. // Routing on /health alone would send traffic into a loading model. srv := engineStub(t, false, "llama-3.3-70b") - err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b") + err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "") if err == nil { t.Fatal("an unhealthy engine was reported ready") } @@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) { // /health passes but the model is not listed: the engine is up and serving // something else, or still registering. srv := engineStub(t, true, "some-other-model") - err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b") + err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "") if err == nil { t.Fatal("a model that is not served was reported ready") } @@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) { func TestModelCheckCanBeSkipped(t *testing.T) { srv := engineStub(t, true) h := NewHTTPHealth(time.Second, WithoutModelCheck()) - if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil { + if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil { t.Errorf("the model check was not skipped: %v", err) } } func TestUnreachableEngine(t *testing.T) { h := NewHTTPHealth(200 * time.Millisecond) - if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil { + if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil { t.Error("an unreachable engine was reported ready") } } + +func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) { + // An engine started with a key refuses /v1/models without it, and a + // shard that is serving would read as never ready. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer k1" { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`)) + })) + defer srv.Close() + if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil { + t.Errorf("with the key: %v", err) + } + if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil { + t.Error("without the key the engine refused, and that read as ready") + } +} diff --git a/internal/agent/supervisor/supervisor.go b/internal/agent/supervisor/supervisor.go index b010e598..6a6eb278 100644 --- a/internal/agent/supervisor/supervisor.go +++ b/internal/agent/supervisor/supervisor.go @@ -79,7 +79,7 @@ func (c *Config) normalize() { // supervisor can be tested without an HTTP engine. type HealthChecker interface { // Ready reports whether the shard at baseURL serves model. - Ready(ctx context.Context, baseURL, model string) error + Ready(ctx context.Context, baseURL, model, apiKey string) error } // Shard is one supervised engine process. @@ -411,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st last := strings.Join(sh.Logs(10), "\n") return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last) case <-t.C: - if err := s.health.Ready(ctx, baseURL, model); err != nil { + if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil { continue } sh.setState(Ready, nil) diff --git a/internal/agent/supervisor/supervisor_test.go b/internal/agent/supervisor/supervisor_test.go index b6ce7565..1fb998d6 100644 --- a/internal/agent/supervisor/supervisor_test.go +++ b/internal/agent/supervisor/supervisor_test.go @@ -21,7 +21,7 @@ import ( // scriptedHealth reports ready once the flag is set. type scriptedHealth struct{ ready atomic.Bool } -func (h *scriptedHealth) Ready(context.Context, string, string) error { +func (h *scriptedHealth) Ready(context.Context, string, string, string) error { if h.ready.Load() { return nil } diff --git a/internal/bench/bench.go b/internal/bench/bench.go index 10d9962e..2da271e5 100644 --- a/internal/bench/bench.go +++ b/internal/bench/bench.go @@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i ) sampler := newSampler(cfg.MetricsURL, cfg.Engine) + sampler.apiKey = cfg.APIKey sampler.start(ctx) start := time.Now() diff --git a/internal/bench/sampler.go b/internal/bench/sampler.go index 1cde4db2..db2fcf31 100644 --- a/internal/bench/sampler.go +++ b/internal/bench/sampler.go @@ -19,6 +19,7 @@ import ( // Optional, because a run pointed at the gateway has no single engine to read. type sampler struct { url string + apiKey string engine string client *http.Client cancel context.CancelFunc @@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) { if err != nil { return } + if s.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+s.apiKey) + } resp, err := s.client.Do(req) if err != nil { return diff --git a/internal/cli/bench.go b/internal/cli/bench.go index 55e3b293..5c596917 100644 --- a/internal/cli/bench.go +++ b/internal/cli/bench.go @@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { cfg.MetricsURL = metrics cfg.Engine = engine cfg.APIKey = key + // A replica's engine demands its own key. Asked for rather than + // typed: the admin API knows which replica answers at the target. + if cfg.APIKey == "" { + cfg.APIKey = replicaKey(cmd, client, target) + } if len(steps) > 0 { cfg.Concurrency = steps } @@ -227,3 +232,18 @@ func ms(d time.Duration) string { return d.Round(10 * time.Millisecond).String() } } + +// replicaKey is the engine key of the replica at target, or "" when target is +// not a replica (the gateway, say) or the admin API cannot be asked. +func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string { + var reps []adminapi.ReplicaView + if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil { + return "" + } + for _, rep := range reps { + if sameEndpoint(rep.Endpoint, target) { + return rep.EngineKey + } + } + return "" +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 927fb062..51e62b44 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -385,6 +385,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]} } + // Each replica's engine demands a key only the router has. The secret it + // is derived from lives in the store, so every control plane against it — + // and this one after a restart — computes the same key for a replica. + if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil { + secret, err := h.Store().ClusterSecret(ctx, "engine-key") + if err != nil { + return nil, nil, fmt.Errorf("reading the engine key secret: %w", err) + } + ccfg.Reconciler.EngineKey = control.EngineKeys([]byte(secret)) + } else { + log.Warn("engines start without a key: there is no store to keep the secret " + + "they are derived from, so anyone who can reach a node's ports reaches its models") + } + // The service needs the inventory and the status sink, both of which the // controller owns; the controller needs the service as its dispatcher. The // cycle is broken by giving the controller a dispatcher that forwards to diff --git a/internal/config/config.go b/internal/config/config.go index a3c56b3a..e0f5a177 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -404,8 +404,11 @@ type Scoring struct { // Endpoint is one engine replica. Static in phase 1. type Endpoint struct { - ID string `koanf:"id"` - URL string `koanf:"url"` + ID string `koanf:"id"` + URL string `koanf:"url"` + // APIKey is sent to the engine as a bearer token, for an engine started + // with one. + APIKey string `koanf:"api_key"` Models []string `koanf:"models"` // Engine names the inference engine, which decides what its metrics are // called. Empty means vLLM's names. diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 20d986bc..ed041df7 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -238,9 +238,13 @@ type ModelView struct { // ReplicaView is a replica as `ctl replicas` shows it. type ReplicaView struct { - ID string `json:"id"` - Model string `json:"model"` - Profile string `json:"profile"` + ID string `json:"id"` + // EngineKey is the key this replica's engine demands, for `ctl bench` + // and anyone else holding the admin listener who needs to reach an engine + // directly. The client-facing pages never render it. + EngineKey string `json:"engine_key,omitempty"` + Model string `json:"model"` + Profile string `json:"profile"` // Stale is true when the profile has been edited since this replica was // placed, so it is running a definition the catalog no longer holds. // @@ -471,9 +475,10 @@ func (a *API) Replicas(model string) []ReplicaView { ID: rep.ID, Model: rep.Model, Profile: rep.Profile, State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric, Endpoint: rep.Endpoint, Err: rep.Err, - Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP), - AgeSec: int64(time.Since(rep.CreatedAt).Seconds()), - IdleSec: -1, + EngineKey: a.ctl.EngineKey(rep.ID), + Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP), + AgeSec: int64(time.Since(rep.CreatedAt).Seconds()), + IdleSec: -1, } if rep.LastUsedAt.IsZero() { v.IdleSec = -1 diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 69ba97ce..8bfba45b 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -191,6 +191,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { Pool: cmd.Pool, Fabric: cmd.Fabric, ProfileFp: cmd.ProfileFP, + ApiKey: cmd.Spec.APIKey, } } @@ -211,6 +212,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration Pool: m.GetPool(), Fabric: m.GetFabric(), ProfileFP: m.GetProfileFp(), + APIKey: m.GetApiKey(), Profile: engines.Profile{ Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(), TP: int(p.GetTp()), PP: int(p.GetPp()), diff --git a/internal/control/control.go b/internal/control/control.go index 57b97cb8..73f62d9a 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -4,6 +4,9 @@ package control import ( "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "fmt" "log/slog" "strings" @@ -304,6 +307,7 @@ func (c *Controller) endpoints() []endpoint.Info { ID: rep.ID, URL: strings.TrimRight(rep.Endpoint, "/"), Models: names, + APIKey: c.engineKey(rep.ID), Engine: prof.Engine, Pool: rep.Pool, Weight: rep.Weight, @@ -321,3 +325,29 @@ func fingerprint(infos []endpoint.Info) string { } return b.String() } + +// EngineKey is the key a replica's engine demands, empty when engines run +// without one. +func (c *Controller) EngineKey(replicaID string) string { return c.engineKey(replicaID) } + +func (c *Controller) engineKey(replicaID string) string { + if c.cfg.Reconciler.EngineKey == nil { + return "" + } + return c.cfg.Reconciler.EngineKey(replicaID) +} + +// EngineKeys derives each replica's engine key from one secret. +// +// HMAC rather than a stored random key per replica: the router needs the key +// for every replica it sends to, including ones adopted after a restart that +// this process never started, and a second control plane has to agree. A +// function of a shared secret and the replica id gives all three without a +// table to keep in step. A key taken from one node opens that replica only. +func EngineKeys(secret []byte) func(replicaID string) string { + return func(replicaID string) string { + m := hmac.New(sha256.New, secret) + m.Write([]byte("llmgw engine key\x00" + replicaID)) + return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40] + } +} diff --git a/internal/control/control_test.go b/internal/control/control_test.go new file mode 100644 index 00000000..d546e65f --- /dev/null +++ b/internal/control/control_test.go @@ -0,0 +1,19 @@ +package control + +import "testing" + +func TestEngineKeysAreTheSameForTheSameReplicaAndNoOther(t *testing.T) { + // A control plane that restarts, or a second one, must compute the key a + // running engine was started with; a key taken from one node must open + // that replica only. + a, b := EngineKeys([]byte("s")), EngineKeys([]byte("s")) + if a("r1") != b("r1") { + t.Error("the same secret gave two keys for one replica") + } + if a("r1") == a("r2") { + t.Error("two replicas share a key") + } + if a("r1") == EngineKeys([]byte("other"))("r1") { + t.Error("the key does not depend on the secret") + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index e906ae0c..c266799e 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -69,6 +69,11 @@ type Config struct { PortRange [2]int // MgmtInterface is the ethernet interface used for NCCL bootstrap. MgmtInterface string + // EngineKey gives each replica the key its engine demands. Derived from + // the replica id rather than stored, so a control plane that restarts — + // or a second one — computes the same key for a replica it adopts. Nil + // starts engines without one. + EngineKey func(replicaID string) string // EvictionGrace is how long a replica must have gone unasked-for before it // may be stopped to make room for another model. // @@ -883,6 +888,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac MgmtInterface: r.cfg.MgmtInterface, HTTPPort: s.HTTPPort, Profile: espec, + APIKey: r.engineKey(id), }, StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, @@ -1380,3 +1386,10 @@ func (r *Reconciler) profileMemory(model, profile string) int { } return 0 } + +func (r *Reconciler) engineKey(replicaID string) string { + if r.cfg.EngineKey == nil { + return "" + } + return r.cfg.EngineKey(replicaID) +} diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index ebfbbc9a..a2df6161 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1721,3 +1721,23 @@ func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) { t.Errorf("an unreported fingerprint should stay empty, got %q", fp) } } + +func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) { + // Engines listen on every address a node has. Started without a key, any + // machine that could reach the port reached the model past the gateway's + // authentication, quotas and accounting — checked from a neighbouring node + // on the live cluster. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id } + + h.r.Reconcile(t.Context()) + started := h.disp.startedShards() + if len(started) == 0 { + t.Fatal("nothing was started") + } + for _, c := range started { + if c.Spec.APIKey != "key-for-"+c.ReplicaID { + t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey) + } + } +} diff --git a/internal/mirror/safefetch.go b/internal/mirror/safefetch.go index bfa37acd..290b2a3d 100644 --- a/internal/mirror/safefetch.go +++ b/internal/mirror/safefetch.go @@ -84,4 +84,3 @@ func validSource(source string) error { } return checkScheme(u) } - diff --git a/internal/router/endpoint/endpoint.go b/internal/router/endpoint/endpoint.go index 9c8206ca..940b7b5c 100644 --- a/internal/router/endpoint/endpoint.go +++ b/internal/router/endpoint/endpoint.go @@ -22,6 +22,9 @@ type Info struct { ID string // URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash. URL string + // APIKey is what the engine demands of every request, sent as a bearer + // token. Empty for an engine that asks for none. + APIKey string // Models lists the model names this replica serves. Models []string // Engine names the inference engine serving here, which decides what its diff --git a/internal/router/proxy/proxy.go b/internal/router/proxy/proxy.go index 8a713f75..693e13d1 100644 --- a/internal/router/proxy/proxy.go +++ b/internal/router/proxy/proxy.go @@ -148,6 +148,8 @@ type Request struct { Body []byte // Header carries headers to forward (authorization is deliberately not). Header http.Header + // APIKey is the engine's own key, sent in place of the client's. + APIKey string // Stream says whether an SSE response is expected. Stream bool // SuppressUsageChunk drops the final usage-only chunk from the client while @@ -218,6 +220,9 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco } req.Header = cloneForUpstream(r.Header) req.Header.Set("Content-Type", "application/json") + if r.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+r.APIKey) + } req.ContentLength = int64(len(r.Body)) // A non-streaming request waits for the whole completion before a single @@ -518,8 +523,7 @@ func copyHeaders(dst, src http.Header) { // cloneForUpstream forwards only what the engine needs. The client's // Authorization header is deliberately dropped: it authenticates the caller to -// llmgw, not llmgw to the engine, and engines are reached over a trusted -// network (plan §14). +// llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do. func cloneForUpstream(h http.Header) http.Header { out := make(http.Header, 4) for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} { diff --git a/internal/router/proxy/proxy_test.go b/internal/router/proxy/proxy_test.go index 6e9bc3bb..fb031687 100644 --- a/internal/router/proxy/proxy_test.go +++ b/internal/router/proxy/proxy_test.go @@ -355,3 +355,22 @@ func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) { t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed) } } + +func TestTheEngineGetsLlmgwsKeyNotTheClients(t *testing.T) { + var got string + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Get("Authorization") + io.WriteString(w, `{}`) + })) + defer up.Close() + + client := http.Header{"Authorization": {"Bearer sk-the-callers-own"}} + _, err := testProxy().Do(t.Context(), httptest.NewRecorder(), + Request{URL: up.URL, Body: []byte(`{}`), Header: client, APIKey: "llmgw-engine"}) + if err != nil { + t.Fatal(err) + } + if got != "Bearer llmgw-engine" { + t.Errorf("the engine was sent %q", got) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 9e3c40e1..23a782d5 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -373,6 +373,7 @@ func toInfos(eps []config.Endpoint) []endpoint.Info { out = append(out, endpoint.Info{ ID: e.ID, URL: strings.TrimRight(e.URL, "/"), + APIKey: e.APIKey, Models: e.Models, Engine: e.Engine, Pool: e.Pool, @@ -673,6 +674,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa out, perr := r.px.Do(ctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, + APIKey: dec.Endpoint.APIKey, Body: req.UpstreamBody, Header: req.Header, Stream: req.Stream, diff --git a/internal/router/scrape/poller.go b/internal/router/scrape/poller.go index f9242da3..231915f0 100644 --- a/internal/router/scrape/poller.go +++ b/internal/router/scrape/poller.go @@ -157,7 +157,7 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) { } func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) { - stats, err := p.fetch(ctx, in.URL, p.mappingFor(in)) + stats, err := p.fetch(ctx, in.URL, in.APIKey, p.mappingFor(in)) if err != nil { if ctx.Err() != nil { return @@ -201,7 +201,7 @@ func (p *Poller) report(in endpoint.Info, err error, reported *string) { *reported = "unreachable" } -func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) { +func (p *Poller) fetch(ctx context.Context, base, apiKey string, mapping Mapping) (endpoint.Stats, error) { ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) defer cancel() @@ -209,6 +209,10 @@ func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpo if err != nil { return endpoint.Stats{}, err } + // llama.cpp guards /metrics with its key; vLLM does not, and ignores one. + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } resp, err := p.http.Do(req) if err != nil { return endpoint.Stats{}, err diff --git a/internal/store/copy.go b/internal/store/copy.go index 6efa537f..c4dae488 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -38,6 +38,10 @@ var copyOrder = []string{ "threads", "thread_messages", "memory_notes", + // Copied, not regenerated: every running engine was started with a key + // derived from this, and a new secret would lock the router out of all + // of them at the moment of the move. + "cluster_secrets", } // notCopied are tables deliberately left behind. diff --git a/internal/store/migrations/postgres/0011_cluster_secrets.sql b/internal/store/migrations/postgres/0011_cluster_secrets.sql new file mode 100644 index 00000000..2ac9fc13 --- /dev/null +++ b/internal/store/migrations/postgres/0011_cluster_secrets.sql @@ -0,0 +1,12 @@ +-- Secrets the control planes share and nobody types: generated on first use, +-- read by every control plane against this store afterwards. +-- +-- The first is the one each replica's engine key is derived from. It has to be +-- the same for every control plane — the router sends a replica the key it was +-- started with, whichever control plane started it — and the same after a +-- restart, when running replicas are adopted rather than restarted. +CREATE TABLE cluster_secrets ( + name TEXT PRIMARY KEY, + value TEXT NOT NULL, + created_at BIGINT NOT NULL +); diff --git a/internal/store/migrations/sqlite/0011_cluster_secrets.sql b/internal/store/migrations/sqlite/0011_cluster_secrets.sql new file mode 100644 index 00000000..2ac9fc13 --- /dev/null +++ b/internal/store/migrations/sqlite/0011_cluster_secrets.sql @@ -0,0 +1,12 @@ +-- Secrets the control planes share and nobody types: generated on first use, +-- read by every control plane against this store afterwards. +-- +-- The first is the one each replica's engine key is derived from. It has to be +-- the same for every control plane — the router sends a replica the key it was +-- started with, whichever control plane started it — and the same after a +-- restart, when running replicas are adopted rather than restarted. +CREATE TABLE cluster_secrets ( + name TEXT PRIMARY KEY, + value TEXT NOT NULL, + created_at BIGINT NOT NULL +); diff --git a/internal/store/secrets.go b/internal/store/secrets.go new file mode 100644 index 00000000..1dc0bbb6 --- /dev/null +++ b/internal/store/secrets.go @@ -0,0 +1,31 @@ +package store + +import ( + "context" + "crypto/rand" + "encoding/hex" + "time" +) + +// ClusterSecret returns the named secret, creating a random one the first time +// it is asked for. +// +// Insert-if-absent and then read, so two control planes asking at once end up +// with the same value: the loser's insert changes nothing and it reads the +// winner's. +func (s *DB) ClusterSecret(ctx context.Context, name string) (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + if _, err := s.db.ExecContext(ctx, + `INSERT INTO cluster_secrets (name, value, created_at) VALUES (?, ?, ?) + ON CONFLICT (name) DO NOTHING`, + name, hex.EncodeToString(b), time.Now().UnixMilli()); err != nil { + return "", err + } + var v string + err := s.db.QueryRowContext(ctx, + `SELECT value FROM cluster_secrets WHERE name = ?`, name).Scan(&v) + return v, err +} diff --git a/internal/store/secrets_test.go b/internal/store/secrets_test.go new file mode 100644 index 00000000..3e7d4eb4 --- /dev/null +++ b/internal/store/secrets_test.go @@ -0,0 +1,25 @@ +package store + +import ( + "path/filepath" + "testing" +) + +func TestAClusterSecretIsMadeOnceAndKept(t *testing.T) { + db, err := Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + a, err := db.ClusterSecret(t.Context(), "engine-key") + if err != nil || len(a) != 64 { + t.Fatalf("%q %v", a, err) + } + b, err := db.ClusterSecret(t.Context(), "engine-key") + if err != nil || a != b { + t.Errorf("a second read made a new secret: %q then %q", a, b) + } + if c, _ := db.ClusterSecret(t.Context(), "other"); c == a { + t.Error("two names share a secret") + } +}