Give every engine a key only the router holds
Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d167686cf1
commit
9853936e0b
34 files changed
+356
-29
No files matched your search
@@ -2525,7 +2525,11 @@ type StartShard struct {
|
||||
Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"`
|
||||
// profile_fp identifies the version of the profile, so the agent can say
|
||||
// which one a running shard was started from. The agent never interprets it.
|
||||
ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"`
|
||||
ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"`
|
||||
// api_key is what the engine demands of every request, derived by the
|
||||
// control plane from the replica id. The router sends it; nothing else
|
||||
// that can reach the node's ports has it.
|
||||
ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -2665,6 +2669,13 @@ func (x *StartShard) GetProfileFp() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *StartShard) GetApiKey() string {
|
||||
if x != nil {
|
||||
return x.ApiKey
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type EngineProfile struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
@@ -3693,7 +3704,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" +
|
||||
"\vbytes_total\x18\x03 \x01(\x03R\n" +
|
||||
"bytesTotal\"\xd9\x03\n" +
|
||||
"bytesTotal\"\xf2\x03\n" +
|
||||
"\n" +
|
||||
"StartShard\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -3716,7 +3727,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04pool\x18\r \x01(\tR\x04pool\x12\x16\n" +
|
||||
"\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" +
|
||||
"\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xfd\x02\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" +
|
||||
"\rEngineProfile\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
|
||||
|
||||
@@ -524,6 +524,10 @@ message StartShard {
|
||||
// profile_fp identifies the version of the profile, so the agent can say
|
||||
// which one a running shard was started from. The agent never interprets it.
|
||||
string profile_fp = 15;
|
||||
// api_key is what the engine demands of every request, derived by the
|
||||
// control plane from the replica id. The router sends it; nothing else
|
||||
// that can reach the node's ports has it.
|
||||
string api_key = 16;
|
||||
}
|
||||
|
||||
message EngineProfile {
|
||||
|
||||
@@ -71,6 +71,7 @@ rather than repeated.
|
||||
| `lease_ttl` | `time.Duration` | [Control](#control) |
|
||||
| `id` | `string` | [Endpoint](#endpoint) |
|
||||
| `url` | `string` | [Endpoint](#endpoint) |
|
||||
| `api_key` | `string` | [Endpoint](#endpoint) |
|
||||
| `models` | `[]string` | [Endpoint](#endpoint) |
|
||||
| `engine` | `string` | [Endpoint](#endpoint) |
|
||||
| `pool` | `string` | [Endpoint](#endpoint) |
|
||||
@@ -566,6 +567,13 @@ reading the alert. Zero takes the default.
|
||||
|
||||
`string`
|
||||
|
||||
### `api_key`
|
||||
|
||||
`string`
|
||||
|
||||
APIKey is sent to the engine as a bearer token, for an engine started
|
||||
with one.
|
||||
|
||||
### `models`
|
||||
|
||||
`[]string`
|
||||
|
||||
@@ -70,6 +70,13 @@ type ShardSpec struct {
|
||||
HTTPPort int
|
||||
Profile Profile
|
||||
|
||||
// APIKey is what the engine demands of every request. Engines listen on
|
||||
// all of a node's addresses; without a key anybody who can reach the port
|
||||
// talks to the model directly, past the gateway's authentication, quotas
|
||||
// and accounting. Given to the engine in its environment rather than on
|
||||
// its command line, which every user on the node can read.
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
|
||||
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||
// are carried so the agent can report them back after a restart; Fabric
|
||||
// also says, with an "nvl:" prefix, that the replica is inside a
|
||||
|
||||
@@ -120,8 +120,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
|
||||
env := baseEnv(s)
|
||||
if s.APIKey != "" {
|
||||
env["LLAMA_API_KEY"] = s.APIKey
|
||||
}
|
||||
return []Command{{
|
||||
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
||||
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -245,3 +245,19 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
|
||||
t.Errorf("expected it passed through, got %q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
|
||||
// The command line is readable by every user on the node; the
|
||||
// environment of a root process is not.
|
||||
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
|
||||
Profile: Profile{ModelPath: "/m.gguf"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
|
||||
t.Errorf("the key is on the command line: %v", cmds[0].Args)
|
||||
}
|
||||
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
|
||||
t.Error("llama-server was not given its key")
|
||||
}
|
||||
}
|
||||
@@ -96,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
)
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
// SGLang reads its key only from the command line, so on a node with
|
||||
// other users it is visible to them in the process list.
|
||||
if s.APIKey != "" && s.Leader() {
|
||||
args = append(args, "--api-key", s.APIKey)
|
||||
}
|
||||
|
||||
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||
}
|
||||
@@ -150,6 +150,9 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
// Profile arguments last, so an operator can override anything above.
|
||||
args = append(args, p.Args...)
|
||||
|
||||
if s.APIKey != "" {
|
||||
env["VLLM_API_KEY"] = s.APIKey
|
||||
}
|
||||
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth
|
||||
}
|
||||
|
||||
// Ready reports whether the engine at baseURL is serving model.
|
||||
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
||||
if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
|
||||
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
|
||||
if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
|
||||
return fmt.Errorf("health: %w", err)
|
||||
}
|
||||
if !h.requireName || model == "" {
|
||||
@@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
|
||||
if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
|
||||
return fmt.Errorf("models: %w", err)
|
||||
}
|
||||
for _, d := range list.Data {
|
||||
@@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
||||
return fmt.Errorf("engine is up but does not serve %q yet", model)
|
||||
}
|
||||
|
||||
func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
|
||||
func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
|
||||
|
||||
func TestReadyWhenServingModel(t *testing.T) {
|
||||
srv := engineStub(t, true, "llama-3.3-70b")
|
||||
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
|
||||
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
|
||||
t.Errorf("a serving engine was reported unready: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) {
|
||||
// The case that matters: the process is up but the weights are not loaded.
|
||||
// Routing on /health alone would send traffic into a loading model.
|
||||
srv := engineStub(t, false, "llama-3.3-70b")
|
||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
||||
if err == nil {
|
||||
t.Fatal("an unhealthy engine was reported ready")
|
||||
}
|
||||
@@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
|
||||
// /health passes but the model is not listed: the engine is up and serving
|
||||
// something else, or still registering.
|
||||
srv := engineStub(t, true, "some-other-model")
|
||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
||||
if err == nil {
|
||||
t.Fatal("a model that is not served was reported ready")
|
||||
}
|
||||
@@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
|
||||
func TestModelCheckCanBeSkipped(t *testing.T) {
|
||||
srv := engineStub(t, true)
|
||||
h := NewHTTPHealth(time.Second, WithoutModelCheck())
|
||||
if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
|
||||
if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
|
||||
t.Errorf("the model check was not skipped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreachableEngine(t *testing.T) {
|
||||
h := NewHTTPHealth(200 * time.Millisecond)
|
||||
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
|
||||
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
|
||||
t.Error("an unreachable engine was reported ready")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
|
||||
// An engine started with a key refuses /v1/models without it, and a
|
||||
// shard that is serving would read as never ready.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer k1" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
|
||||
t.Errorf("with the key: %v", err)
|
||||
}
|
||||
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
|
||||
t.Error("without the key the engine refused, and that read as ready")
|
||||
}
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func (c *Config) normalize() {
|
||||
// supervisor can be tested without an HTTP engine.
|
||||
type HealthChecker interface {
|
||||
// Ready reports whether the shard at baseURL serves model.
|
||||
Ready(ctx context.Context, baseURL, model string) error
|
||||
Ready(ctx context.Context, baseURL, model, apiKey string) error
|
||||
}
|
||||
|
||||
// Shard is one supervised engine process.
|
||||
@@ -411,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
last := strings.Join(sh.Logs(10), "\n")
|
||||
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||
case <-t.C:
|
||||
if err := s.health.Ready(ctx, baseURL, model); err != nil {
|
||||
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
|
||||
continue
|
||||
}
|
||||
sh.setState(Ready, nil)
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// scriptedHealth reports ready once the flag is set.
|
||||
type scriptedHealth struct{ ready atomic.Bool }
|
||||
|
||||
func (h *scriptedHealth) Ready(context.Context, string, string) error {
|
||||
func (h *scriptedHealth) Ready(context.Context, string, string, string) error {
|
||||
if h.ready.Load() {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i
|
||||
)
|
||||
|
||||
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
|
||||
sampler.apiKey = cfg.APIKey
|
||||
sampler.start(ctx)
|
||||
|
||||
start := time.Now()
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
// Optional, because a run pointed at the gateway has no single engine to read.
|
||||
type sampler struct {
|
||||
url string
|
||||
apiKey string
|
||||
engine string
|
||||
client *http.Client
|
||||
cancel context.CancelFunc
|
||||
@@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if s.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||
}
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cfg.MetricsURL = metrics
|
||||
cfg.Engine = engine
|
||||
cfg.APIKey = key
|
||||
// A replica's engine demands its own key. Asked for rather than
|
||||
// typed: the admin API knows which replica answers at the target.
|
||||
if cfg.APIKey == "" {
|
||||
cfg.APIKey = replicaKey(cmd, client, target)
|
||||
}
|
||||
if len(steps) > 0 {
|
||||
cfg.Concurrency = steps
|
||||
}
|
||||
@@ -227,3 +232,18 @@ func ms(d time.Duration) string {
|
||||
return d.Round(10 * time.Millisecond).String()
|
||||
}
|
||||
}
|
||||
|
||||
// replicaKey is the engine key of the replica at target, or "" when target is
|
||||
// not a replica (the gateway, say) or the admin API cannot be asked.
|
||||
func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, rep := range reps {
|
||||
if sameEndpoint(rep.Endpoint, target) {
|
||||
return rep.EngineKey
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -385,6 +385,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
|
||||
}
|
||||
|
||||
// Each replica's engine demands a key only the router has. The secret it
|
||||
// is derived from lives in the store, so every control plane against it —
|
||||
// and this one after a restart — computes the same key for a replica.
|
||||
if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
|
||||
secret, err := h.Store().ClusterSecret(ctx, "engine-key")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("reading the engine key secret: %w", err)
|
||||
}
|
||||
ccfg.Reconciler.EngineKey = control.EngineKeys([]byte(secret))
|
||||
} else {
|
||||
log.Warn("engines start without a key: there is no store to keep the secret " +
|
||||
"they are derived from, so anyone who can reach a node's ports reaches its models")
|
||||
}
|
||||
|
||||
// The service needs the inventory and the status sink, both of which the
|
||||
// controller owns; the controller needs the service as its dispatcher. The
|
||||
// cycle is broken by giving the controller a dispatcher that forwards to
|
||||
|
||||
@@ -404,8 +404,11 @@ type Scoring struct {
|
||||
|
||||
// Endpoint is one engine replica. Static in phase 1.
|
||||
type Endpoint struct {
|
||||
ID string `koanf:"id"`
|
||||
URL string `koanf:"url"`
|
||||
ID string `koanf:"id"`
|
||||
URL string `koanf:"url"`
|
||||
// APIKey is sent to the engine as a bearer token, for an engine started
|
||||
// with one.
|
||||
APIKey string `koanf:"api_key"`
|
||||
Models []string `koanf:"models"`
|
||||
// Engine names the inference engine, which decides what its metrics are
|
||||
// called. Empty means vLLM's names.
|
||||
|
||||
@@ -238,9 +238,13 @@ type ModelView struct {
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
type ReplicaView struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
ID string `json:"id"`
|
||||
// EngineKey is the key this replica's engine demands, for `ctl bench`
|
||||
// and anyone else holding the admin listener who needs to reach an engine
|
||||
// directly. The client-facing pages never render it.
|
||||
EngineKey string `json:"engine_key,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
// Stale is true when the profile has been edited since this replica was
|
||||
// placed, so it is running a definition the catalog no longer holds.
|
||||
//
|
||||
@@ -471,9 +475,10 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
|
||||
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
|
||||
Endpoint: rep.Endpoint, Err: rep.Err,
|
||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||
IdleSec: -1,
|
||||
EngineKey: a.ctl.EngineKey(rep.ID),
|
||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||
IdleSec: -1,
|
||||
}
|
||||
if rep.LastUsedAt.IsZero() {
|
||||
v.IdleSec = -1
|
||||
|
||||
@@ -191,6 +191,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
Pool: cmd.Pool,
|
||||
Fabric: cmd.Fabric,
|
||||
ProfileFp: cmd.ProfileFP,
|
||||
ApiKey: cmd.Spec.APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,6 +212,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
Pool: m.GetPool(),
|
||||
Fabric: m.GetFabric(),
|
||||
ProfileFP: m.GetProfileFp(),
|
||||
APIKey: m.GetApiKey(),
|
||||
Profile: engines.Profile{
|
||||
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
||||
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
||||
|
||||
@@ -4,6 +4,9 @@ package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
@@ -304,6 +307,7 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
ID: rep.ID,
|
||||
URL: strings.TrimRight(rep.Endpoint, "/"),
|
||||
Models: names,
|
||||
APIKey: c.engineKey(rep.ID),
|
||||
Engine: prof.Engine,
|
||||
Pool: rep.Pool,
|
||||
Weight: rep.Weight,
|
||||
@@ -321,3 +325,29 @@ func fingerprint(infos []endpoint.Info) string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// EngineKey is the key a replica's engine demands, empty when engines run
|
||||
// without one.
|
||||
func (c *Controller) EngineKey(replicaID string) string { return c.engineKey(replicaID) }
|
||||
|
||||
func (c *Controller) engineKey(replicaID string) string {
|
||||
if c.cfg.Reconciler.EngineKey == nil {
|
||||
return ""
|
||||
}
|
||||
return c.cfg.Reconciler.EngineKey(replicaID)
|
||||
}
|
||||
|
||||
// EngineKeys derives each replica's engine key from one secret.
|
||||
//
|
||||
// HMAC rather than a stored random key per replica: the router needs the key
|
||||
// for every replica it sends to, including ones adopted after a restart that
|
||||
// this process never started, and a second control plane has to agree. A
|
||||
// function of a shared secret and the replica id gives all three without a
|
||||
// table to keep in step. A key taken from one node opens that replica only.
|
||||
func EngineKeys(secret []byte) func(replicaID string) string {
|
||||
return func(replicaID string) string {
|
||||
m := hmac.New(sha256.New, secret)
|
||||
m.Write([]byte("llmgw engine key\x00" + replicaID))
|
||||
return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package control
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEngineKeysAreTheSameForTheSameReplicaAndNoOther(t *testing.T) {
|
||||
// A control plane that restarts, or a second one, must compute the key a
|
||||
// running engine was started with; a key taken from one node must open
|
||||
// that replica only.
|
||||
a, b := EngineKeys([]byte("s")), EngineKeys([]byte("s"))
|
||||
if a("r1") != b("r1") {
|
||||
t.Error("the same secret gave two keys for one replica")
|
||||
}
|
||||
if a("r1") == a("r2") {
|
||||
t.Error("two replicas share a key")
|
||||
}
|
||||
if a("r1") == EngineKeys([]byte("other"))("r1") {
|
||||
t.Error("the key does not depend on the secret")
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,11 @@ type Config struct {
|
||||
PortRange [2]int
|
||||
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
||||
MgmtInterface string
|
||||
// EngineKey gives each replica the key its engine demands. Derived from
|
||||
// the replica id rather than stored, so a control plane that restarts —
|
||||
// or a second one — computes the same key for a replica it adopts. Nil
|
||||
// starts engines without one.
|
||||
EngineKey func(replicaID string) string
|
||||
// EvictionGrace is how long a replica must have gone unasked-for before it
|
||||
// may be stopped to make room for another model.
|
||||
//
|
||||
@@ -883,6 +888,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
MgmtInterface: r.cfg.MgmtInterface,
|
||||
HTTPPort: s.HTTPPort,
|
||||
Profile: espec,
|
||||
APIKey: r.engineKey(id),
|
||||
},
|
||||
StartupTimeout: prof.StartupTimeout,
|
||||
Pool: pl.Pool,
|
||||
@@ -1380,3 +1386,10 @@ func (r *Reconciler) profileMemory(model, profile string) int {
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *Reconciler) engineKey(replicaID string) string {
|
||||
if r.cfg.EngineKey == nil {
|
||||
return ""
|
||||
}
|
||||
return r.cfg.EngineKey(replicaID)
|
||||
}
|
||||
@@ -1721,3 +1721,23 @@ func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
|
||||
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) {
|
||||
// Engines listen on every address a node has. Started without a key, any
|
||||
// machine that could reach the port reached the model past the gateway's
|
||||
// authentication, quotas and accounting — checked from a neighbouring node
|
||||
// on the live cluster.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id }
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
started := h.disp.startedShards()
|
||||
if len(started) == 0 {
|
||||
t.Fatal("nothing was started")
|
||||
}
|
||||
for _, c := range started {
|
||||
if c.Spec.APIKey != "key-for-"+c.ReplicaID {
|
||||
t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,4 +84,3 @@ func validSource(source string) error {
|
||||
}
|
||||
return checkScheme(u)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ type Info struct {
|
||||
ID string
|
||||
// URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash.
|
||||
URL string
|
||||
// APIKey is what the engine demands of every request, sent as a bearer
|
||||
// token. Empty for an engine that asks for none.
|
||||
APIKey string
|
||||
// Models lists the model names this replica serves.
|
||||
Models []string
|
||||
// Engine names the inference engine serving here, which decides what its
|
||||
|
||||
@@ -148,6 +148,8 @@ type Request struct {
|
||||
Body []byte
|
||||
// Header carries headers to forward (authorization is deliberately not).
|
||||
Header http.Header
|
||||
// APIKey is the engine's own key, sent in place of the client's.
|
||||
APIKey string
|
||||
// Stream says whether an SSE response is expected.
|
||||
Stream bool
|
||||
// SuppressUsageChunk drops the final usage-only chunk from the client while
|
||||
@@ -218,6 +220,9 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
|
||||
}
|
||||
req.Header = cloneForUpstream(r.Header)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if r.APIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+r.APIKey)
|
||||
}
|
||||
req.ContentLength = int64(len(r.Body))
|
||||
|
||||
// A non-streaming request waits for the whole completion before a single
|
||||
@@ -518,8 +523,7 @@ func copyHeaders(dst, src http.Header) {
|
||||
|
||||
// cloneForUpstream forwards only what the engine needs. The client's
|
||||
// Authorization header is deliberately dropped: it authenticates the caller to
|
||||
// llmgw, not llmgw to the engine, and engines are reached over a trusted
|
||||
// network (plan §14).
|
||||
// llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do.
|
||||
func cloneForUpstream(h http.Header) http.Header {
|
||||
out := make(http.Header, 4)
|
||||
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} {
|
||||
|
||||
@@ -355,3 +355,22 @@ func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) {
|
||||
t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEngineGetsLlmgwsKeyNotTheClients(t *testing.T) {
|
||||
var got string
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got = r.Header.Get("Authorization")
|
||||
io.WriteString(w, `{}`)
|
||||
}))
|
||||
defer up.Close()
|
||||
|
||||
client := http.Header{"Authorization": {"Bearer sk-the-callers-own"}}
|
||||
_, err := testProxy().Do(t.Context(), httptest.NewRecorder(),
|
||||
Request{URL: up.URL, Body: []byte(`{}`), Header: client, APIKey: "llmgw-engine"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "Bearer llmgw-engine" {
|
||||
t.Errorf("the engine was sent %q", got)
|
||||
}
|
||||
}
|
||||
@@ -373,6 +373,7 @@ func toInfos(eps []config.Endpoint) []endpoint.Info {
|
||||
out = append(out, endpoint.Info{
|
||||
ID: e.ID,
|
||||
URL: strings.TrimRight(e.URL, "/"),
|
||||
APIKey: e.APIKey,
|
||||
Models: e.Models,
|
||||
Engine: e.Engine,
|
||||
Pool: e.Pool,
|
||||
@@ -673,6 +674,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
|
||||
out, perr := r.px.Do(ctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
Body: req.UpstreamBody,
|
||||
Header: req.Header,
|
||||
Stream: req.Stream,
|
||||
|
||||
@@ -157,7 +157,7 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
|
||||
}
|
||||
|
||||
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
|
||||
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
|
||||
stats, err := p.fetch(ctx, in.URL, in.APIKey, p.mappingFor(in))
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
@@ -201,7 +201,7 @@ func (p *Poller) report(in endpoint.Info, err error, reported *string) {
|
||||
*reported = "unreachable"
|
||||
}
|
||||
|
||||
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
|
||||
func (p *Poller) fetch(ctx context.Context, base, apiKey string, mapping Mapping) (endpoint.Stats, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -209,6 +209,10 @@ func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpo
|
||||
if err != nil {
|
||||
return endpoint.Stats{}, err
|
||||
}
|
||||
// llama.cpp guards /metrics with its key; vLLM does not, and ignores one.
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return endpoint.Stats{}, err
|
||||
|
||||
@@ -38,6 +38,10 @@ var copyOrder = []string{
|
||||
"threads",
|
||||
"thread_messages",
|
||||
"memory_notes",
|
||||
// Copied, not regenerated: every running engine was started with a key
|
||||
// derived from this, and a new secret would lock the router out of all
|
||||
// of them at the moment of the move.
|
||||
"cluster_secrets",
|
||||
}
|
||||
|
||||
// notCopied are tables deliberately left behind.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Secrets the control planes share and nobody types: generated on first use,
|
||||
-- read by every control plane against this store afterwards.
|
||||
--
|
||||
-- The first is the one each replica's engine key is derived from. It has to be
|
||||
-- the same for every control plane — the router sends a replica the key it was
|
||||
-- started with, whichever control plane started it — and the same after a
|
||||
-- restart, when running replicas are adopted rather than restarted.
|
||||
CREATE TABLE cluster_secrets (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
created_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Secrets the control planes share and nobody types: generated on first use,
|
||||
-- read by every control plane against this store afterwards.
|
||||
--
|
||||
-- The first is the one each replica's engine key is derived from. It has to be
|
||||
-- the same for every control plane — the router sends a replica the key it was
|
||||
-- started with, whichever control plane started it — and the same after a
|
||||
-- restart, when running replicas are adopted rather than restarted.
|
||||
CREATE TABLE cluster_secrets (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
created_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,31 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClusterSecret returns the named secret, creating a random one the first time
|
||||
// it is asked for.
|
||||
//
|
||||
// Insert-if-absent and then read, so two control planes asking at once end up
|
||||
// with the same value: the loser's insert changes nothing and it reads the
|
||||
// winner's.
|
||||
func (s *DB) ClusterSecret(ctx context.Context, name string) (string, error) {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO cluster_secrets (name, value, created_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT (name) DO NOTHING`,
|
||||
name, hex.EncodeToString(b), time.Now().UnixMilli()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var v string
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT value FROM cluster_secrets WHERE name = ?`, name).Scan(&v)
|
||||
return v, err
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAClusterSecretIsMadeOnceAndKept(t *testing.T) {
|
||||
db, err := Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
a, err := db.ClusterSecret(t.Context(), "engine-key")
|
||||
if err != nil || len(a) != 64 {
|
||||
t.Fatalf("%q %v", a, err)
|
||||
}
|
||||
b, err := db.ClusterSecret(t.Context(), "engine-key")
|
||||
if err != nil || a != b {
|
||||
t.Errorf("a second read made a new secret: %q then %q", a, b)
|
||||
}
|
||||
if c, _ := db.ClusterSecret(t.Context(), "other"); c == a {
|
||||
t.Error("two names share a secret")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user