Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
106 lines
3.2 KiB
Go
106 lines
3.2 KiB
Go
package supervisor
|
|
|
|
import (
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// engineStub serves configurable /health and /v1/models responses.
|
|
func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
|
|
t.Helper()
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
|
if !healthy {
|
|
http.Error(w, "loading", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
})
|
|
mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, _ *http.Request) {
|
|
var b strings.Builder
|
|
b.WriteString(`{"object":"list","data":[`)
|
|
for i, m := range models {
|
|
if i > 0 {
|
|
b.WriteString(",")
|
|
}
|
|
b.WriteString(`{"id":"` + m + `"}`)
|
|
}
|
|
b.WriteString(`]}`)
|
|
io.WriteString(w, b.String())
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestReadyWhenServingModel(t *testing.T) {
|
|
srv := engineStub(t, true, "llama-3.3-70b")
|
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
|
|
t.Errorf("a serving engine was reported unready: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNotReadyWhileLoading(t *testing.T) {
|
|
// The case that matters: the process is up but the weights are not loaded.
|
|
// Routing on /health alone would send traffic into a loading model.
|
|
srv := engineStub(t, false, "llama-3.3-70b")
|
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
|
if err == nil {
|
|
t.Fatal("an unhealthy engine was reported ready")
|
|
}
|
|
if !strings.Contains(err.Error(), "health") {
|
|
t.Errorf("err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestNotReadyWhenModelMissing(t *testing.T) {
|
|
// /health passes but the model is not listed: the engine is up and serving
|
|
// something else, or still registering.
|
|
srv := engineStub(t, true, "some-other-model")
|
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
|
if err == nil {
|
|
t.Fatal("a model that is not served was reported ready")
|
|
}
|
|
if !strings.Contains(err.Error(), "does not serve") {
|
|
t.Errorf("err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestModelCheckCanBeSkipped(t *testing.T) {
|
|
srv := engineStub(t, true)
|
|
h := NewHTTPHealth(time.Second, WithoutModelCheck())
|
|
if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
|
|
t.Errorf("the model check was not skipped: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestUnreachableEngine(t *testing.T) {
|
|
h := NewHTTPHealth(200 * time.Millisecond)
|
|
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
|
|
t.Error("an unreachable engine was reported ready")
|
|
}
|
|
}
|
|
|
|
func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
|
|
// An engine started with a key refuses /v1/models without it, and a
|
|
// shard that is serving would read as never ready.
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Header.Get("Authorization") != "Bearer k1" {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
|
|
}))
|
|
defer srv.Close()
|
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
|
|
t.Errorf("with the key: %v", err)
|
|
}
|
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
|
|
t.Error("without the key the engine refused, and that read as ready")
|
|
}
|
|
}
|