Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
103 lines
2.5 KiB
Go
103 lines
2.5 KiB
Go
package supervisor
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// HTTPHealth checks an engine over HTTP.
|
|
//
|
|
// Readiness is two questions, not one (plan §5.4): /health says the process is
|
|
// up, and /v1/models says the weights finished loading and the model is being
|
|
// served. An engine answers /health long before it can serve a request, so
|
|
// routing on /health alone sends traffic into a model that is still loading.
|
|
type HTTPHealth struct {
|
|
client *http.Client
|
|
healthPath string
|
|
modelsPath string
|
|
requireName bool
|
|
}
|
|
|
|
// HTTPHealthOption customizes the checker.
|
|
type HTTPHealthOption func(*HTTPHealth)
|
|
|
|
// WithHealthPath overrides the readiness path.
|
|
func WithHealthPath(p string) HTTPHealthOption {
|
|
return func(h *HTTPHealth) { h.healthPath = p }
|
|
}
|
|
|
|
// WithoutModelCheck skips the /v1/models step, for engines that do not expose it.
|
|
func WithoutModelCheck() HTTPHealthOption {
|
|
return func(h *HTTPHealth) { h.requireName = false }
|
|
}
|
|
|
|
// NewHTTPHealth builds a checker.
|
|
func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth {
|
|
if timeout <= 0 {
|
|
timeout = 3 * time.Second
|
|
}
|
|
h := &HTTPHealth{
|
|
client: &http.Client{Timeout: timeout},
|
|
healthPath: "/health",
|
|
modelsPath: "/v1/models",
|
|
requireName: true,
|
|
}
|
|
for _, o := range opts {
|
|
o(h)
|
|
}
|
|
return h
|
|
}
|
|
|
|
// Ready reports whether the engine at baseURL is serving model.
|
|
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
|
|
if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
|
|
return fmt.Errorf("health: %w", err)
|
|
}
|
|
if !h.requireName || model == "" {
|
|
return nil
|
|
}
|
|
|
|
var list struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
|
|
return fmt.Errorf("models: %w", err)
|
|
}
|
|
for _, d := range list.Data {
|
|
if d.ID == model {
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("engine is up but does not serve %q yet", model)
|
|
}
|
|
|
|
func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if apiKey != "" {
|
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
|
}
|
|
resp, err := h.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10))
|
|
return fmt.Errorf("status %s", resp.Status)
|
|
}
|
|
if out == nil {
|
|
return nil
|
|
}
|
|
return json.NewDecoder(resp.Body).Decode(out)
|
|
}
|