A standby's portal is the leader's; the changelog as the release notes read it
A proxy left pointing at a standby showed an empty cluster: the portal and the admin pages read the standby's own memory. A standby now passes its client port to the leader's, all but /health, /ready and /leader, which must describe the machine a load balancer is choosing; and its admin port passes the dashboard as well as the API. The leader trusts the control planes' X-Forwarded-For, so clients keep their own address. The changelog's candidate section is "Unreleased" again: release notes for an rc read that section, and v0.11.3-rc.1 got none. The e2e wait for two endpoints is 30s: under the race detector on a busy runner, 15s was missed once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c5b34978f5
commit
abaf086fd5
7 files changed
+172
-23
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
@@ -140,9 +141,15 @@ func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) {
|
||||
if got["actor"] != "root@ctl-b" {
|
||||
t.Errorf("who asked was lost: %q", got["actor"])
|
||||
}
|
||||
// What is not the API stays on the machine: the dashboard, pprof.
|
||||
if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" {
|
||||
t.Errorf("a non-API path was forwarded: %v", got)
|
||||
// The dashboard's pages are the leader's too.
|
||||
if _, got := c.ask(t, "/nodes/cuda01"); got["path"] != "/nodes/cuda01" {
|
||||
t.Errorf("a dashboard page was answered here: %v", got)
|
||||
}
|
||||
// What is about this process stays on it: pprof, its metrics.
|
||||
for _, p := range []string{"/debug/pprof/", "/metrics"} {
|
||||
if _, got := c.ask(t, p); got["path"] != "local" {
|
||||
t.Errorf("%s was forwarded: %v", p, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,3 +193,52 @@ func TestAStandbyWithNoLeaderSaysSo(t *testing.T) {
|
||||
t.Errorf("%d %v", resp.StatusCode, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandbysPortalIsTheLeaders(t *testing.T) {
|
||||
c := newForwardCluster(t)
|
||||
// The leader's client port, on the address the store lists for it.
|
||||
var got http.Header
|
||||
var gotHost string
|
||||
leader := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got, gotHost = r.Header.Clone(), r.Host
|
||||
_, _ = w.Write([]byte("leader " + r.URL.Path))
|
||||
}))
|
||||
defer leader.Close()
|
||||
_, port, _ := net.SplitHostPort(strings.TrimPrefix(leader.URL, "http://"))
|
||||
|
||||
local := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("standby " + r.URL.Path)) })
|
||||
srv := httptest.NewServer(c.standby.public(local, port))
|
||||
defer srv.Close()
|
||||
get := func(path string, hdr map[string]string) string {
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil)
|
||||
req.Host = "llmgw.example"
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
if body := get("/admin/replicas", map[string]string{"X-Forwarded-For": "203.0.113.7", "X-Forwarded-Proto": "https"}); body != "leader /admin/replicas" {
|
||||
t.Fatalf("the portal was answered by %q", body)
|
||||
}
|
||||
if gotHost != "llmgw.example" || got.Get("X-Forwarded-Proto") != "https" ||
|
||||
!strings.HasPrefix(got.Get("X-Forwarded-For"), "203.0.113.7, ") || got.Get(hdrForwardedBy) != "ctl-b" {
|
||||
t.Errorf("host %q, headers %v", gotHost, got)
|
||||
}
|
||||
// What a load balancer asks is about this machine.
|
||||
for _, p := range []string{"/ready", "/health", "/leader"} {
|
||||
if body := get(p, nil); body != "standby "+p {
|
||||
t.Errorf("%s answered by %q", p, body)
|
||||
}
|
||||
}
|
||||
// Forwarded once only.
|
||||
if body := get("/v1/models", map[string]string{hdrForwardedBy: "ctl-c"}); body != "standby /v1/models" {
|
||||
t.Errorf("a forwarded request was forwarded again: %q", body)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user