Give two control planes a shared address, and a probe that means something
The last piece of HA, and it is a deployment pattern rather than code: llmgw picks one leader and agents follow it, but client traffic arrives in front of both and llmgw is not in front of anything. Writing the pattern found a bug in the pattern. The first draft said to health-check `/health` and claimed a standby still routes to replicas the leader placed, so sending it traffic was fine. It does not. A standby's reconciler places nothing and it holds no agent streams to adopt from, so it publishes an empty endpoint list — a load balancer following that advice would have sent half the traffic to something that answers every request with 503. So there is now a `/ready` that means "this process has somewhere to send a request": 503 on a standby, 503 on a leader whose replicas have all failed, 200 when there is something to serve. `/health` keeps meaning the process is up, which is a different and less useful question. Both are open, because a load balancer has no key. There is a gap during failover where neither passes — three seconds when the leader stops cleanly, up to the lease TTL when it does not — and the document says so rather than implying otherwise. Traffic cannot go to a control plane that has nothing to route to. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a35e8796d2
commit
2bf7de1a43
6 files changed
+257
-1
No files matched your search
@@ -0,0 +1,68 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
)
|
||||
|
||||
// probe fetches a path with no key, as a load balancer would.
|
||||
func probe(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func TestReadinessMeansSomethingHealthDoesNot(t *testing.T) {
|
||||
// A standby control plane is up, answers /health, and has no endpoints —
|
||||
// its reconciler placed nothing and it holds no agent streams to adopt
|
||||
// from. A load balancer checking /health would send it half the traffic,
|
||||
// and half the traffic would be a 503.
|
||||
h := start(t, 2)
|
||||
|
||||
if code := probe(t, h.router.URL+"/health"); code != http.StatusOK {
|
||||
t.Errorf("/health with endpoints = %d", code)
|
||||
}
|
||||
if code := probe(t, h.router.URL+"/ready"); code != http.StatusOK {
|
||||
t.Errorf("/ready with endpoints = %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNothingToServeReadyRefusesAndHealthDoesNot(t *testing.T) {
|
||||
// The case the endpoint exists for. Both are true statements: the process
|
||||
// is alive, and it cannot serve.
|
||||
h := start(t, 0, func(c *config.Server) {
|
||||
c.Endpoints = nil
|
||||
})
|
||||
|
||||
if code := probe(t, h.router.URL+"/health"); code != http.StatusOK {
|
||||
t.Errorf("/health on a live process with no endpoints = %d, want 200", code)
|
||||
}
|
||||
if code := probe(t, h.router.URL+"/ready"); code != http.StatusServiceUnavailable {
|
||||
t.Errorf("/ready with nothing to serve = %d, want 503", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBothProbesWorkWithoutAKey(t *testing.T) {
|
||||
// A load balancer does not have one, and an endpoint it cannot reach is
|
||||
// an endpoint that reports every gateway as down.
|
||||
path, mint := provision(t)
|
||||
_ = mint(t, "somebody")
|
||||
h := start(t, 2, paidConfig(path))
|
||||
|
||||
for _, p := range []string{"/health", "/ready"} {
|
||||
if code := probe(t, h.router.URL+p); code != http.StatusOK {
|
||||
t.Errorf("%s on an authenticated gateway = %d", p, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user