From 524f0296034b377f7c8104509e7f086e7f53d3ce Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 30 Oct 2026 19:13:15 +0100 Subject: [PATCH] External providers left out of admission's saturation; the scale-out section in the backlog Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 20 ++++++++++++++++++++ CHANGELOG.md | 4 ++++ internal/router/admission/admission_test.go | 17 +++++++++++++++++ internal/router/admission/saturation.go | 8 ++++++++ 4 files changed, 49 insertions(+) diff --git a/BACKLOG.md b/BACKLOG.md index 62ff492..cf7ebf6 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -46,6 +46,8 @@ providers, of L5 cache-write and audio prices. L6 and L7 are llmgw-80's; L8 (A2A cluster's hardware holds, with CPU and GPU tok/s). Next M2 (every download pinned and checked against Hugging Face, done 2026-10-30), then M5 (engines without the internet), M3, M4, M6. +**Scale.** O: more than one control plane answering clients; O1 (measuring one router's ceiling) first. + **Governance.** 7. **C1**: a tenant-admin role, and forms hidden from viewers. 8. **Audit before and after**: what a change changed, and changes made with the CLI recorded too. @@ -1910,6 +1912,24 @@ served only from a definition written by hand. In order: - N16. Versions apart from llmgw's releases (Strata's signed update feed, `--rollback-engine`); offline bundles. - N17. `cache_salt` for KB isolation if Strata caches prefixes; `ctl bench` with its metrics path. +## O. More than one place answering clients (written 2026-10-30) + +Every client request goes through the leader's router today: the floating address is on the leader, and a standby +passes `/v1` on to it (a standby's router sees an empty cluster). That is failover, not scale: the cluster serves what +one control plane's router can. In order: + +- O1. Measure the ceiling first: one router in front of a fast fake engine (`sim-engine`, through an external model), + streams at 1 to 256 at once from another machine, the leader's CPU beside it. Is it the router, the guardrails, the + usage writes or TLS? Written down with the host's load (pve1 is oversubscribed). +- O2. Every control plane answers `/v1` itself (active-active): + - the leader's replica table followed live by the standbys, so their routers route; + - rate limits and budgets kept cluster-wide, not per process (J6), or three routers allow three times as much; + - a session's requests to one router and one replica (consistent hashing on the session), so the engines' prefix + caches keep paying; + - in front: IPVS on the keepalived the setup already installs, DNS round-robin, or the customer's balancer (J7, L9). +- O3. Only if O2 is not enough: stateless gateway machines (`llmgw server --role gateway`) in front of the leader's + control plane, as many as the traffic needs. + ## D. Facts worth knowing before starting - Admin API: `127.0.0.1:9090`, behind the token in diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af98d8..c339e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,10 @@ APIs change between minors without a deprecation period. 60 GB transfer at 23 GB (`stream error … CANCEL`), the mirror deleted what had arrived, and `ctl model serve` failed. The mirror now tries again up to six times, carrying on from where it got to (`Range`), keeps what arrived when it gives up, and carries on from it after a restart too. +- **A model from an outside provider took three or four requests at once.** Admission read every request in flight to + an endpoint it does not measure as a queue, so an external model (OpenAI, another llmgw cluster) was held to what a + saturation of 0.85 allows of "in flight / 4"; the rest waited and timed out with 504. An outside provider queues for + itself and says 429 when full, which its breaker already hears: it no longer counts towards saturation. ## v0.16.1 — 2026-10-30 diff --git a/internal/router/admission/admission_test.go b/internal/router/admission/admission_test.go index d1b0cf2..0c0602c 100644 --- a/internal/router/admission/admission_test.go +++ b/internal/router/admission/admission_test.go @@ -543,3 +543,20 @@ func TestABandForBulkWorkWaitsLongerThanTheRest(t *testing.T) { t.Errorf("bronze gave up after %v, before its own queue_ttl", d) } } + +// A provider outside the cluster queues for itself: requests in flight to it +// are not a queue here. Counted as one, an external model took three or four +// at once and the rest timed out waiting. +func TestRequestsInFlightToAnExternalProviderAreNotAQueue(t *testing.T) { + th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} + ext := snap("openai", 0, 0, 200, 0, false) + ext.Engine = endpoint.EngineExternal + if got := Saturation([]endpoint.Snapshot{ext}, 0, th); got >= 0.5 { + t.Errorf("200 in flight to an external provider read as saturation %v", got) + } + // An engine of the cluster with the same in flight and nothing learned + // of its capacity still reads as loaded. + if got := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 200, 0, false)}, 0, th); got < 1 { + t.Errorf("an engine's unreported burst ignored: %v", got) + } +} diff --git a/internal/router/admission/saturation.go b/internal/router/admission/saturation.go index 41e4335..bc9a5ce 100644 --- a/internal/router/admission/saturation.go +++ b/internal/router/admission/saturation.go @@ -53,6 +53,14 @@ func Saturation(eps []endpoint.Snapshot, unreported int, t Thresholds) float64 { if e.Breaker.String() == "open" { continue } + // A provider outside the cluster queues for itself and says 429 when + // full, which its breaker hears: nothing here measures its queue, and + // counting every request in flight to it as waiting held an external + // model to three or four at once, the rest timing out in the queue. + if e.Engine == endpoint.EngineExternal { + n++ + continue + } // The engine's own queue is the truth, but it is one scrape interval // old. Fill that gap with what the router knows without any lag: how