From 40c884a50a14718df3cd1e0001eeba2bf8ea03e9 Mon Sep 17 00:00:00 2001 From: ilker Date: Wed, 16 Sep 2026 01:29:41 +0200 Subject: [PATCH] Add a changelog and map releases onto roadmap phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Versions track the phases in the plan: a phase is released once its exit criterion is met, not when its code lands. That keeps "released" meaning something — v0.1.0 ships sim-engine and the routing libraries, and says plainly that the gateway does not serve traffic yet. scripts/gitea/milestones.py creates the matching Gitea milestones. Idempotent and never destructive, so it can be re-run after the roadmap shifts. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 67 +++++++++++++++ README.md | 23 +++-- scripts/gitea/milestones.py | 166 ++++++++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 9 deletions(-) create mode 100644 CHANGELOG.md create mode 100755 scripts/gitea/milestones.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..81c4e05 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,67 @@ +# Changelog + +Versions map onto the roadmap phases in [llmgw-plan.md](llmgw-plan.md) §11. A phase +is only released once its **exit criterion** is met, not when its code lands. + +| Version | Phase | Scope | State | +|---|---|---|---| +| v0.1.0 | 0 | Skeleton and sim-engine | released | +| v0.2.0 | 1 | Static router: streaming proxy, filter/score/pick | in progress | +| v0.3.0 | 1.5 | Admission control: priority bands, saturation detector | planned | +| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | planned | +| v0.5.0 | 3 | Control plane: catalog, placement, reconciler | planned | +| v0.6.0 | 4 | Multi-node replicas and fabric awareness | planned | +| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned | + +Until v1.0.0 the minor version is the breaking one: configuration formats and Go +APIs change between minors without a deprecation period. + +## v0.1.0 — 2026-09-16 + +First tagged version. It is useful for exactly one thing: developing and testing +a router without GPUs. **The gateway does not serve traffic yet** — `llmgw server` +returns "not implemented". + +### Added + +**sim-engine** (`llmgw sim-engine`) — an OpenAI-compatible fake inference engine +that models the behaviours a router has to cope with: + +- continuous batching with a bounded concurrent sequence count +- KV cache block accounting, queueing when blocks run out, including + head-of-line blocking +- automatic prefix caching (chained block hashes over a block-level LRU) and its + effect on time to first token +- per-token latency that grows with batch size +- dropping a request and freeing its blocks when the client disconnects +- `/metrics` under vLLM's real metric names +- failure injection (`--fail-rate`) and deterministic runs (`--seed`) + +**Routing primitives** — libraries, not yet wired into a server: + +- `router/scheduler`: the filter → score → pick pipeline. Scorers for queue + depth, KV occupancy, router-side inflight, drain rate, capacity weight and + session affinity (bounded-load consistent hashing). Pickers: max-score, p2c, + softmax, round-robin. Built-in profiles: chat, batch, long-context, debug. +- `router/endpoint`: live endpoint registry with inflight counting, staleness + detection and drain rate derived from engine counters. +- `router/scrape`: parses the engine metric contract (plan §7.6) and refuses + endpoints missing any of the mandatory three. The metric name mapping is + overridable, because names drift between engine versions. +- `router/breaker`: cooldown and circuit breaking in one component, with + exponential backoff and half-open probes. + +### Known limitations + +- `llmgw server`, `llmgw agent` and `llmgw ctl` are stubs. +- The scheduler has no HTTP layer yet; nothing routes real requests. +- No admission control, so a batch workload can still starve interactive traffic + once the router exists. That is what v0.3.0 is for. +- Prefix awareness is tier 0 only (session affinity). Tiers 1 and 2 are later. + +### Notes + +- The project language is English throughout: code, comments, tests, commits and + the design document. +- Plan §7.6 corrects an error from plan v0.1: the KV usage metric is + `vllm:kv_cache_usage_perc`, not `vllm:gpu_cache_usage_perc`. diff --git a/README.md b/README.md index 030cac5..b233102 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,20 @@ OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md). ## Status -| Phase | Scope | State | -|---|---|---| -| 0 | Skeleton and sim-engine | done | -| 1 | Static router (filter/score/pick, streaming proxy) | in progress | -| 1.5 | Admission control (priority bands) | todo | -| 2 | Agent: GPU/IB discovery, single-node engines | todo | -| 3 | Control plane: catalog, scheduler, reconciler | todo | -| 4 | Multi-node placement and fabric awareness | todo | -| 5 | Tenancy, prefix affinity, benchmarking | todo | +Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md). + +| Version | Phase | Scope | State | +|---|---|---|---| +| v0.1.0 | 0 | Skeleton and sim-engine | released | +| v0.2.0 | 1 | Static router (filter/score/pick, streaming proxy) | in progress | +| v0.3.0 | 1.5 | Admission control (priority bands) | planned | +| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | planned | +| v0.5.0 | 3 | Control plane: catalog, scheduler, reconciler | planned | +| v0.6.0 | 4 | Multi-node placement and fabric awareness | planned | +| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned | + +**v0.1.0 does not serve traffic**: `llmgw server` is still a stub. What works +today is `sim-engine` plus the routing libraries it exists to test. ## Building diff --git a/scripts/gitea/milestones.py b/scripts/gitea/milestones.py new file mode 100755 index 0000000..7a75141 --- /dev/null +++ b/scripts/gitea/milestones.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Create the roadmap milestones in Gitea. + +Milestones mirror the phases in llmgw-plan.md §11 and the versions in +CHANGELOG.md, so an issue can be filed against the phase that owns it. + +Idempotent: a milestone whose title already exists is left untouched, including +its state and due date. Nothing is ever deleted. + +Usage: + GITEA_TOKEN= scripts/gitea/milestones.py [--repo ilker/llmgw] [--dry-run] + +The token needs write access to the repository. Create one under +Settings → Applications → Access Tokens with the `write:issue` scope. +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request + +DEFAULT_INSTANCE = "https://gitea.mynodes.xyz" +DEFAULT_REPO = "ilker/llmgw" + +# Due dates are derived from the durations in plan §11 laid end to end from +# 2026-09-16. They are estimates, not commitments; adjust them in Gitea rather +# than here once reality disagrees. +MILESTONES = [ + { + "title": "v0.1.0 — Phase 0: skeleton and sim-engine", + "description": ( + "CLI skeleton plus the fake inference engine everything else is tested " + "against. Exit criterion: sim-engine models batching, KV blocks, prefix " + "caching and cancellation, and exposes the §7.6 metric contract." + ), + "due_on": "2026-09-16T23:59:59Z", + "closed": True, + }, + { + "title": "v0.2.0 — Phase 1: static router", + "description": ( + "OpenAI handlers, streaming proxy with cancellation propagation, a static " + "endpoint list, the filter/score/pick pipeline and metric scraping.\n\n" + "Exit criterion: 1000 concurrent streams behind 3 sim-engines; upstream " + "cancellation on client disconnect verified; router overhead p99 < 2 ms." + ), + "due_on": "2026-09-30T23:59:59Z", + }, + { + "title": "v0.3.0 — Phase 1.5: admission control", + "description": ( + "Saturation detector, priority bands, round-robin tenant fairness, queue " + "capacity and TTL.\n\n" + "Exit criterion: while 500 concurrent batch requests flow through the " + "bronze band, interactive TTFT p99 in the platinum band stays under 1.5x " + "its idle value. This test stays in CI permanently." + ), + "due_on": "2026-10-07T23:59:59Z", + }, + { + "title": "v0.4.0 — Phase 2: agent", + "description": ( + "GPU and IB discovery, the engine interface, the process supervisor and " + "the control plane stream.\n\n" + "Exit criterion: start and stop a model on a real node via ctl." + ), + "due_on": "2026-10-21T23:59:59Z", + }, + { + "title": "v0.5.0 — Phase 3: control plane", + "description": ( + "SQLite store, catalog, inventory, single-node placement, reconciler, and " + "a dynamic EndpointSource for the router.\n\n" + "Exit criterion: shutting down a node moves its replicas elsewhere while " + "the router keeps serving." + ), + "due_on": "2026-11-11T23:59:59Z", + }, + { + "title": "v0.6.0 — Phase 4: multi-node and fabrics", + "description": ( + "Fabric identity, multi-node placement under the fabric constraint, NCCL " + "environment generation, replica-level failure handling.\n\n" + "Exit criterion: a 2-node replica on each fabric; killing a node rebuilds " + "the replica on the same fabric and never spreads it across two." + ), + "due_on": "2026-11-25T23:59:59Z", + }, + { + "title": "v1.0.0 — Phase 5: production features", + "description": ( + "Tenancy and budgets, routing profiles from configuration, the approximate " + "prefix tree, fallback models and pool spillover, ctl bench, tracing, " + "systemd units and documentation." + ), + "due_on": "2026-12-16T23:59:59Z", + }, + { + "title": "Phase 6 — v2 ideas", + "description": ( + "No due date; a parking lot. Precise KV index, admission displacement, " + "autoscaling with preemption, control plane HA, prefill/decode " + "disaggregation, LoRA routing, external providers, admin UI." + ), + }, +] + + +def request(method, url, token, payload=None): + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"token {token}") + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = resp.read() + return json.loads(body) if body else None + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace")[:300] + raise SystemExit(f"{method} {url} failed: {e.code} {detail}") from e + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--instance", default=os.environ.get("GITEA_INSTANCE", DEFAULT_INSTANCE)) + ap.add_argument("--repo", default=os.environ.get("GITEA_REPO", DEFAULT_REPO)) + ap.add_argument("--dry-run", action="store_true", help="show what would be created") + args = ap.parse_args() + + token = os.environ.get("GITEA_TOKEN", "") + if not token and not args.dry_run: + sys.exit("GITEA_TOKEN is not set (use --dry-run to preview)") + + base = f"{args.instance.rstrip('/')}/api/v1/repos/{args.repo}/milestones" + + existing = {} + if not args.dry_run: + for m in request("GET", f"{base}?state=all&limit=100", token) or []: + existing[m["title"]] = m + + for spec in MILESTONES: + title = spec["title"] + if title in existing: + print(f"skip {title} (already exists, id={existing[title]['id']})") + continue + if args.dry_run: + print(f"create {title}") + continue + + payload = {"title": title, "description": spec.get("description", "")} + if "due_on" in spec: + payload["due_on"] = spec["due_on"] + + created = request("POST", base, token, payload) + print(f"create {title} (id={created['id']})") + + # The create endpoint ignores `state`, so close it separately. + if spec.get("closed"): + request("PATCH", f"{base}/{created['id']}", token, {"state": "closed"}) + print(f"close {title}") + + +if __name__ == "__main__": + main()