Merge C21 quality suites and gating
Conflicts: docs.go (manual, OpenAPI and suites all kept), modelserve Job (trace and eval both kept). The eval routes are added to the OpenAPI route table, update and promote documented with their optional body, and the document regenerated. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
d41104a452
51 files changed
+5204
-49
No files matched your search
@@ -26,6 +26,9 @@
|
||||
{
|
||||
"name": "capacity"
|
||||
},
|
||||
{
|
||||
"name": "eval"
|
||||
},
|
||||
{
|
||||
"name": "explain-placement"
|
||||
},
|
||||
@@ -212,6 +215,122 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/eval/runs": {
|
||||
"get": {
|
||||
"operationId": "getEvalRuns",
|
||||
"summary": "Quality runs, the newest first",
|
||||
"tags": [
|
||||
"eval"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "model",
|
||||
"in": "query",
|
||||
"description": "Only this model's runs.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"in": "query",
|
||||
"description": "How many; a sensible number when absent.",
|
||||
"schema": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/eval.Live"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/eval/runs/{id}": {
|
||||
"get": {
|
||||
"operationId": "getEvalRunsId",
|
||||
"summary": "One quality run, with each case",
|
||||
"tags": [
|
||||
"eval"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The quality run's ID, as starting it answered.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/eval.Live"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/eval/suites": {
|
||||
"get": {
|
||||
"operationId": "getEvalSuites",
|
||||
"summary": "The quality suites the control plane can run by name",
|
||||
"tags": [
|
||||
"eval"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/eval.SuiteInfo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/explain-placement": {
|
||||
"get": {
|
||||
"operationId": "getExplainPlacement",
|
||||
@@ -1063,6 +1182,148 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/eval": {
|
||||
"post": {
|
||||
"operationId": "postModelsNameEval",
|
||||
"summary": "Ask a model a quality suite",
|
||||
"description": "Started at once and answered with the run, to be followed at /api/v1/eval/runs/{id}.",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The model's name in the catalog.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/eval.Request"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "Accepted: started, and running on.",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/eval.Live"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/eval-gate": {
|
||||
"put": {
|
||||
"operationId": "putModelsNameEvalGate",
|
||||
"summary": "Hold a model's updates and promotions to a quality suite",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The model's name in the catalog.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/EvalGateRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/EvalGateView"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"operationId": "deleteModelsNameEvalGate",
|
||||
"summary": "Stop holding a model's updates to a suite",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The model's name in the catalog.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/EvalGateView"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/headroom": {
|
||||
"get": {
|
||||
"operationId": "getModelsNameHeadroom",
|
||||
@@ -1215,6 +1476,7 @@
|
||||
"post": {
|
||||
"operationId": "postModelsNamePromote",
|
||||
"summary": "Make a canary's weights the model's",
|
||||
"description": "With a quality suite, or the model's standing gate, the canary is asked it first and kept below the score.",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
@@ -1232,6 +1494,16 @@
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": false,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/modelserve.Change"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "Accepted: started, and running on.",
|
||||
@@ -1487,6 +1759,7 @@
|
||||
"post": {
|
||||
"operationId": "postModelsNameUpdate",
|
||||
"summary": "Move a model to its repository's newer weights",
|
||||
"description": "With a quality suite, or the model's standing gate, the new weights are asked it first as a hidden canary.",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
@@ -1504,6 +1777,16 @@
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": false,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/modelserve.Change"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "Accepted: started, and running on.",
|
||||
@@ -2796,6 +3079,33 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"EvalGateRequest": {
|
||||
"type": "object",
|
||||
"description": "EvalGateRequest sets a model's standing quality suite; an empty suite takes it away.",
|
||||
"properties": {
|
||||
"suite": {
|
||||
"type": "string"
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number"
|
||||
},
|
||||
"judge": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"EvalGateView": {
|
||||
"type": "object",
|
||||
"description": "EvalGateView is a model's standing quality gate, as it is now; Eval is absent when it has none.",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"eval": {
|
||||
"$ref": "#/components/schemas/catalog.EvalGate"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ExplainView": {
|
||||
"type": "object",
|
||||
"description": "ExplainView is why a model could not be placed.",
|
||||
@@ -3371,6 +3681,10 @@
|
||||
"canary_of": {
|
||||
"type": "string"
|
||||
},
|
||||
"eval": {
|
||||
"$ref": "#/components/schemas/catalog.EvalGate",
|
||||
"description": "Eval is the quality suite its new weights must pass, when it has one."
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"description": "Description is what the model is for, when somebody said."
|
||||
@@ -4663,6 +4977,24 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"catalog.EvalGate": {
|
||||
"type": "object",
|
||||
"description": "EvalGate is a quality suite a model's new weights must pass.",
|
||||
"properties": {
|
||||
"suite": {
|
||||
"type": "string",
|
||||
"description": "Suite is a suite's name in control.eval_dir, or one the binary carries (arithmetic, json-format)."
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number",
|
||||
"description": "MinScore is the score, 0 to 1, to pass at; zero is the suite's own threshold."
|
||||
},
|
||||
"judge": {
|
||||
"type": "string",
|
||||
"description": "Judge grades the suite's rubric cases, instead of the judge it names."
|
||||
}
|
||||
}
|
||||
},
|
||||
"catalog.External": {
|
||||
"type": "object",
|
||||
"description": "External is a provider outside the cluster that serves a model.",
|
||||
@@ -4757,6 +5089,10 @@
|
||||
"$ref": "#/components/schemas/catalog.Canary",
|
||||
"description": "Canary sends a share of this model's requests to another catalog model — the same model at new weights — to compare the two before moving to it (`llmgw ctl model canary`)."
|
||||
},
|
||||
"eval": {
|
||||
"$ref": "#/components/schemas/catalog.EvalGate",
|
||||
"description": "Eval is a quality suite new weights must pass before they replace this model's: `llmgw ctl model update` and `model promote` run it against them first, and stop below its score."
|
||||
},
|
||||
"routing_profile": {
|
||||
"type": "string",
|
||||
"description": "RoutingProfile selects how the router scores endpoints for this model."
|
||||
@@ -4992,6 +5328,149 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"eval.Live": {
|
||||
"type": "object",
|
||||
"description": "Live is a run as it stands: one running, with how many of its cases are done, or one finished.",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "Model is the model the run is about; Target the catalog model that answered, when that was another — its canary, or the new weights an update tried before moving to them."
|
||||
},
|
||||
"target": {
|
||||
"type": "string"
|
||||
},
|
||||
"repo": {
|
||||
"type": "string",
|
||||
"description": "Repo and Revision are where the weights that answered came from, when that is known."
|
||||
},
|
||||
"revision": {
|
||||
"type": "string"
|
||||
},
|
||||
"suite": {
|
||||
"type": "string",
|
||||
"description": "Suite is its name, SuiteHash the sha256 of its text."
|
||||
},
|
||||
"suite_hash": {
|
||||
"type": "string"
|
||||
},
|
||||
"at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"took_ms": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"score": {
|
||||
"type": "number",
|
||||
"description": "Score is the cases' scores weighed, 0 to 1; Passed whether it reached Threshold."
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number"
|
||||
},
|
||||
"passed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cases": {
|
||||
"type": "integer",
|
||||
"description": "Cases were asked; Below scored under half; Errors got no answer."
|
||||
},
|
||||
"below": {
|
||||
"type": "integer"
|
||||
},
|
||||
"errors": {
|
||||
"type": "integer"
|
||||
},
|
||||
"judge": {
|
||||
"type": "string"
|
||||
},
|
||||
"by": {
|
||||
"type": "string"
|
||||
},
|
||||
"job": {
|
||||
"type": "string",
|
||||
"description": "Job is the update or promotion the run gated, if one did."
|
||||
},
|
||||
"error": {
|
||||
"type": "string",
|
||||
"description": "Error is why the run could not be scored at all."
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/store.EvalCase"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"type": "string"
|
||||
},
|
||||
"done": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"eval.Request": {
|
||||
"type": "object",
|
||||
"description": "Request is a suite to run against a model: one on the control plane, by its name, or one sent whole.",
|
||||
"properties": {
|
||||
"suite": {
|
||||
"type": "string",
|
||||
"description": "Suite is a suite's name in the eval directory, or carried in the binary (arithmetic, json-format)."
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "Content is a suite's text instead, and File the name of the file it was read from, which names the suite unless it names itself."
|
||||
},
|
||||
"file": {
|
||||
"type": "string"
|
||||
},
|
||||
"concurrency": {
|
||||
"type": "integer",
|
||||
"description": "Concurrency, Judge and MinScore are as Options has them."
|
||||
},
|
||||
"judge": {
|
||||
"type": "string"
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"eval.SuiteInfo": {
|
||||
"type": "object",
|
||||
"description": "SuiteInfo is a suite the control plane can run by name.",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"from": {
|
||||
"type": "string",
|
||||
"description": "From is the file it is read from, or \"built in\"."
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"cases": {
|
||||
"type": "integer"
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number"
|
||||
},
|
||||
"judge": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"hash": {
|
||||
"type": "string"
|
||||
},
|
||||
"error": {
|
||||
"type": "string",
|
||||
"description": "Error is why the file cannot be run."
|
||||
}
|
||||
}
|
||||
},
|
||||
"fabrics.Change": {
|
||||
"type": "object",
|
||||
"description": "Change is a fabric's master subnet manager changing, as its ports saw it.",
|
||||
@@ -5367,6 +5846,20 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.Change": {
|
||||
"type": "object",
|
||||
"description": "Change is how an update or a promotion is held to quality.",
|
||||
"properties": {
|
||||
"eval": {
|
||||
"$ref": "#/components/schemas/eval.Request",
|
||||
"description": "Eval is a suite the new weights must pass first. Nil takes the model's own (eval: in its catalog entry), if it has one."
|
||||
},
|
||||
"skip_eval": {
|
||||
"type": "boolean",
|
||||
"description": "SkipEval moves without the model's own suite; the job says so."
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelserve.Job": {
|
||||
"type": "object",
|
||||
"description": "Job is one model being served, and where it stands.",
|
||||
@@ -5406,6 +5899,10 @@
|
||||
},
|
||||
"tokens_per_second": {
|
||||
"type": "number"
|
||||
},
|
||||
"eval": {
|
||||
"$ref": "#/components/schemas/store.EvalRun",
|
||||
"description": "Eval is the quality suite the new weights were asked before they replaced the old, and how they scored, without each case."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5847,6 +6344,118 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"store.EvalCase": {
|
||||
"type": "object",
|
||||
"description": "EvalCase is one case of a run: how its answer scored, and why.",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string"
|
||||
},
|
||||
"weight": {
|
||||
"type": "number"
|
||||
},
|
||||
"score": {
|
||||
"type": "number"
|
||||
},
|
||||
"passed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"answer": {
|
||||
"type": "string"
|
||||
},
|
||||
"detail": {
|
||||
"type": "string"
|
||||
},
|
||||
"error": {
|
||||
"type": "string"
|
||||
},
|
||||
"took_ms": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
}
|
||||
},
|
||||
"store.EvalRun": {
|
||||
"type": "object",
|
||||
"description": "EvalRun is one quality suite run against a model, as `llmgw ctl eval` and the gate on an update keep it.",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "Model is the model the run is about; Target the catalog model that answered, when that was another — its canary, or the new weights an update tried before moving to them."
|
||||
},
|
||||
"target": {
|
||||
"type": "string"
|
||||
},
|
||||
"repo": {
|
||||
"type": "string",
|
||||
"description": "Repo and Revision are where the weights that answered came from, when that is known."
|
||||
},
|
||||
"revision": {
|
||||
"type": "string"
|
||||
},
|
||||
"suite": {
|
||||
"type": "string",
|
||||
"description": "Suite is its name, SuiteHash the sha256 of its text."
|
||||
},
|
||||
"suite_hash": {
|
||||
"type": "string"
|
||||
},
|
||||
"at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"took_ms": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"score": {
|
||||
"type": "number",
|
||||
"description": "Score is the cases' scores weighed, 0 to 1; Passed whether it reached Threshold."
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number"
|
||||
},
|
||||
"passed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cases": {
|
||||
"type": "integer",
|
||||
"description": "Cases were asked; Below scored under half; Errors got no answer."
|
||||
},
|
||||
"below": {
|
||||
"type": "integer"
|
||||
},
|
||||
"errors": {
|
||||
"type": "integer"
|
||||
},
|
||||
"judge": {
|
||||
"type": "string"
|
||||
},
|
||||
"by": {
|
||||
"type": "string"
|
||||
},
|
||||
"job": {
|
||||
"type": "string",
|
||||
"description": "Job is the update or promotion the run gated, if one did."
|
||||
},
|
||||
"error": {
|
||||
"type": "string",
|
||||
"description": "Error is why the run could not be scored at all."
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/store.EvalCase"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"store.ExportRow": {
|
||||
"type": "object",
|
||||
"description": "ExportRow is one line of a usage export: a period, the place, person, key and model it is about — those asked for, the rest empty — and what was used.",
|
||||
|
||||
Reference in new issue
Block a user