Keep runtime models, so an added model outlives a restart
The catalog can change while the cluster runs; this is where a change goes so it is still there tomorrow. Models declared in configuration and models added at runtime are two sources on purpose: config-as-code stays declarative and under review, and the database holds what someone added to try it. Configuration wins every conflict, and the direction matters. A request for a reviewed production model must never be answered by something added from a laptop, so a stored model colliding with a declared name is refused when it is added. A declared model cannot be removed at runtime either — it would return on the next restart, leaving the cluster disagreeing with its own config file until then, which is a worse answer than refusing and saying where to edit. The whole catalog is built before anything is written. A model naming a missing fallback, or colliding, or malformed in any of the ways catalog.New refuses, is rejected rather than persisted and then skipped forever after — a model that looks added and never appears is the worst outcome available. Removal is checked the same way, because another model may fall back to the one being removed, and a fallback pointing at nothing is discovered during the outage it was meant to cover. At startup each stored model is validated on its own against what is already accepted, so one bad row is skipped and named rather than taking the cluster with it. The declared models are precisely the ones that must survive that. Catalog types now carry json tags mirroring their koanf ones, so a stored definition is the same shape as the YAML someone writes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1c57a4e378
commit
277add5ccb
5 files changed
+537
-30
No files matched your search
@@ -14,8 +14,8 @@ import (
|
||||
|
||||
// Scaling is how many replicas a model should have.
|
||||
type Scaling struct {
|
||||
Min int `koanf:"min"`
|
||||
Max int `koanf:"max"`
|
||||
Min int `json:"min,omitempty" koanf:"min"`
|
||||
Max int `json:"max,omitempty" koanf:"max"`
|
||||
}
|
||||
|
||||
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
||||
@@ -27,19 +27,19 @@ type SLO struct {
|
||||
|
||||
// Profile is how a model runs on one GPU class.
|
||||
type Profile struct {
|
||||
Name string `koanf:"name"`
|
||||
Engine string `koanf:"engine"`
|
||||
GPUClass string `koanf:"gpu_class"`
|
||||
Name string `json:"name,omitempty" koanf:"name"`
|
||||
Engine string `json:"engine,omitempty" koanf:"engine"`
|
||||
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
||||
|
||||
TP int `koanf:"tp"`
|
||||
PP int `koanf:"pp"`
|
||||
TP int `json:"tp,omitempty" koanf:"tp"`
|
||||
PP int `json:"pp,omitempty" koanf:"pp"`
|
||||
// GPUsPerNode defaults to TP for a single-node profile.
|
||||
GPUsPerNode int `koanf:"gpus_per_node"`
|
||||
GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
|
||||
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
|
||||
Nodes int `koanf:"nodes"`
|
||||
Nodes int `json:"nodes,omitempty" koanf:"nodes"`
|
||||
|
||||
Args []string `koanf:"args"`
|
||||
Env map[string]string `koanf:"env"`
|
||||
Args []string `json:"args,omitempty" koanf:"args"`
|
||||
Env map[string]string `json:"env,omitempty" koanf:"env"`
|
||||
|
||||
// MinMemoryGB is the host RAM this profile needs to run.
|
||||
//
|
||||
@@ -53,7 +53,7 @@ type Profile struct {
|
||||
// this. A node that does not report its memory is not excluded either —
|
||||
// an older agent must not have its nodes quietly emptied by an upgrade of
|
||||
// the control plane.
|
||||
MinMemoryGB int `koanf:"min_memory_gb"`
|
||||
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
||||
|
||||
// NodeSelector restricts this profile to nodes carrying every one of these
|
||||
// labels, with these values.
|
||||
@@ -63,16 +63,16 @@ type Profile struct {
|
||||
// in the inventory says how much memory a node has, so the operator says
|
||||
// which nodes are meant for a model instead of the scheduler inferring it.
|
||||
// Agents have reported labels since the beginning; this is what reads them.
|
||||
NodeSelector map[string]string `koanf:"node_selector"`
|
||||
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
|
||||
|
||||
// Weight is the router's capacity weight. It should come from ctl bench
|
||||
// rather than a guess (plan §6.2.1).
|
||||
Weight float64 `koanf:"weight"`
|
||||
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
||||
// Priority orders profiles during placement; highest that fits wins.
|
||||
Priority int `koanf:"priority"`
|
||||
Priority int `json:"priority,omitempty" koanf:"priority"`
|
||||
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
||||
// tens of minutes, so this is per profile rather than global.
|
||||
StartupTimeout time.Duration `koanf:"startup_timeout"`
|
||||
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs.
|
||||
@@ -91,16 +91,16 @@ func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
||||
|
||||
// Model is a catalog entry as clients see it.
|
||||
type Model struct {
|
||||
Name string `koanf:"name"`
|
||||
Aliases []string `koanf:"aliases"`
|
||||
Name string `json:"name,omitempty" koanf:"name"`
|
||||
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
||||
// Source is the weights path or a HuggingFace id.
|
||||
Source string `koanf:"source"`
|
||||
Profiles []Profile `koanf:"profiles"`
|
||||
Scaling Scaling `koanf:"scaling"`
|
||||
Fallbacks []string `koanf:"fallbacks"`
|
||||
Source string `json:"source,omitempty" koanf:"source"`
|
||||
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
||||
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// RoutingProfile selects how the router scores endpoints for this model.
|
||||
RoutingProfile string `koanf:"routing_profile"`
|
||||
SLO SLO `koanf:"slo"`
|
||||
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
||||
SLO SLO `json:"slo,omitempty" koanf:"slo"`
|
||||
|
||||
// ContextLength is how many tokens one request may use, prompt and
|
||||
// completion together, and MaxOutput bounds the completion alone. Zero
|
||||
@@ -112,11 +112,11 @@ type Model struct {
|
||||
// the one an operator writes in the engine's arguments — llama.cpp divides
|
||||
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
|
||||
// is 2048 a request, and nothing said so.
|
||||
ContextLength int `koanf:"context_length"`
|
||||
MaxOutput int `koanf:"max_output"`
|
||||
ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
|
||||
MaxOutput int `json:"max_output,omitempty" koanf:"max_output"`
|
||||
|
||||
// Capabilities is what this model can do beyond plain completion.
|
||||
Capabilities Capabilities `koanf:"capabilities"`
|
||||
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
|
||||
}
|
||||
|
||||
// Capabilities describes what a client may ask a model for.
|
||||
@@ -131,9 +131,9 @@ type Model struct {
|
||||
// then behaves as though the model is broken, and the only way to find out was
|
||||
// to try it. That cost an afternoon.
|
||||
type Capabilities struct {
|
||||
Tools bool `koanf:"tools"`
|
||||
Vision bool `koanf:"vision"`
|
||||
Embeddings bool `koanf:"embeddings"`
|
||||
Tools bool `json:"tools,omitempty" koanf:"tools"`
|
||||
Vision bool `json:"vision,omitempty" koanf:"vision"`
|
||||
Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
|
||||
}
|
||||
|
||||
// Catalog is the set of models the cluster should serve.
|
||||
|
||||
Reference in new issue
Block a user