The cluster's settings live in the store, and every change takes effect
Everything in server.yaml that is not one machine's own is now a versioned document in the store (cluster_settings), the same on every control plane. The first start against a store imports the file's cluster keys, with the baseline guard policy when there is none; after that those keys in a file are ignored, with a warning. Each control plane polls the document's version and its own server.yaml's hash. A change is validated and applied: guard live (Gate.Replace), anything else by a graceful restart that systemd brings back, standbys first and the leader 30s later. A version that does not load on a machine is not applied; at start the newest one that loads is used, so a bad edit cannot keep every control plane down. llmgw ctl config show/get/set/unset/edit/history/rollback, the admin API under /api/v1/settings, and the console's Settings screen, which layers the store's document over the machine's file, write it. Commands beside the control plane that read cluster keys (init, node add, server add, backup, upgrade) read the layered configuration; init --bundle writes offline to the store once it holds the settings. A standby that becomes the leader re-reads models, engine choices and facility holds from the store. Retires appendGuardIfMissing: the import does it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbbfce235e
commit
6289b01ed6
57 files changed
+2579
-224
No files matched your search
@@ -67,7 +67,8 @@ type API struct {
|
||||
// nil without a store.
|
||||
federation *federation.Service
|
||||
// guard is the guardrail policies' library and attachments.
|
||||
guard *GuardAdmin
|
||||
guard *GuardAdmin
|
||||
settings *Settings
|
||||
// revoke records a removed node, so its certificate is refused.
|
||||
revoke func(ctx context.Context, nodeID string, at time.Time) error
|
||||
|
||||
@@ -111,6 +112,7 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
a.registerGuard(mux)
|
||||
a.registerSettings(mux)
|
||||
mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
|
||||
mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
|
||||
mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
|
||||
|
||||
@@ -21,10 +21,11 @@ type GuardStore interface {
|
||||
DetachGuard(ctx context.Context, a guard.Attachment) (bool, error)
|
||||
}
|
||||
|
||||
// GuardAdmin is what the guard endpoints need: the library from server.yaml,
|
||||
// the store, and a way to have the gate read attachments again at once.
|
||||
// GuardAdmin is what the guard endpoints need: the library in force (the
|
||||
// cluster's settings), the store, and a way to have the gate read
|
||||
// attachments again at once.
|
||||
type GuardAdmin struct {
|
||||
Library guard.Config
|
||||
Library func() guard.Config
|
||||
Store GuardStore
|
||||
Invalidate func()
|
||||
}
|
||||
@@ -199,7 +200,7 @@ type GuardAttachmentRequest struct {
|
||||
func (a *API) registerGuard(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard: in server.yaml")
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
att, err := a.guard.Store.GuardAttachments(r.Context())
|
||||
@@ -207,7 +208,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
v := GuardView{Default: a.guard.Library.Default, Attachments: []GuardAttachmentView{}}
|
||||
v := GuardView{Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
|
||||
var names map[string]string
|
||||
if a.quotas != nil {
|
||||
names, _ = a.scopeNames(r.Context())
|
||||
@@ -215,12 +216,12 @@ func (a *API) registerGuard(mux Mux) {
|
||||
for _, at := range att {
|
||||
v.Attachments = append(v.Attachments, GuardAttachmentView{Attachment: at, Name: names[at.Scope+":"+at.ScopeID]})
|
||||
}
|
||||
for name := range a.guard.Library.Policies {
|
||||
for name := range a.guard.Library().Policies {
|
||||
v.Policies = append(v.Policies, name)
|
||||
}
|
||||
slices.Sort(v.Policies)
|
||||
for _, name := range v.Policies {
|
||||
v.Details = append(v.Details, policyView(name, a.guard.Library.Policies[name]))
|
||||
v.Details = append(v.Details, policyView(name, a.guard.Library().Policies[name]))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
@@ -275,7 +276,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
change := func(attach bool) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard: in server.yaml")
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
var at GuardAttachmentRequest
|
||||
@@ -289,8 +290,8 @@ func (a *API) registerGuard(mux Mux) {
|
||||
return
|
||||
}
|
||||
att.Policy = at.Policy
|
||||
if _, ok := a.guard.Library.Policies[att.Policy]; !ok && attach {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("no policy %q in server.yaml's guard.policies", att.Policy))
|
||||
if _, ok := a.guard.Library().Policies[att.Policy]; !ok && attach {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("no policy %q in the cluster's guard.policies", att.Policy))
|
||||
return
|
||||
}
|
||||
changed := true
|
||||
|
||||
@@ -36,8 +36,10 @@ func TestGuardPoliciesAreAttachedByTarget(t *testing.T) {
|
||||
st := &memGuard{}
|
||||
invalidated := 0
|
||||
a.SetGuard(&GuardAdmin{
|
||||
Library: guard.Config{Default: []string{"base"}, Policies: map[string]guard.Policy{"base": {}, "finance": {}}},
|
||||
Store: st, Invalidate: func() { invalidated++ },
|
||||
Library: func() guard.Config {
|
||||
return guard.Config{Default: []string{"base"}, Policies: map[string]guard.Policy{"base": {}, "finance": {}}}
|
||||
},
|
||||
Store: st, Invalidate: func() { invalidated++ },
|
||||
})
|
||||
if code := call(t, a, "POST", "/api/v1/guard/attachments", `{"target":"key:k1","policy":"finance"}`, nil); code != 200 {
|
||||
t.Fatalf("attach: %d", code)
|
||||
@@ -138,10 +140,11 @@ func (m *memGuardHistory) GuardCounts(context.Context, time.Time) ([]store.Guard
|
||||
func TestTheGuardrailsSayWhatEachPolicyDoesAndWhatTheyDid(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
h := &memGuardHistory{}
|
||||
a.SetGuard(&GuardAdmin{Store: h, Library: guard.Config{Policies: map[string]guard.Policy{
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"base": {Fail: "closed", Input: guard.Rules{PII: map[string]string{"email": "redact"}, Injection: "block"},
|
||||
Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block", Categories: []string{"S1", "S9"}}},
|
||||
}}})
|
||||
}}
|
||||
a.SetGuard(&GuardAdmin{Store: h, Library: func() guard.Config { return lib }})
|
||||
|
||||
var v GuardView
|
||||
if code := call(t, a, "GET", "/api/v1/guard", "", &v); code != 200 || len(v.Details) != 1 {
|
||||
|
||||
@@ -12,7 +12,7 @@ func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n }
|
||||
// notifyTest sends a test event to every webhook and says how each answered.
|
||||
func (a *API) notifyTest(w http.ResponseWriter, r *http.Request) {
|
||||
if a.notifier == nil {
|
||||
writeError(w, http.StatusNotFound, "no webhooks: server.yaml has no notify.webhooks")
|
||||
writeError(w, http.StatusNotFound, "no webhooks: the cluster's settings have no notify.webhooks (llmgw ctl config set notify.webhooks ...)")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, a.notifier.Test(r.Context()))
|
||||
|
||||
@@ -131,7 +131,7 @@ var Routes = []Route{
|
||||
Description: "With the defaults the catalog filled in. By its name only, not an alias.",
|
||||
Response: catalog.Model{}},
|
||||
{Method: "DELETE", Path: "/api/v1/models/{name}", Summary: "Remove a model added at runtime",
|
||||
Description: "A model declared in configuration is refused: it would come back on the next restart.",
|
||||
Description: "A model declared in the cluster's settings is refused: it is changed there.",
|
||||
Response: Removed{}},
|
||||
{Method: "POST", Path: "/api/v1/models/{name}/scale", Summary: "Set how many replicas a model runs",
|
||||
Request: ScaleRequest{}, Response: ScaleResponse{}},
|
||||
@@ -259,6 +259,19 @@ var Routes = []Route{
|
||||
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
|
||||
// The cluster's settings: everything in server.yaml that is not one
|
||||
// machine's own, the same on every control plane.
|
||||
{Method: "GET", Path: "/api/v1/settings", Summary: "The cluster's settings in force, or one earlier version",
|
||||
Query: []Param{{Name: "version", Description: "An earlier version; the newest when absent."}},
|
||||
Response: SettingsView{}},
|
||||
{Method: "GET", Path: "/api/v1/settings/history", Summary: "Versions of the cluster's settings, newest first",
|
||||
Query: []Param{{Name: "limit", Description: "How many; 20 when absent."}},
|
||||
Response: []SettingsView{}},
|
||||
{Method: "PUT", Path: "/api/v1/settings", Summary: "Change the cluster's settings: in force on every control plane within seconds",
|
||||
Request: SettingsRequest{}, Response: SettingsView{}},
|
||||
{Method: "POST", Path: "/api/v1/settings/rollback", Summary: "Put an earlier version of the cluster's settings back in force",
|
||||
Request: SettingsRollbackRequest{}, Response: SettingsView{}},
|
||||
|
||||
// Usage, requests and the record of changes.
|
||||
{Method: "GET", Path: "/api/v1/usage", Summary: "What each group of the tenancy tree spent",
|
||||
Query: []Param{
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// SettingsStore keeps the cluster's settings.
|
||||
type SettingsStore interface {
|
||||
ClusterSettings(ctx context.Context) (store.SettingsVersion, error)
|
||||
ClusterSettingsAt(ctx context.Context, version int64) (store.SettingsVersion, error)
|
||||
ClusterSettingsHistory(ctx context.Context, limit int) ([]store.SettingsVersion, error)
|
||||
PutClusterSettings(ctx context.Context, base int64, body, by, note string) (int64, error)
|
||||
}
|
||||
|
||||
// Settings is what the settings endpoints need: where they are kept, and a
|
||||
// check that a document would load — laid over this control plane's own
|
||||
// server.yaml, validated as it would be at start.
|
||||
type Settings struct {
|
||||
Store SettingsStore
|
||||
Check func(body []byte) error
|
||||
// LocalKeys are those each machine's server.yaml keeps, which the
|
||||
// document cannot set.
|
||||
LocalKeys []string
|
||||
}
|
||||
|
||||
// SetSettings enables the settings endpoints.
|
||||
func (a *API) SetSettings(s *Settings) { a.settings = s }
|
||||
|
||||
// SettingsView is one version of the cluster's settings.
|
||||
type SettingsView struct {
|
||||
Version int64 `json:"version"`
|
||||
Body string `json:"body"`
|
||||
Note string `json:"note,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by,omitempty"`
|
||||
LocalKeys []string `json:"local_keys,omitempty"`
|
||||
}
|
||||
|
||||
// SettingsRequest is a new version of the cluster's settings, made from the
|
||||
// version Base, which must still be the newest.
|
||||
type SettingsRequest struct {
|
||||
Base int64 `json:"base"`
|
||||
Body string `json:"body"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// SettingsRollbackRequest puts an earlier version's settings back in force,
|
||||
// as a new version.
|
||||
type SettingsRollbackRequest struct {
|
||||
Version int64 `json:"version"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
func settingsView(v store.SettingsVersion) SettingsView {
|
||||
return SettingsView{Version: v.Version, Body: v.Body, Note: v.Note, At: v.At, By: v.By}
|
||||
}
|
||||
|
||||
func (a *API) registerSettings(mux Mux) {
|
||||
need := func(w http.ResponseWriter) bool {
|
||||
if a.settings == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the cluster's settings in")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
mux.HandleFunc("GET /api/v1/settings", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
var v store.SettingsVersion
|
||||
var err error
|
||||
if q := r.URL.Query().Get("version"); q != "" {
|
||||
n, perr := strconv.ParseInt(q, 10, 64)
|
||||
if perr != nil {
|
||||
writeError(w, http.StatusBadRequest, "version is a number")
|
||||
return
|
||||
}
|
||||
v, err = a.settings.Store.ClusterSettingsAt(r.Context(), n)
|
||||
if err == nil && v.Version == 0 {
|
||||
writeError(w, http.StatusNotFound, "there is no such version")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
v, err = a.settings.Store.ClusterSettings(r.Context())
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := settingsView(v)
|
||||
out.LocalKeys = a.settings.LocalKeys
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/settings/history", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
limit := 20
|
||||
if q := r.URL.Query().Get("limit"); q != "" {
|
||||
if n, err := strconv.Atoi(q); err == nil && n > 0 && n <= 500 {
|
||||
limit = n
|
||||
}
|
||||
}
|
||||
list, err := a.settings.Store.ClusterSettingsHistory(r.Context(), limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]SettingsView, len(list))
|
||||
for i, v := range list {
|
||||
out[i] = settingsView(v)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
})
|
||||
put := func(w http.ResponseWriter, r *http.Request, base int64, body, note string) {
|
||||
if err := a.settings.Check([]byte(body)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
n, err := a.settings.Store.PutClusterSettings(r.Context(), base, body, actorOf(r), note)
|
||||
switch {
|
||||
case errors.Is(err, store.ErrSettingsChanged):
|
||||
writeError(w, http.StatusConflict, err.Error())
|
||||
return
|
||||
case err != nil:
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
v, err := a.settings.Store.ClusterSettingsAt(r.Context(), n)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, settingsView(v))
|
||||
}
|
||||
mux.HandleFunc("PUT /api/v1/settings", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
var req SettingsRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
put(w, r, req.Base, req.Body, req.Note)
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/settings/rollback", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
var req SettingsRollbackRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
old, err := a.settings.Store.ClusterSettingsAt(r.Context(), req.Version)
|
||||
if err == nil && old.Version == 0 {
|
||||
writeError(w, http.StatusNotFound, "there is no such version")
|
||||
return
|
||||
}
|
||||
cur, err2 := a.settings.Store.ClusterSettings(r.Context())
|
||||
if err = errors.Join(err, err2); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
note := req.Note
|
||||
if note == "" {
|
||||
note = "back to version " + strconv.FormatInt(req.Version, 10)
|
||||
}
|
||||
put(w, r, cur.Version, old.Body, note)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type memSettings struct{ versions []store.SettingsVersion }
|
||||
|
||||
func (m *memSettings) ClusterSettings(context.Context) (store.SettingsVersion, error) {
|
||||
if len(m.versions) == 0 {
|
||||
return store.SettingsVersion{}, nil
|
||||
}
|
||||
return m.versions[len(m.versions)-1], nil
|
||||
}
|
||||
|
||||
func (m *memSettings) ClusterSettingsAt(_ context.Context, v int64) (store.SettingsVersion, error) {
|
||||
if v < 1 || int(v) > len(m.versions) {
|
||||
return store.SettingsVersion{}, nil
|
||||
}
|
||||
return m.versions[v-1], nil
|
||||
}
|
||||
|
||||
func (m *memSettings) ClusterSettingsHistory(context.Context, int) ([]store.SettingsVersion, error) {
|
||||
var out []store.SettingsVersion
|
||||
for i := len(m.versions) - 1; i >= 0; i-- {
|
||||
out = append(out, m.versions[i])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *memSettings) PutClusterSettings(_ context.Context, base int64, body, by, note string) (int64, error) {
|
||||
if base != int64(len(m.versions)) {
|
||||
return 0, store.ErrSettingsChanged
|
||||
}
|
||||
m.versions = append(m.versions, store.SettingsVersion{Version: base + 1, Body: body, By: by, Note: note, At: time.Now()})
|
||||
return base + 1, nil
|
||||
}
|
||||
|
||||
func TestSettingsAreCheckedKeptAndRolledBack(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
st := &memSettings{}
|
||||
a.SetSettings(&Settings{Store: st, Check: func(b []byte) error {
|
||||
if strings.Contains(string(b), "listen") {
|
||||
return errors.New("listen is each machine's")
|
||||
}
|
||||
return nil
|
||||
}})
|
||||
var v SettingsView
|
||||
if code := call(t, a, "PUT", "/api/v1/settings", `{"base":0,"body":"router:\n retries: 2\n"}`, &v); code != 200 || v.Version != 1 {
|
||||
t.Fatalf("first: %d %+v", code, v)
|
||||
}
|
||||
if code := call(t, a, "PUT", "/api/v1/settings", `{"base":1,"body":"listen: :1\n"}`, nil); code != 400 {
|
||||
t.Errorf("a document the check refuses: %d", code)
|
||||
}
|
||||
if code := call(t, a, "PUT", "/api/v1/settings", `{"base":0,"body":"router: {}\n"}`, nil); code != 409 {
|
||||
t.Errorf("a change from an old version: %d", code)
|
||||
}
|
||||
if code := call(t, a, "PUT", "/api/v1/settings", `{"base":1,"body":"router:\n retries: 5\n"}`, nil); code != 200 {
|
||||
t.Fatalf("second: %d", code)
|
||||
}
|
||||
if code := call(t, a, "POST", "/api/v1/settings/rollback", `{"version":1}`, &v); code != 200 ||
|
||||
v.Version != 3 || v.Body != "router:\n retries: 2\n" || v.Note != "back to version 1" {
|
||||
t.Errorf("rollback: %d %+v", code, v)
|
||||
}
|
||||
var h []SettingsView
|
||||
if code := call(t, a, "GET", "/api/v1/settings/history", "", &h); code != 200 || len(h) != 3 || h[0].Version != 3 {
|
||||
t.Errorf("history: %d %+v", code, h)
|
||||
}
|
||||
if code := call(t, a, "GET", "/api/v1/settings?version=9", "", nil); code != 404 {
|
||||
t.Errorf("a version that is not there: %d", code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user