Several keys for one outside provider
An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
@@ -1533,6 +1533,11 @@ type External struct {
|
||||
// rather than the key itself, so the key is in neither the catalog, the
|
||||
// store nor the audit log; empty for a provider that asks for none.
|
||||
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
||||
// APIKeyFiles are several keys for the provider, in place of
|
||||
// APIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that
|
||||
// answers 429 or 401 is passed over while its breaker is open and the
|
||||
// request goes to another. A provider's limits are per key.
|
||||
APIKeyFiles []ExternalKey `json:"api_key_files,omitempty" koanf:"api_key_files"`
|
||||
// Peer names the federated llmgw cluster this model is offered by, for
|
||||
// a model that federation added (`llmgw ctl federation add`) rather
|
||||
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
||||
@@ -1551,6 +1556,14 @@ type External struct {
|
||||
Output string `json:"output,omitempty" koanf:"output"`
|
||||
}
|
||||
|
||||
// ExternalKey is one of a provider's keys.
|
||||
type ExternalKey struct {
|
||||
// File holds the key, on the control plane, as APIKeyFile does.
|
||||
File string `json:"file" koanf:"file"`
|
||||
// Weight is its share of the requests against the others'; zero is 1.
|
||||
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
||||
}
|
||||
|
||||
// APIComfyUI is External.API for a ComfyUI server.
|
||||
const APIComfyUI = "comfyui"
|
||||
|
||||
@@ -1568,6 +1581,20 @@ func (e *External) validate(model string, profiles int) error {
|
||||
if strings.TrimSpace(e.Model) == "" {
|
||||
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
||||
}
|
||||
if e.APIKeyFile != "" && len(e.APIKeyFiles) > 0 {
|
||||
return fmt.Errorf("model %q: api_key_file or api_key_files, not both", model)
|
||||
}
|
||||
for i, k := range e.APIKeyFiles {
|
||||
if strings.TrimSpace(k.File) == "" {
|
||||
return fmt.Errorf("model %q: api_key_files[%d] names no file", model, i)
|
||||
}
|
||||
if k.Weight < 0 {
|
||||
return fmt.Errorf("model %q: api_key_files[%d] has a negative weight", model, i)
|
||||
}
|
||||
}
|
||||
if len(e.APIKeyFiles) > 0 && e.Peer != "" {
|
||||
return fmt.Errorf("model %q: a federated model is reached with its peer's key, not api_key_files", model)
|
||||
}
|
||||
switch e.API {
|
||||
case "", "openai":
|
||||
if e.Workflow != "" || e.Output != "" {
|
||||
|
||||
@@ -429,3 +429,23 @@ func TestAGuardModelSaysHowItIsReadAndAlwaysRuns(t *testing.T) {
|
||||
t.Error("a judge is not critical by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProvidersKeysAreOneListOrOneFile(t *testing.T) {
|
||||
ext := func(e External) Model {
|
||||
e.URL, e.Model = "https://api.example.com", "m"
|
||||
return Model{Name: "cloud", Source: "external", External: &e}
|
||||
}
|
||||
for _, bad := range []External{
|
||||
{APIKeyFile: "/k", APIKeyFiles: []ExternalKey{{File: "/k2"}}},
|
||||
{APIKeyFiles: []ExternalKey{{File: " "}}},
|
||||
{APIKeyFiles: []ExternalKey{{File: "/k", Weight: -1}}},
|
||||
{Peer: "p", APIKeyFiles: []ExternalKey{{File: "/k"}}},
|
||||
} {
|
||||
if _, err := New([]Model{ext(bad)}); err == nil {
|
||||
t.Errorf("taken: %+v", bad)
|
||||
}
|
||||
}
|
||||
if _, err := New([]Model{ext(External{APIKeyFiles: []ExternalKey{{File: "/a", Weight: 2}, {File: "/b"}}})}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
+28
-12
@@ -438,15 +438,6 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
continue
|
||||
}
|
||||
}
|
||||
key := ""
|
||||
if f := m.External.APIKeyFile; f != "" {
|
||||
b, err := secrets.Read(context.Background(), f)
|
||||
if err != nil {
|
||||
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
// A peer's endpoint says so in its id, which is what usage records
|
||||
// as the replica that answered.
|
||||
id := "external:" + m.Name
|
||||
@@ -462,14 +453,39 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
}
|
||||
engine, workflow = endpoint.EngineComfyUI, w
|
||||
}
|
||||
out = append(out, endpoint.Info{
|
||||
base := endpoint.Info{
|
||||
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||
Models: append([]string{m.Name}, m.Aliases...),
|
||||
Engine: engine, Pool: "external", Weight: 1,
|
||||
UpstreamModel: m.External.Model,
|
||||
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
|
||||
Workflow: workflow,
|
||||
})
|
||||
}
|
||||
keys := m.External.APIKeyFiles
|
||||
if len(keys) == 0 {
|
||||
keys = []catalog.ExternalKey{{File: m.External.APIKeyFile}}
|
||||
}
|
||||
for i, k := range keys {
|
||||
e := base
|
||||
// One endpoint a key, numbered, so usage says which answered
|
||||
// and a breaker stands for one key — never the key itself.
|
||||
if len(m.External.APIKeyFiles) > 0 {
|
||||
e.ID = fmt.Sprintf("%s#%d", id, i+1)
|
||||
if k.Weight > 0 {
|
||||
e.Weight = k.Weight
|
||||
}
|
||||
}
|
||||
if k.File != "" {
|
||||
b, err := secrets.Read(context.Background(), k.File)
|
||||
if err != nil {
|
||||
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name,
|
||||
"endpoint", e.ID, "err", err)
|
||||
continue
|
||||
}
|
||||
e.APIKey = strings.TrimSpace(string(b))
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -73,3 +73,28 @@ func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Several keys for one provider: an endpoint each, numbered, with its
|
||||
// weight; one whose file cannot be read is left out and the others stay.
|
||||
func TestEachOfAProvidersKeysIsAnEndpointOfItsOwn(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a, c := filepath.Join(dir, "a.key"), filepath.Join(dir, "c.key")
|
||||
_ = os.WriteFile(a, []byte("sk-a\n"), 0o600)
|
||||
_ = os.WriteFile(c, []byte("sk-c"), 0o600)
|
||||
cat, err := catalog.New([]catalog.Model{{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini",
|
||||
APIKeyFiles: []catalog.ExternalKey{{File: a, Weight: 3}, {File: filepath.Join(dir, "gone")}, {File: c}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordSink{}
|
||||
New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler)).Publish(t.Context())
|
||||
got := map[string]endpoint.Info{}
|
||||
for _, e := range sink.infos {
|
||||
got[e.ID] = e
|
||||
}
|
||||
if len(got) != 2 || got["external:cloud#1"].APIKey != "sk-a" || got["external:cloud#1"].Weight != 3 ||
|
||||
got["external:cloud#3"].APIKey != "sk-c" || got["external:cloud#3"].Weight != 1 {
|
||||
t.Errorf("published %+v", sink.infos)
|
||||
}
|
||||
}
|
||||
@@ -125,11 +125,16 @@ var Reference = []Block{
|
||||
{Key: "url", Type: "string", Doc: "URL is the provider's base address, without /v1: https://api.openai.com.", Nested: ""},
|
||||
{Key: "model", Type: "string", Doc: "Model is what the provider calls it; requests are rewritten to it.", Nested: ""},
|
||||
{Key: "api_key_file", Type: "string", Doc: "APIKeyFile holds the provider's key, on the control plane. A file\nrather than the key itself, so the key is in neither the catalog, the\nstore nor the audit log; empty for a provider that asks for none.", Nested: ""},
|
||||
{Key: "api_key_files", Type: "[]ExternalKey", Doc: "APIKeyFiles are several keys for the provider, in place of\nAPIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that\nanswers 429 or 401 is passed over while its breaker is open and the\nrequest goes to another. A provider's limits are per key.", Nested: ""},
|
||||
{Key: "peer", Type: "string", Doc: "Peer names the federated llmgw cluster this model is offered by, for\na model that federation added (`llmgw ctl federation add`) rather\nthan one written by hand. Such a model is named PEER/MODEL, comes and\ngoes with the peer's own list, and is sent only while the peer is up.", Nested: ""},
|
||||
{Key: "api", Type: "string", Doc: "API is what the provider speaks: OpenAI's, unless this says comfyui —\na ComfyUI server, asked for pictures at /v1/images/generations and\nsent the workflow below with the request's prompt in it. Model is then\nthe checkpoint file the workflow loads, as ComfyUI lists it.", Nested: ""},
|
||||
{Key: "workflow", Type: "string", Doc: "Workflow is the ComfyUI workflow, in its API format: a built-in's name\n(flux-schnell, the default, or flux-dev) or a JSON file on the control\nplane.", Nested: ""},
|
||||
{Key: "output", Type: "string", Doc: "Output is what it makes: pictures, or meshes (see ComfyUI.Output).", Nested: ""},
|
||||
}},
|
||||
{Package: "catalog", Name: "ExternalKey", Title: "ExternalKey", Doc: "", Fields: []Field{
|
||||
{Key: "file", Type: "string", Doc: "File holds the key, on the control plane, as APIKeyFile does.", Nested: ""},
|
||||
{Key: "weight", Type: "float64", Doc: "Weight is its share of the requests against the others'; zero is 1.", Nested: ""},
|
||||
}},
|
||||
{Package: "config", Name: "Facility", Title: "Facility", Doc: "", Fields: []Field{
|
||||
{Key: "budgets", Type: "[]FacilityBudget", Doc: "Budgets are the domains with a power cap.", Nested: ""},
|
||||
{Key: "margin", Type: "float64", Doc: "Margin is the share of every cap kept free, 0.1 for ten percent. Zero\nkeeps none, for caps already given derated.", Nested: ""},
|
||||
|
||||
@@ -43,10 +43,15 @@ const (
|
||||
// CanaryRolledBack is a model's canary taken out because it served
|
||||
// worse than the model it was a candidate to replace.
|
||||
CanaryRolledBack = "canary_rolled_back"
|
||||
// ExternalKeyRefused is an outside provider answering 401 to one of a
|
||||
// model's keys: revoked, expired or mistyped. Requests go to its other
|
||||
// keys meanwhile, where it has any.
|
||||
ExternalKeyRefused = "external_key_refused"
|
||||
)
|
||||
|
||||
// Kinds are every event there is.
|
||||
var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable, JudgeUnavailable, CanaryRolledBack}
|
||||
var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable, JudgeUnavailable, CanaryRolledBack,
|
||||
ExternalKeyRefused}
|
||||
|
||||
// Event is one thing that happened.
|
||||
type Event struct {
|
||||
|
||||
@@ -131,7 +131,18 @@ const defaultActivationWait = 45 * time.Second
|
||||
// SetNotifier tells the accountant where to say a budget is spent. Nothing
|
||||
// without a store, where there are no budgets.
|
||||
func (r *Router) SetNotifier(n *notify.Notifier) {
|
||||
r.notifier = n
|
||||
if r.acct != nil {
|
||||
r.acct.SetNotifier(n)
|
||||
}
|
||||
}
|
||||
|
||||
// keyRefused says that an outside provider answered 401 to one of a model's
|
||||
// keys, by its endpoint, which numbers it: never the key.
|
||||
func (r *Router) keyRefused(model, endpointID string) {
|
||||
r.log.Warn("an outside provider refused a key", "model", model, "endpoint", endpointID)
|
||||
r.notifier.Notify(notify.ExternalKeyRefused, endpointID,
|
||||
"the provider of "+model+" answered 401 to the key of "+endpointID+
|
||||
": revoked, expired or mistyped; replace its file on the control plane",
|
||||
map[string]any{"model": model, "endpoint": endpointID})
|
||||
}
|
||||
@@ -6,11 +6,15 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
@@ -87,3 +91,68 @@ func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two keys for one provider, one of them at its limit, or refused: every
|
||||
// request is answered by the other key, the failing one's breaker opens, and
|
||||
// a refused key is said, by its endpoint and never its value.
|
||||
func TestAKeyAtItsLimitOrRefusedIsPassedOverForAnother(t *testing.T) {
|
||||
for _, status := range []int{http.StatusTooManyRequests, http.StatusUnauthorized} {
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") == "Bearer key-a" {
|
||||
http.Error(w, "no", status)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"model":"m","choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||||
}))
|
||||
t.Cleanup(provider.Close)
|
||||
cfg := baseConfig()
|
||||
cfg.Models = []catalog.Model{{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: provider.URL, Model: "m"}}}
|
||||
rt := newTestRouter(t, cfg)
|
||||
key := func(id, k string) endpoint.Info {
|
||||
return endpoint.Info{ID: id, URL: provider.URL, Models: []string{"cloud"}, APIKey: k,
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "m"}
|
||||
}
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{key("external:cloud#1", "key-a"), key("external:cloud#2", "key-b")})
|
||||
n, events := notifytest.Catch(t)
|
||||
rt.SetNotifier(n)
|
||||
|
||||
// Until the failing key's breaker opens: how many requests the
|
||||
// scheduler sends it before then is its own choice.
|
||||
open := func() bool {
|
||||
for _, s := range rt.reg.Snapshots("cloud") {
|
||||
if s.ID == "external:cloud#1" && s.Breaker != breaker.Closed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for i := 0; i < 100 && !open(); i++ {
|
||||
if rec := post(t, rt.Handler(), map[string]any{"model": "cloud", "messages": []any{}}, nil); rec.Code != http.StatusOK {
|
||||
t.Fatalf("key-a answering %d: request %d got %d: %s", status, i, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
for _, s := range rt.reg.Snapshots("cloud") {
|
||||
if s.ID == "external:cloud#1" && s.Breaker == breaker.Closed {
|
||||
t.Errorf("%d: the failing key's breaker is still closed", status)
|
||||
}
|
||||
if s.ID == "external:cloud#2" && s.Breaker != breaker.Closed {
|
||||
t.Errorf("%d: the good key's breaker opened", status)
|
||||
}
|
||||
}
|
||||
var refused []notify.Event
|
||||
for _, e := range events() {
|
||||
if e.Kind == notify.ExternalKeyRefused {
|
||||
refused = append(refused, e)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case status == http.StatusTooManyRequests && len(refused) != 0:
|
||||
t.Errorf("a key at its limit was said to be refused: %+v", refused)
|
||||
case status == http.StatusUnauthorized && (len(refused) != 1 || refused[0].Subject != "external:cloud#1" ||
|
||||
strings.Contains(refused[0].Message, "key-a")):
|
||||
t.Errorf("a refused key was said as %+v", refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
@@ -116,6 +117,8 @@ type Router struct {
|
||||
// federation is which cluster this is, to its peers; nil until said.
|
||||
federation atomic.Pointer[federationID]
|
||||
acct *accounting.Accountant
|
||||
// notifier says what the operator must act on; nil says nothing.
|
||||
notifier *notify.Notifier
|
||||
// guard judges requests and answers; nil judges nothing. guardLibrary
|
||||
// is the policies it was built from, for the pages that show them.
|
||||
guard guard.Guard
|
||||
@@ -1193,6 +1196,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
// restart. Not relayed; loaded again and tried again.
|
||||
preq.RetryStatus = http.StatusNotFound
|
||||
}
|
||||
// One of an outside provider's several keys refused: the request
|
||||
// is the provider's to answer with another of them, and the key's
|
||||
// breaker counts it. With one key the 401 is the client's to see.
|
||||
if dec.Endpoint.Engine == endpoint.EngineExternal && strings.Contains(dec.Endpoint.ID, "#") {
|
||||
preq.RetryStatus = http.StatusUnauthorized
|
||||
}
|
||||
if dec.Endpoint.Peer != "" {
|
||||
preq.Extra = r.peerHeaders(ctx, dec.Endpoint, req.Header)
|
||||
}
|
||||
@@ -1209,6 +1218,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
} else {
|
||||
out, perr = r.px.Do(uctx, w, preq)
|
||||
}
|
||||
if dec.Endpoint.Engine == endpoint.EngineExternal && out.Status == http.StatusUnauthorized {
|
||||
r.keyRefused(model, dec.Endpoint.ID)
|
||||
}
|
||||
releaseAdapter()
|
||||
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
|
||||
endSpan(uspan, perr)
|
||||
|
||||
Reference in new issue
Block a user