Several keys for one outside provider
CI / test (push) Successful in 8m52s

An external model takes api_key_files, each a file and an optional weight,
in place of api_key_file. Each key is an endpoint of its own
(external:MODEL#1, #2), so the router's scoring, retries and breakers work
per key. A key answering 429 is passed over while its breaker is open. A
key answering 401 is passed over too, and external_key_refused is sent,
naming the endpoint and never the key; with a single key the 401 is still
the client's. A key whose file cannot be read is left out, and usage
records which key answered by its endpoint.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-17 08:56:30 +02:00
1 parent 688b205604
commit fe310576e7
14 files changed
+283 -15

No files matched your search

+28 -12
View File
@@ -438,15 +438,6 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
continue
}
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := secrets.Read(context.Background(), f)
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
continue
}
key = strings.TrimSpace(string(b))
}
// A peer's endpoint says so in its id, which is what usage records
// as the replica that answered.
id := "external:" + m.Name
@@ -462,14 +453,39 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
}
engine, workflow = endpoint.EngineComfyUI, w
}
out = append(out, endpoint.Info{
base := endpoint.Info{
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Models: append([]string{m.Name}, m.Aliases...),
Engine: engine, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
Workflow: workflow,
})
}
keys := m.External.APIKeyFiles
if len(keys) == 0 {
keys = []catalog.ExternalKey{{File: m.External.APIKeyFile}}
}
for i, k := range keys {
e := base
// One endpoint a key, numbered, so usage says which answered
// and a breaker stands for one key — never the key itself.
if len(m.External.APIKeyFiles) > 0 {
e.ID = fmt.Sprintf("%s#%d", id, i+1)
if k.Weight > 0 {
e.Weight = k.Weight
}
}
if k.File != "" {
b, err := secrets.Read(context.Background(), k.File)
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name,
"endpoint", e.ID, "err", err)
continue
}
e.APIKey = strings.TrimSpace(string(b))
}
out = append(out, e)
}
}
return out
}