Several keys for one outside provider
An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
+28
-12
@@ -438,15 +438,6 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
continue
|
||||
}
|
||||
}
|
||||
key := ""
|
||||
if f := m.External.APIKeyFile; f != "" {
|
||||
b, err := secrets.Read(context.Background(), f)
|
||||
if err != nil {
|
||||
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
// A peer's endpoint says so in its id, which is what usage records
|
||||
// as the replica that answered.
|
||||
id := "external:" + m.Name
|
||||
@@ -462,14 +453,39 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
}
|
||||
engine, workflow = endpoint.EngineComfyUI, w
|
||||
}
|
||||
out = append(out, endpoint.Info{
|
||||
base := endpoint.Info{
|
||||
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||
Models: append([]string{m.Name}, m.Aliases...),
|
||||
Engine: engine, Pool: "external", Weight: 1,
|
||||
UpstreamModel: m.External.Model,
|
||||
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
|
||||
Workflow: workflow,
|
||||
})
|
||||
}
|
||||
keys := m.External.APIKeyFiles
|
||||
if len(keys) == 0 {
|
||||
keys = []catalog.ExternalKey{{File: m.External.APIKeyFile}}
|
||||
}
|
||||
for i, k := range keys {
|
||||
e := base
|
||||
// One endpoint a key, numbered, so usage says which answered
|
||||
// and a breaker stands for one key — never the key itself.
|
||||
if len(m.External.APIKeyFiles) > 0 {
|
||||
e.ID = fmt.Sprintf("%s#%d", id, i+1)
|
||||
if k.Weight > 0 {
|
||||
e.Weight = k.Weight
|
||||
}
|
||||
}
|
||||
if k.File != "" {
|
||||
b, err := secrets.Read(context.Background(), k.File)
|
||||
if err != nil {
|
||||
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name,
|
||||
"endpoint", e.ID, "err", err)
|
||||
continue
|
||||
}
|
||||
e.APIKey = strings.TrimSpace(string(b))
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user