An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
@@ -34,6 +34,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
@@ -116,6 +117,8 @@ type Router struct {
|
||||
// federation is which cluster this is, to its peers; nil until said.
|
||||
federation atomic.Pointer[federationID]
|
||||
acct *accounting.Accountant
|
||||
// notifier says what the operator must act on; nil says nothing.
|
||||
notifier *notify.Notifier
|
||||
// guard judges requests and answers; nil judges nothing. guardLibrary
|
||||
// is the policies it was built from, for the pages that show them.
|
||||
guard guard.Guard
|
||||
@@ -1193,6 +1196,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
// restart. Not relayed; loaded again and tried again.
|
||||
preq.RetryStatus = http.StatusNotFound
|
||||
}
|
||||
// One of an outside provider's several keys refused: the request
|
||||
// is the provider's to answer with another of them, and the key's
|
||||
// breaker counts it. With one key the 401 is the client's to see.
|
||||
if dec.Endpoint.Engine == endpoint.EngineExternal && strings.Contains(dec.Endpoint.ID, "#") {
|
||||
preq.RetryStatus = http.StatusUnauthorized
|
||||
}
|
||||
if dec.Endpoint.Peer != "" {
|
||||
preq.Extra = r.peerHeaders(ctx, dec.Endpoint, req.Header)
|
||||
}
|
||||
@@ -1209,6 +1218,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
} else {
|
||||
out, perr = r.px.Do(uctx, w, preq)
|
||||
}
|
||||
if dec.Endpoint.Engine == endpoint.EngineExternal && out.Status == http.StatusUnauthorized {
|
||||
r.keyRefused(model, dec.Endpoint.ID)
|
||||
}
|
||||
releaseAdapter()
|
||||
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
|
||||
endSpan(uspan, perr)
|
||||
|
||||
Reference in new issue
Block a user