An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
@@ -1533,6 +1533,11 @@ type External struct {
|
||||
// rather than the key itself, so the key is in neither the catalog, the
|
||||
// store nor the audit log; empty for a provider that asks for none.
|
||||
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
||||
// APIKeyFiles are several keys for the provider, in place of
|
||||
// APIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that
|
||||
// answers 429 or 401 is passed over while its breaker is open and the
|
||||
// request goes to another. A provider's limits are per key.
|
||||
APIKeyFiles []ExternalKey `json:"api_key_files,omitempty" koanf:"api_key_files"`
|
||||
// Peer names the federated llmgw cluster this model is offered by, for
|
||||
// a model that federation added (`llmgw ctl federation add`) rather
|
||||
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
||||
@@ -1551,6 +1556,14 @@ type External struct {
|
||||
Output string `json:"output,omitempty" koanf:"output"`
|
||||
}
|
||||
|
||||
// ExternalKey is one of a provider's keys.
|
||||
type ExternalKey struct {
|
||||
// File holds the key, on the control plane, as APIKeyFile does.
|
||||
File string `json:"file" koanf:"file"`
|
||||
// Weight is its share of the requests against the others'; zero is 1.
|
||||
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
||||
}
|
||||
|
||||
// APIComfyUI is External.API for a ComfyUI server.
|
||||
const APIComfyUI = "comfyui"
|
||||
|
||||
@@ -1568,6 +1581,20 @@ func (e *External) validate(model string, profiles int) error {
|
||||
if strings.TrimSpace(e.Model) == "" {
|
||||
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
||||
}
|
||||
if e.APIKeyFile != "" && len(e.APIKeyFiles) > 0 {
|
||||
return fmt.Errorf("model %q: api_key_file or api_key_files, not both", model)
|
||||
}
|
||||
for i, k := range e.APIKeyFiles {
|
||||
if strings.TrimSpace(k.File) == "" {
|
||||
return fmt.Errorf("model %q: api_key_files[%d] names no file", model, i)
|
||||
}
|
||||
if k.Weight < 0 {
|
||||
return fmt.Errorf("model %q: api_key_files[%d] has a negative weight", model, i)
|
||||
}
|
||||
}
|
||||
if len(e.APIKeyFiles) > 0 && e.Peer != "" {
|
||||
return fmt.Errorf("model %q: a federated model is reached with its peer's key, not api_key_files", model)
|
||||
}
|
||||
switch e.API {
|
||||
case "", "openai":
|
||||
if e.Workflow != "" || e.Output != "" {
|
||||
|
||||
@@ -429,3 +429,23 @@ func TestAGuardModelSaysHowItIsReadAndAlwaysRuns(t *testing.T) {
|
||||
t.Error("a judge is not critical by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProvidersKeysAreOneListOrOneFile(t *testing.T) {
|
||||
ext := func(e External) Model {
|
||||
e.URL, e.Model = "https://api.example.com", "m"
|
||||
return Model{Name: "cloud", Source: "external", External: &e}
|
||||
}
|
||||
for _, bad := range []External{
|
||||
{APIKeyFile: "/k", APIKeyFiles: []ExternalKey{{File: "/k2"}}},
|
||||
{APIKeyFiles: []ExternalKey{{File: " "}}},
|
||||
{APIKeyFiles: []ExternalKey{{File: "/k", Weight: -1}}},
|
||||
{Peer: "p", APIKeyFiles: []ExternalKey{{File: "/k"}}},
|
||||
} {
|
||||
if _, err := New([]Model{ext(bad)}); err == nil {
|
||||
t.Errorf("taken: %+v", bad)
|
||||
}
|
||||
}
|
||||
if _, err := New([]Model{ext(External{APIKeyFiles: []ExternalKey{{File: "/a", Weight: 2}, {File: "/b"}}})}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user