Ilker ManapandClaude Opus 5.5
94eec557ad
A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
90 lines
3.4 KiB
Go
90 lines
3.4 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
func TestALocalModelWithNoCapacityFallsBackToAnExternalProvider(t *testing.T) {
|
|
local := newFakeEngine(t, "local")
|
|
local.status.Store(http.StatusServiceUnavailable)
|
|
|
|
var calls atomic.Int64
|
|
var gotModel, gotAuth atomic.Value
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/metrics" {
|
|
t.Error("the provider was scraped for metrics")
|
|
}
|
|
calls.Add(1)
|
|
gotAuth.Store(r.Header.Get("Authorization"))
|
|
var body map[string]any
|
|
b, _ := io.ReadAll(r.Body)
|
|
_ = json.Unmarshal(b, &body)
|
|
gotModel.Store(fmt.Sprint(body["model"]))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, `{"model":"gpt-4o-mini","choices":[{"message":{"content":"from outside"}}],"usage":{"prompt_tokens":3,"completion_tokens":2}}`)
|
|
}))
|
|
t.Cleanup(provider.Close)
|
|
|
|
cfg := baseConfig(config.Endpoint{ID: "l", URL: local.srv.URL, Models: []string{"local"}})
|
|
cloud := catalog.Model{Name: "cloud", Source: "external",
|
|
External: &catalog.External{URL: provider.URL, Model: "gpt-4o-mini"}}
|
|
cfg.Models = []catalog.Model{catModel("local", nil, "cloud"), cloud}
|
|
rt := newTestRouter(t, cfg)
|
|
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
|
{ID: "l", URL: local.srv.URL, Models: []string{"local"}},
|
|
{ID: "external:cloud", URL: provider.URL, Models: []string{"cloud"}, APIKey: "provider-key",
|
|
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "gpt-4o-mini"},
|
|
})
|
|
|
|
rec := post(t, rt.Handler(), map[string]any{"model": "local", "messages": []any{}}, nil)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("the provider was called %d times", calls.Load())
|
|
}
|
|
if gotModel.Load() != "gpt-4o-mini" {
|
|
t.Errorf("the provider was asked for %v, not its own name for the model", gotModel.Load())
|
|
}
|
|
if gotAuth.Load() != "Bearer provider-key" {
|
|
t.Errorf("the provider got %v, not its key", gotAuth.Load())
|
|
}
|
|
if got := rec.Header().Get("x-llmgw-served-model"); got != "cloud" {
|
|
t.Errorf("the client should be told it was served by cloud, got %q", got)
|
|
}
|
|
// Never scraped, and never stale for it.
|
|
for _, s := range rt.reg.Snapshots("cloud") {
|
|
if s.Stale {
|
|
t.Error("an external endpoint reads as stale")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) {
|
|
good := catalog.Model{Name: "cloud", Source: "external",
|
|
External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini"}}
|
|
if _, err := catalog.New([]catalog.Model{good}); err != nil {
|
|
t.Fatalf("a valid external model was refused: %v", err)
|
|
}
|
|
for name, m := range map[string]catalog.Model{
|
|
"profiles": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b", Model: "m"},
|
|
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp"}}},
|
|
"no url": {Name: "x", Source: "external", External: &catalog.External{URL: "api.example.com", Model: "m"}},
|
|
"no model": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b"}},
|
|
} {
|
|
if _, err := catalog.New([]catalog.Model{m}); err == nil {
|
|
t.Errorf("%s: accepted", name)
|
|
}
|
|
}
|
|
}
|