package router import ( "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "sync/atomic" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) func TestALocalModelWithNoCapacityFallsBackToAnExternalProvider(t *testing.T) { local := newFakeEngine(t, "local") local.status.Store(http.StatusServiceUnavailable) var calls atomic.Int64 var gotModel, gotAuth atomic.Value provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/metrics" { t.Error("the provider was scraped for metrics") } calls.Add(1) gotAuth.Store(r.Header.Get("Authorization")) var body map[string]any b, _ := io.ReadAll(r.Body) _ = json.Unmarshal(b, &body) gotModel.Store(fmt.Sprint(body["model"])) w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"model":"gpt-4o-mini","choices":[{"message":{"content":"from outside"}}],"usage":{"prompt_tokens":3,"completion_tokens":2}}`) })) t.Cleanup(provider.Close) cfg := baseConfig(config.Endpoint{ID: "l", URL: local.srv.URL, Models: []string{"local"}}) cloud := catalog.Model{Name: "cloud", Source: "external", External: &catalog.External{URL: provider.URL, Model: "gpt-4o-mini"}} cfg.Models = []catalog.Model{catModel("local", nil, "cloud"), cloud} rt := newTestRouter(t, cfg) rt.SetEndpoints(t.Context(), []endpoint.Info{ {ID: "l", URL: local.srv.URL, Models: []string{"local"}}, {ID: "external:cloud", URL: provider.URL, Models: []string{"cloud"}, APIKey: "provider-key", Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "gpt-4o-mini"}, }) rec := post(t, rt.Handler(), map[string]any{"model": "local", "messages": []any{}}, nil) if rec.Code != http.StatusOK { t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) } if calls.Load() != 1 { t.Fatalf("the provider was called %d times", calls.Load()) } if gotModel.Load() != "gpt-4o-mini" { t.Errorf("the provider was asked for %v, not its own name for the model", gotModel.Load()) } if gotAuth.Load() != "Bearer provider-key" { t.Errorf("the provider got %v, not its key", gotAuth.Load()) } if got := rec.Header().Get("x-llmgw-served-model"); got != "cloud" { t.Errorf("the client should be told it was served by cloud, got %q", got) } // Never scraped, and never stale for it. for _, s := range rt.reg.Snapshots("cloud") { if s.Stale { t.Error("an external endpoint reads as stale") } } } func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) { good := catalog.Model{Name: "cloud", Source: "external", External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini"}} if _, err := catalog.New([]catalog.Model{good}); err != nil { t.Fatalf("a valid external model was refused: %v", err) } for name, m := range map[string]catalog.Model{ "profiles": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b", Model: "m"}, Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp"}}}, "no url": {Name: "x", Source: "external", External: &catalog.External{URL: "api.example.com", Model: "m"}}, "no model": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b"}}, } { if _, err := catalog.New([]catalog.Model{m}); err == nil { t.Errorf("%s: accepted", name) } } }