Ilker ManapandClaude Opus 5.5
94eec557ad
A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
76 lines
2.6 KiB
Go
76 lines
2.6 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
type recordSink struct {
|
|
mu sync.Mutex
|
|
infos []endpoint.Info
|
|
times int
|
|
}
|
|
|
|
func (s *recordSink) SetEndpoints(_ context.Context, infos []endpoint.Info) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.infos, s.times = infos, s.times+1
|
|
}
|
|
|
|
type noDispatch struct{}
|
|
|
|
func (noDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil }
|
|
func (noDispatch) StopShard(context.Context, string, string) error { return nil }
|
|
func (noDispatch) DrainShard(context.Context, string, string) error { return nil }
|
|
|
|
func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.T) {
|
|
dir := t.TempDir()
|
|
keyFile := filepath.Join(dir, "provider.key")
|
|
if err := os.WriteFile(keyFile, []byte("sk-provider-1\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
models := []catalog.Model{
|
|
{Name: "cloud", Source: "external", Aliases: []string{"gpt"}, Scaling: catalog.Scaling{Min: 3, Max: 3},
|
|
External: &catalog.External{URL: "https://api.example.com/", Model: "gpt-4o-mini", APIKeyFile: keyFile}},
|
|
{Name: "gone", Source: "external",
|
|
External: &catalog.External{URL: "https://api.example.com", Model: "m", APIKeyFile: filepath.Join(dir, "missing")}},
|
|
}
|
|
cat, err := catalog.New(models)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sink := &recordSink{}
|
|
c := New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
|
|
c.Publish(t.Context())
|
|
|
|
if len(sink.infos) != 1 {
|
|
t.Fatalf("published %+v; the model whose key is missing must not be offered", sink.infos)
|
|
}
|
|
e := sink.infos[0]
|
|
if e.ID != "external:cloud" || e.URL != "https://api.example.com" || e.APIKey != "sk-provider-1" ||
|
|
e.UpstreamModel != "gpt-4o-mini" || e.Engine != endpoint.EngineExternal || len(e.Models) != 2 {
|
|
t.Errorf("endpoint %+v", e)
|
|
}
|
|
// A key rotated on disk is published on the next pass.
|
|
_ = os.WriteFile(keyFile, []byte("sk-provider-2"), 0o600)
|
|
c.Publish(t.Context())
|
|
if sink.times != 2 || sink.infos[0].APIKey != "sk-provider-2" {
|
|
t.Errorf("the rotated key was not published: %d pushes, key %q", sink.times, sink.infos[0].APIKey)
|
|
}
|
|
// Scaling of 3 or not, nothing is wanted of the cluster for a model
|
|
// served elsewhere, and so it never reads as short or unplaceable.
|
|
for _, h := range c.Reconciler().ModelHealth() {
|
|
if h.Model == "cloud" && (h.Desired != 0 || h.Unplaceable) {
|
|
t.Errorf("an external model wants replicas: %+v", h)
|
|
}
|
|
}
|
|
}
|