diff --git a/CHANGELOG.md b/CHANGELOG.md index d12e1aa..fa21d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,19 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### Models served outside the cluster + +A catalog model can be an OpenAI-compatible API elsewhere: `external: {url, +model, api_key_file}` and no profiles. Nothing is placed for it; the router +sends its requests to the provider, rewritten to the provider's name for the +model and with its key, read from the file on every publish so a rotated key is +picked up. Named in another model's `fallbacks`, it is used only when the +cluster has no capacity left. It is never scraped for metrics, and never reads +as stale. Prices, budgets, limits, usage and key allowlists apply as to any +model — and since its prompts leave the cluster, the allowlist is how to decide +who may send them (docs/install.md). `ctl models` and the cluster page mark it +external. + ### Pool spillover A model served from pools of different priority — H100 before A100, say — was diff --git a/docs/install.md b/docs/install.md index f6d93a5..b57be64 100644 --- a/docs/install.md +++ b/docs/install.md @@ -671,6 +671,40 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct which names every candidate and why each was rejected, rather than leaving you to infer it. +### Models served outside the cluster + +A model can be an OpenAI-compatible API elsewhere — a hosted provider, or +another gateway — rather than replicas here. Nothing is placed for it; the +router sends its requests on, with the provider's key and the provider's name +for the model: + +```yaml +- name: cloud-mini + source: external + external: + url: https://api.openai.com # without /v1 + model: gpt-4o-mini # what the provider calls it + api_key_file: /etc/llmgw/secrets/openai.key +``` + +Named in another model's `fallbacks`, it is used only when the cluster has no +capacity left for that model: + +```yaml +- name: qwen2.5-coder-7b-instruct + fallbacks: [cloud-mini] + ... +``` + +The key is read from its file on the control plane, never stored in the +catalog, and a key rotated in the file is used within seconds. A model whose key +file cannot be read is not offered. Everything else applies as to any model: +prices and budgets, rate limits, usage records, and key allowlists. + +**Its prompts leave the cluster.** Decide who may send them: a key or project +whose allowlist leaves the external model out cannot reach it, as a fallback or +by name. The cluster page and `ctl models` mark it `external`. + ### How many replicas, and how much each one carries Replica count is changed while the cluster runs, without editing the diff --git a/docs/reference.md b/docs/reference.md index c76a82d..1c9b5a0 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -78,6 +78,9 @@ rather than repeated. | `pool` | `string` | [Endpoint](#endpoint) | | `weight` | `float64` | [Endpoint](#endpoint) | | `priority` | `int` | [Endpoint](#endpoint) | +| `url` | `string` | [External](#external) | +| `model` | `string` | [External](#external) | +| `api_key_file` | `string` | [External](#external) | | `contact` | `string` | [Landing](#landing) | | `notice` | `string` | [Landing](#landing) | | `bin` | `string` | [LlamaCppConfig](#llamacppconfig) | @@ -95,6 +98,7 @@ rather than repeated. | `scaling` | `Scaling` | [Model](#model) | | `class` | `string` | [Model](#model) | | `owner` | `string` | [Model](#model) | +| `external` | `*External` | [Model](#model) | | `fallbacks` | `[]string` | [Model](#model) | | `routing_profile` | `string` | [Model](#model) | | `slo` | `SLO` | [Model](#model) | @@ -616,6 +620,28 @@ called. Empty means vLLM's names. `int` +## External + +### `url` + +`string` + +URL is the provider's base address, without /v1: https://api.openai.com. + +### `model` + +`string` + +Model is what the provider calls it; requests are rewritten to it. + +### `api_key_file` + +`string` + +APIKeyFile holds the provider's key, on the control plane. A file +rather than the key itself, so the key is in neither the catalog, the +store nor the audit log; empty for a provider that asks for none. + ## Landing ### `contact` @@ -765,6 +791,20 @@ Owner is the part of the tenancy tree whose capacity quota this model counts against, written org:ID, team:ID or project:ID. Empty is shared: held by the cluster, counted against nobody. +### `external` + +`*External` + +External is a model served by someone else — an OpenAI-compatible API +outside the cluster — rather than by replicas placed here. Such a model +has no profiles and nothing is placed for it; the router sends its +requests to the provider. Most useful named in another model's +fallbacks: used only when the cluster has no capacity left. Its +prompts leave the cluster, so which keys may name it is worth an +allowlist. + +Takes a [External](#external) block. + ### `fallbacks` `[]string` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index f34e4c2..41c91ff 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -254,6 +254,8 @@ func modelStatus(m adminapi.ModelView) string { switch { case m.Parked: return "parked" + case m.External != "": + return "external: " + m.External case m.HeldFor != "": return "gave room to " + m.HeldFor case m.Unplaceable: diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 1a89f1c..56d2bf0 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -10,16 +10,18 @@ import ( "context" "encoding/json" "fmt" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" - "gitea.mynodes.xyz/ilker/llmgw/internal/release" "io" "net/http" + "net/url" "slices" "strconv" "strings" "time" "unicode" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control" @@ -312,8 +314,11 @@ type ModelView struct { Warnings []string `json:"warnings,omitempty"` // Class is its capacity class; Preempted the replicas it lost to a higher // class; HeldFor the model it is giving room to now, if any. - Class string `json:"class"` - Owner string `json:"owner,omitempty"` + Class string `json:"class"` + Owner string `json:"owner,omitempty"` + // External is where a model served outside the cluster goes, provider + // host and its name for the model; empty for one served here. + External string `json:"external,omitempty"` Preempted int `json:"preempted,omitempty"` HeldFor string `json:"held_for,omitempty"` } @@ -569,6 +574,7 @@ func (a *API) Models() []ModelView { Desired: h.Desired, Failures: h.Failures, Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable, Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor, + External: externalLabel(m), } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) @@ -1443,3 +1449,15 @@ func reservationWarnings(m catalog.Model, measuredGB float64) []string { } return out } + +// externalLabel is "api.openai.com · gpt-4o-mini" for a model served elsewhere. +func externalLabel(m catalog.Model) string { + if !m.IsExternal() { + return "" + } + host := m.External.URL + if u, err := url.Parse(host); err == nil && u.Host != "" { + host = u.Host + } + return host + " · " + m.External.Model +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 6c34948..e4f3871 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "net/url" "slices" "strings" "time" @@ -160,8 +161,16 @@ type Model struct { // Owner is the part of the tenancy tree whose capacity quota this model // counts against, written org:ID, team:ID or project:ID. Empty is shared: // held by the cluster, counted against nobody. - Owner string `json:"owner,omitempty" koanf:"owner"` - Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` + Owner string `json:"owner,omitempty" koanf:"owner"` + // External is a model served by someone else — an OpenAI-compatible API + // outside the cluster — rather than by replicas placed here. Such a model + // has no profiles and nothing is placed for it; the router sends its + // requests to the provider. Most useful named in another model's + // fallbacks: used only when the cluster has no capacity left. Its + // prompts leave the cluster, so which keys may name it is worth an + // allowlist. + External *External `json:"external,omitempty" koanf:"external"` + Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` SLO SLO `json:"slo,omitempty" koanf:"slo"` @@ -305,7 +314,11 @@ func (m Model) validate() error { if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } - if len(m.Profiles) == 0 { + if m.External != nil { + if err := m.External.validate(m.Name, len(m.Profiles)); err != nil { + return err + } + } else if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } if m.Owner != "" && !validOwner(m.Owner) { @@ -564,3 +577,32 @@ func validOwner(o string) bool { } return false } + +// External is a provider outside the cluster that serves a model. +type External struct { + // URL is the provider's base address, without /v1: https://api.openai.com. + URL string `json:"url" koanf:"url"` + // Model is what the provider calls it; requests are rewritten to it. + Model string `json:"model" koanf:"model"` + // APIKeyFile holds the provider's key, on the control plane. A file + // rather than the key itself, so the key is in neither the catalog, the + // store nor the audit log; empty for a provider that asks for none. + APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"` +} + +func (e *External) validate(model string, profiles int) error { + if profiles > 0 { + return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model) + } + u, err := url.Parse(e.URL) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL) + } + if strings.TrimSpace(e.Model) == "" { + return fmt.Errorf("model %q: external model needs the name the provider knows it by", model) + } + return nil +} + +// IsExternal reports whether a model is served outside the cluster. +func (m Model) IsExternal() bool { return m.External != nil } diff --git a/internal/control/control.go b/internal/control/control.go index 06066c1..54ce8cd 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "os" "strings" "sync" "time" @@ -338,6 +339,34 @@ func (c *Controller) endpoints() []endpoint.Info { Priority: prof.Priority, }) } + return append(out, c.externalEndpoints()...) +} + +// externalEndpoints are the models served outside the cluster, as endpoints: +// one each, with the provider's key read from its file every time, so a key +// rotated on disk is used from the next publish on. +func (c *Controller) externalEndpoints() []endpoint.Info { + var out []endpoint.Info + for _, m := range c.cat.Load().Models() { + if !m.IsExternal() { + continue + } + key := "" + if f := m.External.APIKeyFile; f != "" { + b, err := os.ReadFile(f) //nolint:gosec // the operator's own path + if err != nil { + c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err) + continue + } + key = strings.TrimSpace(string(b)) + } + out = append(out, endpoint.Info{ + ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"), + Models: append([]string{m.Name}, m.Aliases...), APIKey: key, + Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, + UpstreamModel: m.External.Model, + }) + } return out } @@ -345,7 +374,8 @@ func (c *Controller) endpoints() []endpoint.Info { func fingerprint(infos []endpoint.Info) string { var b strings.Builder for _, i := range infos { - fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority) + fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority, + i.UpstreamModel, sha256.Sum256([]byte(i.APIKey))) } return b.String() } diff --git a/internal/control/external_test.go b/internal/control/external_test.go new file mode 100644 index 0000000..d5ea457 --- /dev/null +++ b/internal/control/external_test.go @@ -0,0 +1,75 @@ +package control + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "sync" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" +) + +type recordSink struct { + mu sync.Mutex + infos []endpoint.Info + times int +} + +func (s *recordSink) SetEndpoints(_ context.Context, infos []endpoint.Info) { + s.mu.Lock() + defer s.mu.Unlock() + s.infos, s.times = infos, s.times+1 +} + +type noDispatch struct{} + +func (noDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil } +func (noDispatch) StopShard(context.Context, string, string) error { return nil } +func (noDispatch) DrainShard(context.Context, string, string) error { return nil } + +func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.T) { + dir := t.TempDir() + keyFile := filepath.Join(dir, "provider.key") + if err := os.WriteFile(keyFile, []byte("sk-provider-1\n"), 0o600); err != nil { + t.Fatal(err) + } + models := []catalog.Model{ + {Name: "cloud", Source: "external", Aliases: []string{"gpt"}, Scaling: catalog.Scaling{Min: 3, Max: 3}, + External: &catalog.External{URL: "https://api.example.com/", Model: "gpt-4o-mini", APIKeyFile: keyFile}}, + {Name: "gone", Source: "external", + External: &catalog.External{URL: "https://api.example.com", Model: "m", APIKeyFile: filepath.Join(dir, "missing")}}, + } + cat, err := catalog.New(models) + if err != nil { + t.Fatal(err) + } + sink := &recordSink{} + c := New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler)) + c.Publish(t.Context()) + + if len(sink.infos) != 1 { + t.Fatalf("published %+v; the model whose key is missing must not be offered", sink.infos) + } + e := sink.infos[0] + if e.ID != "external:cloud" || e.URL != "https://api.example.com" || e.APIKey != "sk-provider-1" || + e.UpstreamModel != "gpt-4o-mini" || e.Engine != endpoint.EngineExternal || len(e.Models) != 2 { + t.Errorf("endpoint %+v", e) + } + // A key rotated on disk is published on the next pass. + _ = os.WriteFile(keyFile, []byte("sk-provider-2"), 0o600) + c.Publish(t.Context()) + if sink.times != 2 || sink.infos[0].APIKey != "sk-provider-2" { + t.Errorf("the rotated key was not published: %d pushes, key %q", sink.times, sink.infos[0].APIKey) + } + // Scaling of 3 or not, nothing is wanted of the cluster for a model + // served elsewhere, and so it never reads as short or unplaceable. + for _, h := range c.Reconciler().ModelHealth() { + if h.Model == "cloud" && (h.Desired != 0 || h.Unplaceable) { + t.Errorf("an external model wants replicas: %+v", h) + } + } +} diff --git a/internal/control/reconciler/demand.go b/internal/control/reconciler/demand.go index bbef8cd..7e5aafc 100644 --- a/internal/control/reconciler/demand.go +++ b/internal/control/reconciler/demand.go @@ -94,6 +94,9 @@ func (r *Reconciler) desiredFor(m catalog.Model) int { // baseFor is what a model runs without load: its minimum, or for an on-demand // model one while somebody is asking and none once they stop. func (r *Reconciler) baseFor(m catalog.Model) int { + if m.IsExternal() { + return 0 // served elsewhere: nothing to place, whatever its scaling says + } if !m.Scaling.OnDemand() { return m.Scaling.Min } diff --git a/internal/router/endpoint/endpoint.go b/internal/router/endpoint/endpoint.go index 940b7b5..b2a49d4 100644 --- a/internal/router/endpoint/endpoint.go +++ b/internal/router/endpoint/endpoint.go @@ -37,8 +37,16 @@ type Info struct { Weight float64 // Priority expresses pool preference; higher pools are tried first. Priority int + // UpstreamModel is what the endpoint calls the model, when that differs + // from the catalog's name: an external provider's. Requests are rewritten + // to it. Empty sends the name as the client did. + UpstreamModel string } +// EngineExternal marks an endpoint that is a provider outside the cluster. +// It publishes no metrics, so it is never scraped and never reads as stale. +const EngineExternal = "external" + // Serves reports whether the replica serves the given model. func (i Info) Serves(model string) bool { return slices.Contains(i.Models, model) @@ -242,7 +250,7 @@ func (r *Registry) Snapshots(model string) []Snapshot { continue } var age time.Duration - stale := true + stale := l.info.Engine != EngineExternal if !l.scrapedAt.IsZero() { age = now.Sub(l.scrapedAt) stale = age > r.cfg.StaleAfter diff --git a/internal/router/external_test.go b/internal/router/external_test.go new file mode 100644 index 0000000..37a5509 --- /dev/null +++ b/internal/router/external_test.go @@ -0,0 +1,89 @@ +package router + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" +) + +func TestALocalModelWithNoCapacityFallsBackToAnExternalProvider(t *testing.T) { + local := newFakeEngine(t, "local") + local.status.Store(http.StatusServiceUnavailable) + + var calls atomic.Int64 + var gotModel, gotAuth atomic.Value + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/metrics" { + t.Error("the provider was scraped for metrics") + } + calls.Add(1) + gotAuth.Store(r.Header.Get("Authorization")) + var body map[string]any + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &body) + gotModel.Store(fmt.Sprint(body["model"])) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"model":"gpt-4o-mini","choices":[{"message":{"content":"from outside"}}],"usage":{"prompt_tokens":3,"completion_tokens":2}}`) + })) + t.Cleanup(provider.Close) + + cfg := baseConfig(config.Endpoint{ID: "l", URL: local.srv.URL, Models: []string{"local"}}) + cloud := catalog.Model{Name: "cloud", Source: "external", + External: &catalog.External{URL: provider.URL, Model: "gpt-4o-mini"}} + cfg.Models = []catalog.Model{catModel("local", nil, "cloud"), cloud} + rt := newTestRouter(t, cfg) + rt.SetEndpoints(t.Context(), []endpoint.Info{ + {ID: "l", URL: local.srv.URL, Models: []string{"local"}}, + {ID: "external:cloud", URL: provider.URL, Models: []string{"cloud"}, APIKey: "provider-key", + Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "gpt-4o-mini"}, + }) + + rec := post(t, rt.Handler(), map[string]any{"model": "local", "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if calls.Load() != 1 { + t.Fatalf("the provider was called %d times", calls.Load()) + } + if gotModel.Load() != "gpt-4o-mini" { + t.Errorf("the provider was asked for %v, not its own name for the model", gotModel.Load()) + } + if gotAuth.Load() != "Bearer provider-key" { + t.Errorf("the provider got %v, not its key", gotAuth.Load()) + } + if got := rec.Header().Get("x-llmgw-served-model"); got != "cloud" { + t.Errorf("the client should be told it was served by cloud, got %q", got) + } + // Never scraped, and never stale for it. + for _, s := range rt.reg.Snapshots("cloud") { + if s.Stale { + t.Error("an external endpoint reads as stale") + } + } +} + +func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) { + good := catalog.Model{Name: "cloud", Source: "external", + External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini"}} + if _, err := catalog.New([]catalog.Model{good}); err != nil { + t.Fatalf("a valid external model was refused: %v", err) + } + for name, m := range map[string]catalog.Model{ + "profiles": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b", Model: "m"}, + Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp"}}}, + "no url": {Name: "x", Source: "external", External: &catalog.External{URL: "api.example.com", Model: "m"}}, + "no model": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b"}}, + } { + if _, err := catalog.New([]catalog.Model{m}); err == nil { + t.Errorf("%s: accepted", name) + } + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index d301557..63ed615 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -984,3 +984,23 @@ func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) { w.Header().Set("x-ratelimit-reset-tokens", reset) } } + +// withModel is body with its model set to name, for an endpoint that knows the +// model by another name — an external provider. The body unchanged when it is +// not a JSON object, which the endpoint then refuses as it would have anyway. +func withModel(body []byte, name string) []byte { + var m map[string]json.RawMessage + if err := json.Unmarshal(body, &m); err != nil { + return body + } + encoded, err := json.Marshal(name) + if err != nil { + return body + } + m["model"] = encoded + out, err := json.Marshal(m) + if err != nil { + return body + } + return out +} diff --git a/internal/router/router.go b/internal/router/router.go index 1dba6aa..89cc3fb 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -715,10 +715,14 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa uctx, uspan := tracer().Start(ctx, "upstream", trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID), attribute.String("url.full", dec.Endpoint.URL+req.Path))) + body := req.UpstreamBody + if dec.Endpoint.UpstreamModel != "" { + body = withModel(body, dec.Endpoint.UpstreamModel) + } out, perr := r.px.Do(uctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, APIKey: dec.Endpoint.APIKey, - Body: req.UpstreamBody, + Body: body, Header: withTraceparent(uctx, req.Header), Stream: req.Stream, SuppressUsageChunk: req.SuppressUsageChunk, diff --git a/internal/router/scrape/poller.go b/internal/router/scrape/poller.go index 231915f..096c023 100644 --- a/internal/router/scrape/poller.go +++ b/internal/router/scrape/poller.go @@ -96,6 +96,9 @@ func (p *Poller) Sync(ctx context.Context) { want := make(map[string]endpoint.Info, len(infos)) for _, in := range infos { + if in.Engine == endpoint.EngineExternal { + continue // a provider elsewhere publishes no /metrics + } want[in.ID] = in } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index e3183b5..c34f9ac 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -89,7 +89,7 @@