Serve a model from an external OpenAI-compatible provider
Release / build (push) Successful in 1m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s

A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 23:29:01 +02:00
1 parent d5f02c0a4b
commit 94eec557ad
16 files changed
+394 -12

No files matched your search

+22 -4
View File
@@ -10,16 +10,18 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"unicode"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
@@ -312,8 +314,11 @@ type ModelView struct {
Warnings []string `json:"warnings,omitempty"`
// Class is its capacity class; Preempted the replicas it lost to a higher
// class; HeldFor the model it is giving room to now, if any.
Class string `json:"class"`
Owner string `json:"owner,omitempty"`
Class string `json:"class"`
Owner string `json:"owner,omitempty"`
// External is where a model served outside the cluster goes, provider
// host and its name for the model; empty for one served here.
External string `json:"external,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
}
@@ -569,6 +574,7 @@ func (a *API) Models() []ModelView {
Desired: h.Desired, Failures: h.Failures,
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m),
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -1443,3 +1449,15 @@ func reservationWarnings(m catalog.Model, measuredGB float64) []string {
}
return out
}
// externalLabel is "api.openai.com · gpt-4o-mini" for a model served elsewhere.
func externalLabel(m catalog.Model) string {
if !m.IsExternal() {
return ""
}
host := m.External.URL
if u, err := url.Parse(host); err == nil && u.Host != "" {
host = u.Host
}
return host + " · " + m.External.Model
}
+45 -3
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"slices"
"strings"
"time"
@@ -160,8 +161,16 @@ type Model struct {
// Owner is the part of the tenancy tree whose capacity quota this model
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
// held by the cluster, counted against nobody.
Owner string `json:"owner,omitempty" koanf:"owner"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
Owner string `json:"owner,omitempty" koanf:"owner"`
// External is a model served by someone else — an OpenAI-compatible API
// outside the cluster — rather than by replicas placed here. Such a model
// has no profiles and nothing is placed for it; the router sends its
// requests to the provider. Most useful named in another model's
// fallbacks: used only when the cluster has no capacity left. Its
// prompts leave the cluster, so which keys may name it is worth an
// allowlist.
External *External `json:"external,omitempty" koanf:"external"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
SLO SLO `json:"slo,omitempty" koanf:"slo"`
@@ -305,7 +314,11 @@ func (m Model) validate() error {
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if len(m.Profiles) == 0 {
if m.External != nil {
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
return err
}
} else if len(m.Profiles) == 0 {
return fmt.Errorf("model %q: at least one profile is required", m.Name)
}
if m.Owner != "" && !validOwner(m.Owner) {
@@ -564,3 +577,32 @@ func validOwner(o string) bool {
}
return false
}
// External is a provider outside the cluster that serves a model.
type External struct {
// URL is the provider's base address, without /v1: https://api.openai.com.
URL string `json:"url" koanf:"url"`
// Model is what the provider calls it; requests are rewritten to it.
Model string `json:"model" koanf:"model"`
// APIKeyFile holds the provider's key, on the control plane. A file
// rather than the key itself, so the key is in neither the catalog, the
// store nor the audit log; empty for a provider that asks for none.
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
}
func (e *External) validate(model string, profiles int) error {
if profiles > 0 {
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
}
u, err := url.Parse(e.URL)
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
}
if strings.TrimSpace(e.Model) == "" {
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
}
return nil
}
// IsExternal reports whether a model is served outside the cluster.
func (m Model) IsExternal() bool { return m.External != nil }
+31 -1
View File
@@ -9,6 +9,7 @@ import (
"encoding/hex"
"fmt"
"log/slog"
"os"
"strings"
"sync"
"time"
@@ -338,6 +339,34 @@ func (c *Controller) endpoints() []endpoint.Info {
Priority: prof.Priority,
})
}
return append(out, c.externalEndpoints()...)
}
// externalEndpoints are the models served outside the cluster, as endpoints:
// one each, with the provider's key read from its file every time, so a key
// rotated on disk is used from the next publish on.
func (c *Controller) externalEndpoints() []endpoint.Info {
var out []endpoint.Info
for _, m := range c.cat.Load().Models() {
if !m.IsExternal() {
continue
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := os.ReadFile(f) //nolint:gosec // the operator's own path
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
continue
}
key = strings.TrimSpace(string(b))
}
out = append(out, endpoint.Info{
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
})
}
return out
}
@@ -345,7 +374,8 @@ func (c *Controller) endpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
}
return b.String()
}
+75
View File
@@ -0,0 +1,75 @@
package control
import (
"context"
"log/slog"
"os"
"path/filepath"
"sync"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
type recordSink struct {
mu sync.Mutex
infos []endpoint.Info
times int
}
func (s *recordSink) SetEndpoints(_ context.Context, infos []endpoint.Info) {
s.mu.Lock()
defer s.mu.Unlock()
s.infos, s.times = infos, s.times+1
}
type noDispatch struct{}
func (noDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil }
func (noDispatch) StopShard(context.Context, string, string) error { return nil }
func (noDispatch) DrainShard(context.Context, string, string) error { return nil }
func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.T) {
dir := t.TempDir()
keyFile := filepath.Join(dir, "provider.key")
if err := os.WriteFile(keyFile, []byte("sk-provider-1\n"), 0o600); err != nil {
t.Fatal(err)
}
models := []catalog.Model{
{Name: "cloud", Source: "external", Aliases: []string{"gpt"}, Scaling: catalog.Scaling{Min: 3, Max: 3},
External: &catalog.External{URL: "https://api.example.com/", Model: "gpt-4o-mini", APIKeyFile: keyFile}},
{Name: "gone", Source: "external",
External: &catalog.External{URL: "https://api.example.com", Model: "m", APIKeyFile: filepath.Join(dir, "missing")}},
}
cat, err := catalog.New(models)
if err != nil {
t.Fatal(err)
}
sink := &recordSink{}
c := New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
c.Publish(t.Context())
if len(sink.infos) != 1 {
t.Fatalf("published %+v; the model whose key is missing must not be offered", sink.infos)
}
e := sink.infos[0]
if e.ID != "external:cloud" || e.URL != "https://api.example.com" || e.APIKey != "sk-provider-1" ||
e.UpstreamModel != "gpt-4o-mini" || e.Engine != endpoint.EngineExternal || len(e.Models) != 2 {
t.Errorf("endpoint %+v", e)
}
// A key rotated on disk is published on the next pass.
_ = os.WriteFile(keyFile, []byte("sk-provider-2"), 0o600)
c.Publish(t.Context())
if sink.times != 2 || sink.infos[0].APIKey != "sk-provider-2" {
t.Errorf("the rotated key was not published: %d pushes, key %q", sink.times, sink.infos[0].APIKey)
}
// Scaling of 3 or not, nothing is wanted of the cluster for a model
// served elsewhere, and so it never reads as short or unplaceable.
for _, h := range c.Reconciler().ModelHealth() {
if h.Model == "cloud" && (h.Desired != 0 || h.Unplaceable) {
t.Errorf("an external model wants replicas: %+v", h)
}
}
}
+3
View File
@@ -94,6 +94,9 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
// baseFor is what a model runs without load: its minimum, or for an on-demand
// model one while somebody is asking and none once they stop.
func (r *Reconciler) baseFor(m catalog.Model) int {
if m.IsExternal() {
return 0 // served elsewhere: nothing to place, whatever its scaling says
}
if !m.Scaling.OnDemand() {
return m.Scaling.Min
}