Serve a model from an external OpenAI-compatible provider
Release / build (push) Successful in 1m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s

A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 23:29:01 +02:00
1 parent d5f02c0a4b
commit 94eec557ad
16 files changed
+394 -12

No files matched your search

+31 -1
View File
@@ -9,6 +9,7 @@ import (
"encoding/hex"
"fmt"
"log/slog"
"os"
"strings"
"sync"
"time"
@@ -338,6 +339,34 @@ func (c *Controller) endpoints() []endpoint.Info {
Priority: prof.Priority,
})
}
return append(out, c.externalEndpoints()...)
}
// externalEndpoints are the models served outside the cluster, as endpoints:
// one each, with the provider's key read from its file every time, so a key
// rotated on disk is used from the next publish on.
func (c *Controller) externalEndpoints() []endpoint.Info {
var out []endpoint.Info
for _, m := range c.cat.Load().Models() {
if !m.IsExternal() {
continue
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := os.ReadFile(f) //nolint:gosec // the operator's own path
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
continue
}
key = strings.TrimSpace(string(b))
}
out = append(out, endpoint.Info{
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
})
}
return out
}
@@ -345,7 +374,8 @@ func (c *Controller) endpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
}
return b.String()
}