Serve a model from an external OpenAI-compatible provider

A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 23:29:01 +02:00
1 parent d5f02c0a4b
commit 94eec557ad
16 files changed
+394 -12

No files matched your search

+22 -4
View File
@@ -10,16 +10,18 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"unicode"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
@@ -312,8 +314,11 @@ type ModelView struct {
Warnings []string `json:"warnings,omitempty"`
// Class is its capacity class; Preempted the replicas it lost to a higher
// class; HeldFor the model it is giving room to now, if any.
Class string `json:"class"`
Owner string `json:"owner,omitempty"`
Class string `json:"class"`
Owner string `json:"owner,omitempty"`
// External is where a model served outside the cluster goes, provider
// host and its name for the model; empty for one served here.
External string `json:"external,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
}
@@ -569,6 +574,7 @@ func (a *API) Models() []ModelView {
Desired: h.Desired, Failures: h.Failures,
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
External: externalLabel(m),
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -1443,3 +1449,15 @@ func reservationWarnings(m catalog.Model, measuredGB float64) []string {
}
return out
}
// externalLabel is "api.openai.com · gpt-4o-mini" for a model served elsewhere.
func externalLabel(m catalog.Model) string {
if !m.IsExternal() {
return ""
}
host := m.External.URL
if u, err := url.Parse(host); err == nil && u.Host != "" {
host = u.Host
}
return host + " · " + m.External.Model
}