Merge C24 federation: another llmgw cluster as a provider
Conflicts with facility-aware placement (config, control, admin API routes, store test tables) all keep both; the generated reference and OpenAPI document regenerated. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
33334c8f33
33 files changed
+2945
-24
No files matched your search
@@ -89,6 +89,7 @@ type Controller struct {
|
||||
disp reconciler.Dispatcher
|
||||
sink EndpointSink
|
||||
usage UsageSource
|
||||
peers Peers
|
||||
|
||||
mu sync.Mutex
|
||||
last string // fingerprint of the last published list
|
||||
@@ -148,6 +149,27 @@ func (c *Controller) collectUsage() {
|
||||
}
|
||||
}
|
||||
|
||||
// Peers says which federated clusters' models may be sent requests now.
|
||||
//
|
||||
// Federation keeps it: whether each peer answered its last check, and
|
||||
// whether it still offers the model. A peer model is in the catalog either
|
||||
// way — a local model may name it as a fallback — and is published only
|
||||
// while this says yes, which is what takes a peer that is down out of
|
||||
// routing within a publish.
|
||||
type Peers interface {
|
||||
// PeerRoute is the peer's own cluster id and whether requests sent to
|
||||
// it carry tags, and false when the model is not to be sent anything.
|
||||
PeerRoute(peer, upstreamModel string) (cluster string, sendTags, ok bool)
|
||||
}
|
||||
|
||||
// SetPeers supplies the federation's view of its peers. Without one, no
|
||||
// peer model is published.
|
||||
func (c *Controller) SetPeers(p Peers) {
|
||||
c.mu.Lock()
|
||||
c.peers = p
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// LoadSource reports each model's replicas' load, keyed by catalog name.
|
||||
type LoadSource interface {
|
||||
ModelLoad() map[string]reconciler.Load
|
||||
@@ -352,11 +374,25 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
// one each, with the provider's key read from its file every time, so a key
|
||||
// rotated on disk is used from the next publish on.
|
||||
func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
c.mu.Lock()
|
||||
peers := c.peers
|
||||
c.mu.Unlock()
|
||||
var out []endpoint.Info
|
||||
for _, m := range c.cat.Load().Models() {
|
||||
if !m.IsExternal() {
|
||||
continue
|
||||
}
|
||||
var cluster string
|
||||
var tags bool
|
||||
if m.IsPeer() {
|
||||
ok := false
|
||||
if peers != nil {
|
||||
cluster, tags, ok = peers.PeerRoute(m.External.Peer, m.External.Model)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
key := ""
|
||||
if f := m.External.APIKeyFile; f != "" {
|
||||
b, err := secrets.Read(context.Background(), f)
|
||||
@@ -366,11 +402,18 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
// A peer's endpoint says so in its id, which is what usage records
|
||||
// as the replica that answered.
|
||||
id := "external:" + m.Name
|
||||
if m.IsPeer() {
|
||||
id = "peer:" + m.Name
|
||||
}
|
||||
out = append(out, endpoint.Info{
|
||||
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
|
||||
UpstreamModel: m.External.Model,
|
||||
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -380,8 +423,8 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
func fingerprint(infos []endpoint.Info) string {
|
||||
var b strings.Builder
|
||||
for _, i := range infos {
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
|
||||
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%s|%s|%t|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight,
|
||||
i.Priority, i.UpstreamModel, i.Peer, i.PeerCluster, i.PeerTags, sha256.Sum256([]byte(i.APIKey)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user