Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -0,0 +1,42 @@
|
||||
package federation
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
var (
|
||||
upDesc = prometheus.NewDesc("llmgw_federation_peer_up",
|
||||
"Whether a federated peer answered its last check: 1 up, 0 down and not sent requests.",
|
||||
[]string{"peer"}, nil)
|
||||
latencyDesc = prometheus.NewDesc("llmgw_federation_peer_latency_seconds",
|
||||
"How long a federated peer took to answer /ready at its last check.",
|
||||
[]string{"peer"}, nil)
|
||||
modelsDesc = prometheus.NewDesc("llmgw_federation_peer_models",
|
||||
"How many models a federated peer offers this cluster.",
|
||||
[]string{"peer"}, nil)
|
||||
)
|
||||
|
||||
// Describe and Collect publish each peer's health beside the traffic it
|
||||
// takes, so "overflow went to the other site" and "the other site was slow"
|
||||
// are on one chart.
|
||||
func (s *Service) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- upDesc
|
||||
ch <- latencyDesc
|
||||
ch <- modelsDesc
|
||||
}
|
||||
|
||||
func (s *Service) Collect(ch chan<- prometheus.Metric) {
|
||||
for _, st := range s.List() {
|
||||
up := 0.0
|
||||
if st.Up {
|
||||
up = 1
|
||||
}
|
||||
offered := 0
|
||||
for _, m := range st.Models {
|
||||
if !m.Withdrawn {
|
||||
offered++
|
||||
}
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, up, st.Name)
|
||||
ch <- prometheus.MustNewConstMetric(latencyDesc, prometheus.GaugeValue, st.Latency.Seconds(), st.Name)
|
||||
ch <- prometheus.MustNewConstMetric(modelsDesc, prometheus.GaugeValue, float64(offered), st.Name)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user