Merge C24 federation: another llmgw cluster as a provider
Conflicts with facility-aware placement (config, control, admin API routes, store test tables) all keep both; the generated reference and OpenAPI document regenerated. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
33334c8f33
33 files changed
+2945
-24
No files matched your search
@@ -168,6 +168,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newAuditCmd(client, &asJSON),
|
||||
newNotifyCmd(client, &asJSON),
|
||||
newUsageCmd(client, &asJSON),
|
||||
newFederationCmd(client, &asJSON),
|
||||
newCertCmd(),
|
||||
)
|
||||
return cmd
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
)
|
||||
|
||||
func newFederationCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "federation",
|
||||
Short: "Offer another llmgw cluster's models here, as PEER/MODEL",
|
||||
Long: "A peer is another llmgw gateway, reached with a key it issued to this\n" +
|
||||
"cluster. Its models join this catalog as PEER/MODEL and follow its own\n" +
|
||||
"list; each is sent requests only while the peer's /ready answers, which\n" +
|
||||
"is checked every federation.interval (10s) and timed. Name one as a\n" +
|
||||
"local model's fallback to send overflow to the other site:\n" +
|
||||
" llmgw ctl model fallbacks qwen-7b east/qwen-7b\n\n" +
|
||||
"A project whose requests stay in the cluster (keys residency\n" +
|
||||
"--in-cluster), or a policy that keeps nothing, is never sent to a peer.\n" +
|
||||
"Requests carry x-llmgw-hop, so two clusters federating each other do\n" +
|
||||
"not send one back and forth.",
|
||||
}
|
||||
c.AddCommand(newFederationAddCmd(client), newFederationListCmd(client, asJSON),
|
||||
newFederationRemoveCmd(client), newFederationSyncCmd(client, asJSON))
|
||||
return c
|
||||
}
|
||||
|
||||
func newFederationAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
var req federation.AddRequest
|
||||
c := &cobra.Command{
|
||||
Use: "add NAME --url URL --key-file FILE",
|
||||
Short: "Add another llmgw cluster as a peer, or change one",
|
||||
Long: "The peer is checked first — its /ready, and its /v1/models with the\n" +
|
||||
"key — and refused if either fails. The key file is read on the control\n" +
|
||||
"plane (or env:NAME, vault:PATH#FIELD), at every check, so a rotated\n" +
|
||||
"key is used without a change here; every control plane needs it.\n\n" +
|
||||
"--send-tags makes each request sent to the peer carry x-llmgw-tags\n" +
|
||||
"federated_from=<this cluster>,org=,team=,project=<ids>, so the peer's\n" +
|
||||
"usage can say who here asked. Off unless given: the peer otherwise sees\n" +
|
||||
"one key, this cluster's, and nothing about who is behind it.",
|
||||
Example: " llmgw ctl federation add east --url https://llm.east.example --key-file /etc/llmgw/east.key\n" +
|
||||
" llmgw ctl federation add east --url https://llm.east.example --key-file /etc/llmgw/east.key --send-tags",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Name = args[0]
|
||||
body, _ := json.Marshal(req)
|
||||
var st federation.Status
|
||||
if err := client().post(cmd.Context(), "/api/v1/federation", body, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s added: %d models, %s\n", st.Name, len(st.Models), healthText(st))
|
||||
for _, m := range st.Models {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), " "+m.Name)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.URL, "url", "", "the peer gateway's client address, without /v1")
|
||||
f.StringVar(&req.KeyFile, "key-file", "", "file on the control plane holding the key the peer issued to this cluster")
|
||||
f.BoolVar(&req.SendTags, "send-tags", false, "tell the peer which cluster and project each request is from")
|
||||
_ = c.MarkFlagRequired("url")
|
||||
_ = c.MarkFlagRequired("key-file")
|
||||
return c
|
||||
}
|
||||
|
||||
func newFederationListCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "Peers, whether each is up, how far away, and the models it offers",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var v adminapi.FederationView
|
||||
if err := client().get(cmd.Context(), "/api/v1/federation", &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "this cluster: %s\n", v.Cluster)
|
||||
if len(v.Peers) == 0 {
|
||||
fmt.Fprintln(out, "no peers: llmgw ctl federation add NAME --url URL --key-file FILE")
|
||||
return nil
|
||||
}
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "PEER\tURL\tHEALTH\tLATENCY\tTAGS\tMODELS")
|
||||
for _, p := range v.Peers {
|
||||
tags := "off"
|
||||
if p.SendTags {
|
||||
tags = "on"
|
||||
}
|
||||
lat := "-"
|
||||
if p.Latency > 0 {
|
||||
lat = fmt.Sprintf("%.1fms", p.LatencyMs)
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", p.Name, p.URL, healthText(p), lat, tags, modelList(p))
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newFederationRemoveCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "remove NAME",
|
||||
Short: "Remove a peer and its models",
|
||||
Long: "Refused while a model here falls back to one of the peer's: change its\n" +
|
||||
"fallbacks first (llmgw ctl model fallbacks).",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var out adminapi.Removed
|
||||
if err := client().delete(cmd.Context(), "/api/v1/federation/"+url.PathEscape(args[0]), &out); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s removed\n", out.Removed)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newFederationSyncCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "sync NAME",
|
||||
Short: "Check a peer now and take its current models, rather than at the next interval",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var st federation.Status
|
||||
if err := client().post(cmd.Context(), "/api/v1/federation/"+url.PathEscape(args[0])+"/sync", nil, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(st)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s, %s\n", st.Name, healthText(st), modelList(st))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// healthText is "up" or "down: why", with how long ago it was checked.
|
||||
func healthText(st federation.Status) string {
|
||||
ago := ""
|
||||
if !st.Checked.IsZero() {
|
||||
ago = fmt.Sprintf(" (%s ago)", time.Since(st.Checked).Round(time.Second))
|
||||
}
|
||||
if st.Up {
|
||||
return "up" + ago
|
||||
}
|
||||
return "down" + ago + ": " + st.Error
|
||||
}
|
||||
|
||||
// modelList is the peer's models by their names here, a withdrawn one
|
||||
// marked as such.
|
||||
func modelList(st federation.Status) string {
|
||||
if len(st.Models) == 0 {
|
||||
return "-"
|
||||
}
|
||||
names := make([]string, 0, len(st.Models))
|
||||
for _, m := range st.Models {
|
||||
n := m.Name
|
||||
if m.Withdrawn {
|
||||
n += " (withdrawn)"
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
func newModelFallbacksCmd(client func() *ctlClient) *cobra.Command {
|
||||
var clear bool
|
||||
c := &cobra.Command{
|
||||
Use: "fallbacks MODEL [FALLBACK...]",
|
||||
Short: "Set the models a model falls back to when it has no capacity",
|
||||
Long: "Replaces the model's fallbacks, tried in order when nothing serving it\n" +
|
||||
"has room. A federation peer's model, PEER/MODEL, sends the overflow to\n" +
|
||||
"the other cluster. A model added at runtime only; one declared in\n" +
|
||||
"server.yaml has its fallbacks there. --clear takes them all away.",
|
||||
Example: " llmgw ctl model fallbacks qwen-7b east/qwen-7b\n" +
|
||||
" llmgw ctl model fallbacks qwen-7b qwen-3b east/qwen-7b\n" +
|
||||
" llmgw ctl model fallbacks qwen-7b --clear",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) == 1 && !clear {
|
||||
return fmt.Errorf("name the fallbacks, or give --clear to take them away")
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.FallbacksRequest{Fallbacks: args[1:]})
|
||||
var v adminapi.FallbacksView
|
||||
if err := client().put(cmd.Context(), "/api/v1/models/"+args[0]+"/fallbacks", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(v.Fallbacks) == 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s has no fallbacks\n", v.Model)
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s falls back to %s\n", v.Model, strings.Join(v.Fallbacks, ", then "))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&clear, "clear", false, "take every fallback away")
|
||||
return c
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
newModelRollbackCmd(client), newModelOriginCmd(client),
|
||||
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON),
|
||||
newModelGateCmd(client))
|
||||
newModelGateCmd(client), newModelFallbacksCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -41,6 +43,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -176,6 +179,21 @@ func newServerCmd() *cobra.Command {
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
|
||||
// Which cluster this is, to federated peers: the id in the hop
|
||||
// header that stops a request going round two clusters that
|
||||
// federate each other, and the name in the tags a peer may be
|
||||
// sent. The id is the store's, so every control plane of one
|
||||
// cluster says the same.
|
||||
clusterID, err := federationID(ctx, rt.Store())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clusterName := cfg.Federation.Name
|
||||
if clusterName == "" {
|
||||
clusterName = host
|
||||
}
|
||||
rt.SetFederationIdentity(clusterID, clusterName)
|
||||
|
||||
// Webhooks for what needs a person, when server.yaml names any.
|
||||
notifier, err := notify.New(ctx, cfg.Notify, log)
|
||||
if err != nil {
|
||||
@@ -333,6 +351,21 @@ func newServerCmd() *cobra.Command {
|
||||
if st := rt.Store(); st != nil {
|
||||
svc := modelsvc.New(cfg.Models, st, admin, log)
|
||||
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
|
||||
// Other llmgw clusters' models, offered as PEER/MODEL:
|
||||
// read before the catalog is built, so a model that falls
|
||||
// back to one is whole at startup.
|
||||
fed := federation.New(federation.Config{Self: clusterID, Interval: cfg.Federation.Interval},
|
||||
st, svc, admin.Catalog, log)
|
||||
if err := fed.Load(ctx); err != nil {
|
||||
log.Warn("could not read the federation peers; continuing without them", "err", err)
|
||||
}
|
||||
svc.SetFederated(fed.Models)
|
||||
admin.SetPeers(fed)
|
||||
api.SetFederation(fed)
|
||||
if err := rt.RegisterMetrics(fed); err != nil {
|
||||
return err
|
||||
}
|
||||
go fed.Run(ctx)
|
||||
// Rebuild the catalog from configuration plus what was
|
||||
// stored, and put that in force — otherwise models added
|
||||
// before the last restart would be silently absent until
|
||||
@@ -965,3 +998,25 @@ func withLimits(s *http.Server) *http.Server {
|
||||
s.MaxHeaderBytes = 1 << 20
|
||||
return s
|
||||
}
|
||||
|
||||
// federationID is this cluster's id among federated peers: kept in the store,
|
||||
// so every control plane of the cluster and every restart says the same. A
|
||||
// gateway with no store makes one for its lifetime, which is all a router
|
||||
// with no peers of its own needs to refuse a request that comes back to it.
|
||||
func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
|
||||
if st == nil {
|
||||
b := make([]byte, 8)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
secret, err := st.ClusterSecret(ctx, "cluster-id")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("this cluster's id for federation: %w", err)
|
||||
}
|
||||
// A prefix is plenty to tell clusters apart, and the id is sent to
|
||||
// peers: nothing is derived from it, but there is no reason to send
|
||||
// more of a secret than needed.
|
||||
return secret[:16], nil
|
||||
}
|
||||
@@ -42,6 +42,7 @@ const (
|
||||
cReplicas
|
||||
cUsage
|
||||
cQuotas
|
||||
cFederation
|
||||
cMirror
|
||||
cJobs
|
||||
cLogs
|
||||
@@ -61,7 +62,7 @@ const (
|
||||
cCount
|
||||
)
|
||||
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||||
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||||
"Restart all replicas", "Back up now", "Leave"}
|
||||
|
||||
@@ -79,7 +80,10 @@ type consoleData struct {
|
||||
replicas []adminapi.ReplicaView
|
||||
jobs []modelserve.Job
|
||||
audit []store.AuditEvent
|
||||
err error
|
||||
// federation is the other llmgw clusters whose models are offered
|
||||
// here; empty on a control plane without a store.
|
||||
federation adminapi.FederationView
|
||||
err error
|
||||
}
|
||||
|
||||
type (
|
||||
@@ -238,6 +242,7 @@ func (m consoleModel) fetch() tea.Cmd {
|
||||
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
|
||||
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
|
||||
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
|
||||
_ = c.get(ctx, "/api/v1/federation", &d.federation)
|
||||
return consoleDataMsg(d)
|
||||
}
|
||||
}
|
||||
@@ -494,6 +499,9 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cAudit:
|
||||
m.dialog = "error"
|
||||
m.dialogTitle, m.dialogText = auditDialog(m.data.audit[m.row])
|
||||
case cFederation:
|
||||
m.dialog = "error"
|
||||
m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
|
||||
case cServe:
|
||||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||||
case cAccounts:
|
||||
@@ -908,6 +916,8 @@ func (m consoleModel) rowCount() int {
|
||||
return len(m.data.jobs)
|
||||
case cAudit:
|
||||
return len(m.data.audit)
|
||||
case cFederation:
|
||||
return len(m.data.federation.Peers)
|
||||
case cServe:
|
||||
return len(m.hf.models)
|
||||
case cAccounts:
|
||||
@@ -1149,6 +1159,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
return m.jobsPanel(pw)
|
||||
case cAudit:
|
||||
return m.auditPanel(pw)
|
||||
case cFederation:
|
||||
return m.federationPanel(pw)
|
||||
case cMonitor:
|
||||
return m.monitorPanel(pw)
|
||||
case cServe:
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
)
|
||||
|
||||
// The console's federation view: the other llmgw clusters whose models are
|
||||
// offered here, whether each is up and how far away, as `llmgw ctl
|
||||
// federation list` shows them. Peers are added and removed with ctl, which
|
||||
// needs a key file on this machine that a form here could not check.
|
||||
|
||||
func (m consoleModel) federationPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ peer · Enter its models · ← back · Q leave"
|
||||
}
|
||||
v := m.data.federation
|
||||
head := "\n " + sDim.Render("Other llmgw clusters whose models are offered here as PEER/MODEL.")
|
||||
if v.Cluster != "" {
|
||||
head += "\n " + sDim.Render("This cluster's id: "+v.Cluster)
|
||||
}
|
||||
head += "\n\n"
|
||||
if len(v.Peers) == 0 {
|
||||
return "Federation", head + " " + sDim.Render("No peers. Add one with: llmgw ctl federation add NAME --url URL --key-file FILE"), keys
|
||||
}
|
||||
var rows [][]string
|
||||
for _, p := range v.Peers {
|
||||
health := sOK.Render("up")
|
||||
if !p.Up {
|
||||
health = sBad.Render("down")
|
||||
}
|
||||
lat := "-"
|
||||
if p.Latency > 0 {
|
||||
lat = fmt.Sprintf("%.1fms", p.LatencyMs)
|
||||
}
|
||||
offered := 0
|
||||
for _, md := range p.Models {
|
||||
if !md.Withdrawn {
|
||||
offered++
|
||||
}
|
||||
}
|
||||
rows = append(rows, []string{p.Name, health, lat, fmt.Sprint(offered), p.URL})
|
||||
}
|
||||
return "Federation", head + table([]string{"PEER", "HEALTH", "LATENCY", "MODELS", "URL"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
// peerDialog is one peer whole: how it was last found, and its models.
|
||||
func peerDialog(p federation.Status) (string, string) {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s\n", p.URL)
|
||||
if p.Up {
|
||||
fmt.Fprintf(&b, "%s, /ready in %.1fms", sOK.Render("up"), p.LatencyMs)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "%s: %s", sBad.Render("down"), wrap(p.Error, 80, " "))
|
||||
}
|
||||
if !p.Checked.IsZero() {
|
||||
fmt.Fprintf(&b, ", checked %s ago", time.Since(p.Checked).Round(time.Second))
|
||||
}
|
||||
tags := "not told who asks"
|
||||
if p.SendTags {
|
||||
tags = "told this cluster and the project of each request (x-llmgw-tags)"
|
||||
}
|
||||
fmt.Fprintf(&b, "\ncluster %s, key %s; %s\n\n", p.Cluster, p.KeyFile, tags)
|
||||
if len(p.Models) == 0 {
|
||||
b.WriteString(sDim.Render("offers this cluster's key no models") + "\n")
|
||||
}
|
||||
for _, md := range p.Models {
|
||||
line := md.Name
|
||||
if md.Withdrawn {
|
||||
line += sDim.Render(" withdrawn by the peer; kept while a fallback here names it")
|
||||
}
|
||||
b.WriteString(" " + line + "\n")
|
||||
}
|
||||
fmt.Fprintf(&b, "\n%s", sDim.Render("llmgw ctl federation sync|remove "+p.Name+" · any key"))
|
||||
return p.Name, b.String()
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
)
|
||||
|
||||
func TestTheConsoleShowsEachPeerAndOpensItsModels(t *testing.T) {
|
||||
m, _ := consoleWith(t, cFederation)
|
||||
m.data.federation = adminapi.FederationView{Cluster: "c-west", Peers: []federation.Status{
|
||||
{Name: "east", URL: "https://east.example", Cluster: "c-east",
|
||||
Health: federation.Health{Up: true, Latency: 12 * time.Millisecond, Checked: time.Now()},
|
||||
LatencyMs: 12, Models: []federation.ModelStatus{{Name: "east/qwen", Upstream: "qwen"},
|
||||
{Name: "east/old", Upstream: "old", Withdrawn: true}}},
|
||||
{Name: "north", URL: "https://north.example", Health: federation.Health{Error: "the peer did not answer"}},
|
||||
}}
|
||||
var tm tea.Model = m
|
||||
view := tm.View()
|
||||
for _, want := range []string{"east", "12.0ms", "north", "down", "c-west"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Errorf("the federation view has no %q:\n%s", want, view)
|
||||
}
|
||||
}
|
||||
tm = press(tm, "enter")
|
||||
view = tm.View()
|
||||
for _, want := range []string{"east/qwen", "east/old", "withdrawn", "c-east"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Errorf("the peer's dialog has no %q:\n%s", want, view)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user