A control plane being stopped held every agent's stream for the thirty seconds of a graceful shutdown, so the standby led within four seconds and had no nodes for thirty. The agent server is now closed, not drained. A control plane that loses its lease while it lives on lets its agents go, rather than keeping nodes the new leader cannot reach. ctl status and the console's Overview say which control plane leads, which stand by, and whether the floating address is on the leader. The console adds a control plane (Do → Add a control plane), asking for the machine and the floating address. server add says when the mirror would refuse the new control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
132 lines
4.2 KiB
Go
132 lines
4.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func TestStatusSaysWhichControlPlaneLeadsAndWhetherTheOthersStandBy(t *testing.T) {
|
|
render := func(s adminapi.Summary) string {
|
|
cmd := &cobra.Command{}
|
|
var b bytes.Buffer
|
|
cmd.SetOut(&b)
|
|
printSummary(cmd, s)
|
|
return b.String()
|
|
}
|
|
out := render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{
|
|
Members: []adminapi.ControlPlane{
|
|
{Name: "llmgw-ctl", State: "leading", This: true},
|
|
{Name: "llmgw-ctl02", State: "standing by"},
|
|
},
|
|
Floating: "10.0.0.161", FloatingUp: true,
|
|
}})
|
|
if !strings.Contains(out, "CONTROL llmgw-ctl leads (this one), llmgw-ctl02 standing by · 10.0.0.161 on the leader") {
|
|
t.Errorf("status:\n%s", out)
|
|
}
|
|
out = render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{
|
|
Members: []adminapi.ControlPlane{
|
|
{Name: "llmgw-ctl", State: "not answering"},
|
|
{Name: "llmgw-ctl02", State: "standing by", This: true},
|
|
},
|
|
Floating: "10.0.0.161",
|
|
}})
|
|
for _, want := range []string{"NO LEADER", "llmgw-ctl not answering", "10.0.0.161 NOT ANSWERING"} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("%q missing:\n%s", want, out)
|
|
}
|
|
}
|
|
if out := render(adminapi.Summary{}); strings.Contains(out, "CONTROL") {
|
|
t.Error("a single control plane got a control line")
|
|
}
|
|
}
|
|
|
|
func TestTheProbeAsksEachControlPlaneWhetherItLeads(t *testing.T) {
|
|
leading := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
|
defer leading.Close()
|
|
standby := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}))
|
|
defer standby.Close()
|
|
gone := httptest.NewServer(http.NotFoundHandler())
|
|
gone.Close()
|
|
|
|
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
for _, cp := range []store.ControlPlane{
|
|
{URL: "https://10.0.0.1:7443", Name: "one"},
|
|
{URL: "https://10.0.0.2:7443", Name: "two"},
|
|
{URL: "https://10.0.0.3:7443", Name: "three"},
|
|
} {
|
|
if err := st.PutControlPlane(t.Context(), cp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
at := map[string]string{"10.0.0.1": leading.URL, "10.0.0.2": standby.URL, "10.0.0.3": gone.URL, "10.0.0.161": leading.URL}
|
|
p := newControlPlaneProbe(st, ":8080")
|
|
p.self, p.floating = "two", "10.0.0.161"
|
|
p.leaderURL = func(host string) string { return at[host] + "/leader" }
|
|
|
|
s := p.Status()
|
|
got := map[string]adminapi.ControlPlane{}
|
|
for _, m := range s.Members {
|
|
got[m.Name] = m
|
|
}
|
|
if got["one"].State != "leading" || got["two"].State != "standing by" || got["three"].State != "not answering" {
|
|
t.Errorf("states: %+v", got)
|
|
}
|
|
if !got["two"].This || got["one"].This || got["one"].Address != "10.0.0.1" {
|
|
t.Errorf("which and where: %+v", got)
|
|
}
|
|
if !s.FloatingUp {
|
|
t.Error("a floating address on the leader read as down")
|
|
}
|
|
}
|
|
|
|
func TestOneControlPlaneIsNotASetOfThem(t *testing.T) {
|
|
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
if err := st.PutControlPlane(t.Context(), store.ControlPlane{URL: "https://10.0.0.1:7443", Name: "one"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s := newControlPlaneProbe(st, ":8080").Status(); len(s.Members) != 0 {
|
|
t.Errorf("one control plane reported as a set: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestTheFloatingAddressIsReadFromWhatServerAddWrote(t *testing.T) {
|
|
dir := t.TempDir()
|
|
ours := filepath.Join(dir, "ours.conf")
|
|
if err := os.WriteFile(ours, []byte(keepalivedConf("eth0", "10.0.0.161", 24, 110, "abcd1234")), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := floatingAddress(ours); got != "10.0.0.161" {
|
|
t.Errorf("read %q", got)
|
|
}
|
|
theirs := filepath.Join(dir, "theirs.conf")
|
|
if err := os.WriteFile(theirs, []byte("vrrp_instance x {\n virtual_ipaddress {\n 10.9.9.9/24\n }\n}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := floatingAddress(theirs); got != "" {
|
|
t.Errorf("someone else's keepalived read as ours: %q", got)
|
|
}
|
|
if got := floatingAddress(filepath.Join(dir, "none")); got != "" {
|
|
t.Errorf("no file read as %q", got)
|
|
}
|
|
}
|