llmgw/internal/cli/tui_node_overview_test.go

package cli

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/charmbracelet/x/ansi"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)

func writeNodeFile(t *testing.T, path, body string) {
	t.Helper()
	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
		t.Fatal(err)
	}
}

func TestANodesOverviewDialsTheConfiguredControlPlanesThenTheLeadersOwn(t *testing.T) {
	dir := t.TempDir()
	file := filepath.Join(dir, "control-planes.json")
	writeNodeFile(t, file, `["https://10.0.0.150:7443","10.0.0.140:7443","https://cp3.example/agent"]`)
	got := readControlPlanes(file, []string{"10.0.0.140:7443", "http://10.0.0.141"})
	want := []planeCheck{{addr: "10.0.0.140:7443"}, {addr: "10.0.0.141:80"},
		{addr: "10.0.0.150:7443", learned: true}, {addr: "cp3.example:443", learned: true}}
	if len(got) != len(want) {
		t.Fatalf("got %+v, want %+v", got, want)
	}
	for i := range want {
		if got[i] != want[i] {
			t.Errorf("%d: got %+v, want %+v", i, got[i], want[i])
		}
	}
	// Without the file, what agent.yaml says.
	if got := readControlPlanes(filepath.Join(dir, "none.json"), []string{"cp:7443"}); len(got) != 1 || got[0].addr != "cp:7443" {
		t.Errorf("without the file: %+v", got)
	}
}

func TestANodesOverviewReadsTheEnginesFromTheSupervisorsStateFiles(t *testing.T) {
	dir := t.TempDir()
	writeNodeFile(t, filepath.Join(dir, "shard-qwen3-8b-ab12-0.json"), `{"shard_id":"qwen3-8b-ab12-0","replica_id":"qwen3-8b-ab12",
		"pid":4242,"started_at":"2026-10-19T08:00:00Z","spec":{"ReplicaID":"qwen3-8b-ab12","GPUIndices":[0,1],"HTTPPort":8101,
		"Profile":{"Name":"vllm-tp2","Engine":"vllm","ServedModelName":"qwen3-8b"}}}`)
	writeNodeFile(t, filepath.Join(dir, "shard-bge-m3-7-0.json"), `{"replica_id":"bge-m3-7","pid":99,
		"spec":{"GPUIndices":[1002],"HTTPPort":8102,"Profile":{"Engine":"llamacpp"}}}`)
	writeNodeFile(t, filepath.Join(dir, "shard-x.json.tmp"), `{"replica_id":"half-written"}`)
	writeNodeFile(t, filepath.Join(dir, "control-planes.json"), `[]`)
	runs, note := readShardStates(dir, func(pid int) bool { return pid == 4242 })
	if note != "" {
		t.Fatalf("note %q", note)
	}
	if len(runs) != 2 {
		t.Fatalf("got %+v", runs)
	}
	b, q := runs[0], runs[1]
	if q.replica != "qwen3-8b-ab12" || q.model != "qwen3-8b" || q.engine != "vllm" || q.port != 8101 || !q.alive ||
		len(q.gpus) != 2 || !q.started.Equal(time.Date(2026, 10, 19, 8, 0, 0, 0, time.UTC)) {
		t.Errorf("qwen: %+v", q)
	}
	// No served name: the model from the replica's id.
	if b.model != "bge-m3" || b.alive || b.engine != "llamacpp" {
		t.Errorf("bge: %+v", b)
	}
	if runs, note := readShardStates(filepath.Join(dir, "absent"), pidAlive); runs != nil || note != "" {
		t.Errorf("no work dir: %+v %q", runs, note)
	}
}

func TestANodesOverviewNamesTheModelOnEachGPU(t *testing.T) {
	gpus := []nodeGPU{{index: 0}, {index: 1}, {index: 2}}
	runs := []engineRun{
		{model: "qwen3-8b", gpus: []int{0, 1}},
		{model: "bge-m3", gpus: []int{discovery.MIGIndex(1, 2)}},
		{model: "qwen3-8b", gpus: []int{1}},
	}
	got := modelsOnGPUs(gpus, runs)
	if strings.Join(got[0].models, ",") != "qwen3-8b" || strings.Join(got[1].models, ",") != "qwen3-8b,bge-m3" ||
		len(got[2].models) != 0 {
		t.Errorf("got %+v", got)
	}
}

func TestANodesOverviewReadsWhatSystemdSaysOfTheAgent(t *testing.T) {
	s := parseSystemctlShow("LoadState=loaded\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Mon 2026-10-19 08:12:03 UTC\n")
	if s.state != "active" || s.sub != "running" || s.note != "" || s.since.IsZero() || s.since.Minute() != 12 {
		t.Errorf("got %+v", s)
	}
	if s := parseSystemctlShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n"); s.note == "" {
		t.Errorf("a missing service: %+v", s)
	}
	if s := parseSystemctlShow("LoadState=loaded\nActiveState=failed\nSubState=failed\nActiveEnterTimestamp=n/a\n"); s.state != "failed" ||
		!s.since.IsZero() {
		t.Errorf("a failed service: %+v", s)
	}
}

func TestANodesOverviewKeepsTheLastWarningsOfTheJournal(t *testing.T) {
	var b strings.Builder
	for i := 1; i <= 9; i++ {
		b.WriteString("level=WARN msg=w" + string(rune('0'+i)) + "\n")
	}
	got := parseJournal(b.String(), 6)
	if len(got) != 6 || got[0] != "level=WARN msg=w4" || got[5] != "level=WARN msg=w9" {
		t.Errorf("got %q", got)
	}
	if got := parseJournal("-- No entries --\n", 6); len(got) != 0 {
		t.Errorf("no entries: %q", got)
	}
}

// fakeProbe is a node with two cards, one engine and a broken second control
// plane, and no tool of the machine running the test touched.
func fakeProbe(t *testing.T) nodeProbe {
	dir := t.TempDir()
	writeNodeFile(t, filepath.Join(dir, "shard-qwen3-8b-ab12-0.json"), `{"replica_id":"qwen3-8b-ab12","pid":4242,
		"started_at":"2026-10-19T08:00:00Z","spec":{"GPUIndices":[0],"HTTPPort":8101,"Profile":{"Engine":"vllm","ServedModelName":"qwen3-8b"}}}`)
	return nodeProbe{
		configured: []string{"10.0.0.140:7443", "10.0.0.150:7443"},
		planesFile: filepath.Join(dir, "control-planes.json"),
		workDir:    dir, weightsDir: "/var/lib/llmgw-weights", enginesDir: "/opt/llmgw/engines", root: true,
		run: func(_ context.Context, name string, _ ...string) ([]byte, error) {
			switch name {
			case "systemctl":
				return []byte("LoadState=loaded\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Mon 2026-10-19 08:00:00 UTC\n"), nil
			case "journalctl":
				return []byte("level=WARN msg=\"the control plane does not answer\" server=10.0.0.150:7443\n"), nil
			}
			return nil, errors.New(name + " is not installed")
		},
		dial: func(_ context.Context, addr string) error {
			if addr == "10.0.0.150:7443" {
				return errors.New("dial tcp 10.0.0.150:7443: connect: connection refused")
			}
			return nil
		},
		gpus: func(context.Context) []discovery.GPUSensor {
			return []discovery.GPUSensor{
				{Index: 0, TempC: 61, UtilPercent: 87, MemoryUsedMB: 20480, MemoryTotalMB: 24576},
				{Index: 1, TempC: 35, UtilPercent: 0, MemoryUsedMB: 300, MemoryTotalMB: 24576},
			}
		},
		cards: func(context.Context) ([]discovery.GPU, error) {
			return []discovery.GPU{{Index: 0, Name: "NVIDIA RTX A5000"}, {Index: 1, Name: "NVIDIA RTX A5000"}}, nil
		},
		alive:  func(int) bool { return true },
		statfs: func(string) (uint64, uint64, error) { return 120 << 30, 500 << 30, nil },
		now:    func() time.Time { return time.Date(2026, 10, 19, 11, 12, 0, 0, time.UTC) },
	}
}

func TestANodesOverviewShowsWhatRunsOnTheMachine(t *testing.T) {
	ov := fakeProbe(t).gather(context.Background())
	out := ansi.Strip(renderNodeOverview(ov, 100, ov.at))
	t.Log("\n" + out)
	for _, want := range []string{
		"llmgw-agent", "active (running)", "3h12m ago",
		"10.0.0.140:7443", "reachable", "connection refused",
		"NVIDIA RTX A5000", "87%", "20.0/24 GB", "61°C", "qwen3-8b", "free",
		"qwen3-8b-ab12", "vllm", "8101", "4242", "3h12m",
		"/var/lib/llmgw-weights", "120 GB free of 500 GB", "/opt/llmgw/engines",
		"the control plane does not answer",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("the overview lacks %q", want)
		}
	}
	if ov.names[0] != "NVIDIA RTX A5000" {
		t.Errorf("the names are not kept for the next reading: %v", ov.names)
	}
}

func TestANodeWithoutAGPUSaysSoInOneLine(t *testing.T) {
	p := fakeProbe(t)
	p.gpus = func(context.Context) []discovery.GPUSensor { return nil }
	p.cards = func(context.Context) ([]discovery.GPU, error) { return nil, nil }
	p.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("not installed") }
	ov := p.gather(context.Background())
	out := ansi.Strip(renderNodeOverview(ov, 100, ov.at))
	if !strings.Contains(out, "No GPU on this machine.") {
		t.Errorf("no line saying there is no GPU:\n%s", out)
	}
	if strings.Contains(out, "BUSY") {
		t.Errorf("an empty GPU table:\n%s", out)
	}
	// No systemctl or journalctl is said, not a crash.
	if !strings.Contains(out, "systemctl is not available") || !strings.Contains(out, "The journal cannot be read") {
		t.Errorf("missing tools are not said:\n%s", out)
	}
}

func TestANodesConsoleOpensOnItsOverview(t *testing.T) {
	m := newConsoleModel("127.0.0.1:1", filepath.Join(t.TempDir(), "agent.yaml"))
	if m.menu != cOverview || !m.entryOK(cOverview) {
		t.Fatalf("menu %d", m.menu)
	}
	_, body, _ := m.panel()
	if !strings.Contains(body, "reading this machine") {
		t.Errorf("before the first reading: %q", body)
	}
}