diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ba3a36..830ebe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,23 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### Machines nothing may log into: `llmgw join` + +For sites that allow no ssh into a machine, the setup screens offer a fourth +way to reach one: a join code. The setup listens on port 7444 while it waits +and shows a code for each such machine, with the commands that fetch llmgw +onto it; there, `sudo llmgw join ` dials out — nothing dials in — +checks by the certificate fingerprint in the code that it reached the setup +that made it, admits itself with the code's secret, and runs what the setup +sends, printing each step, until the setup says it is done. A code is good +for one machine and an hour, and is under 90 characters so it can be copied +whole from one screen into another. + +To the setup a joined machine is one more way of running a script there: +the checks, PostgreSQL, Patroni and etcd go over it unchanged. Tried with one +machine joining by code and another logged into with a password, in one +setup. + ### Setting up from nothing in screens: `sudo llmgw` Run with nothing after it in a terminal, as root, `llmgw` opens full-screen diff --git a/docs/install.md b/docs/install.md index 5b1a63e..b31945c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -91,7 +91,9 @@ CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw plane, run `sudo llmgw` with nothing after it. With nothing installed it asks whether this is one machine or three for high availability; for three, it asks for the other two — address, user, and an ssh key, the ssh agent, or a -password, and the sudo password if sudo asks for one — checks all three before +password, and the sudo password if sudo asks for one; or, for a machine +nothing may log into, a join code to run there with `sudo llmgw join `, +which dials out to this one on port 7444 — checks all three before installing anything (system, room, clock, ports, internet, whether they reach one another), shows the plan, and installs. Passwords are held in memory for the run only; a sudo password lets the user sudo without one until the install diff --git a/internal/cli/join.go b/internal/cli/join.go new file mode 100644 index 0000000..f7dc8f9 --- /dev/null +++ b/internal/cli/join.go @@ -0,0 +1,61 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "runtime" + + "github.com/spf13/cobra" + + joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +func newJoinCmd() *cobra.Command { + return &cobra.Command{ + Use: "join ", + Short: "Let a setup on another machine set this one up, with the code it gave", + Long: "For a machine nothing may log into: the setup screens (sudo llmgw on\n" + + "the control plane) show a code for each machine they expect. Run this\n" + + "with it, as root, on that machine. It dials out to the setup — nothing\n" + + "dials in — checks it is the setup the code was made by, and runs what\n" + + "the setup sends until it is done: the same checks and installs ssh\n" + + "would carry. Each step is printed here as it runs.\n\n" + + "A code is good for one machine, for an hour, and only while the setup\n" + + "that made it is waiting.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if runtime.GOOS != "linux" { + return fmt.Errorf("join sets up a Linux machine; this is %s", runtime.GOOS) + } + if os.Geteuid() != 0 { + return errors.New("run it as root — sudo llmgw join … — since the setup installs software here") + } + code, err := joinpkg.Parse(args[0]) + if err != nil { + return err + } + out := cmd.OutOrStdout() + role := code.Role + if role == "" { + role = "part of a cluster" + } + fmt.Fprintf(out, "Joining the setup at %s, as %s. It runs as root here until it is done.\n", code.Addr, role) + return joinpkg.Serve(cmd.Context(), code, func(s string) { fmt.Fprintln(out, s) }) + }, + } +} + +// joinInstall is what to run on a machine that has no llmgw yet, to fetch +// this version for its architecture: short lines, since it is copied from a +// screen that cuts long ones off. +func joinInstall() []string { + v := version.Version + return []string{ + `a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`, + fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v), + "sudo chmod 755 /usr/local/bin/llmgw", + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index a1d4613..09e3f44 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -57,6 +57,7 @@ func Root() *cobra.Command { newNodeCmd(), newSelfUpgradeCmd(), newReleaseCmd(), + newJoinCmd(), ) return root } diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index 43fb9a1..9ed5613 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -8,10 +8,12 @@ import ( "os" "strings" "sync" + "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" @@ -34,6 +36,9 @@ const ( pWelcome setupPage = iota pAdmin pMachines + // pWait shows the codes for machines joining rather than logged into, + // until they have all dialled in. + pWait pCheck pPlan pRun @@ -44,10 +49,13 @@ const ( authAgent = iota authKey authPassword + // authJoin is no login at all: the machine runs `llmgw join` with a code + // shown here and dials out to this one. + authJoin authCount ) -var authNames = [authCount]string{"ssh agent", "key file", "password"} +var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"} // Fields of one database machine, in order. const ( @@ -62,6 +70,30 @@ const ( type dbMachine struct { addr, user, secret, sudo textField auth int + // For a machine joining: its code, and it once it has dialled in. + code string + joined *joinpkg.Remote + arrive <-chan *joinpkg.Remote +} + +// visible reports whether a field is shown, and so can be moved to: a joining +// machine is asked nothing but how it joins. +func (d dbMachine) visible(f int) bool { + switch { + case d.auth == authJoin: + return f == mfAuth + case f == mfSecret: + return d.auth != authAgent + } + return true +} + +// dbConn is a database machine as it is reached, whichever way. +type dbConn struct { + remote onboard.Remote + addr string + ssh *onboard.GoSSH + joined *joinpkg.Remote } func newDBMachine(n int) dbMachine { @@ -100,7 +132,10 @@ func (d *dbMachine) field(i int) *textField { // Messages from the work behind the screens. type ( checkDoneMsg struct{ rep preflight.Report } - setupRunMsg struct{} + joinedMsg struct { + i int + r *joinpkg.Remote + } ) // progRef lets work started from Update send to the program it runs in. @@ -120,7 +155,8 @@ type setupModel struct { db [2]dbMachine mfocus int // over both machines' fields, then the check button - remotes []*onboard.GoSSH + conns []dbConn + listener *joinpkg.Listener report preflight.Report checking bool @@ -172,6 +208,13 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, cmd case checkDoneMsg: m.checking, m.report = false, msg.rep + case joinedMsg: + m.db[msg.i].joined = msg.r + if m.page == pWait && m.allJoined() { + m.page = pCheck + cmd := m.startCheck() + return m, cmd + } case initLineMsg: m.run.addLine(string(msg)) case initDoneMsg: @@ -189,8 +232,8 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking { m.page-- - if m.page == pMachines && m.choice == 0 { - m.page = pAdmin + if m.page == pWait { + m.page = pMachines } m.formErr = "" return m, nil @@ -303,23 +346,14 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { n := 2*mfCount + 1 switch k.Type { case tea.KeyTab, tea.KeyDown: - m.mfocus = (m.mfocus + 1) % n - if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent { - m.mfocus++ - } - m.focusMachine() + m.step(1, n) return m, nil case tea.KeyShiftTab, tea.KeyUp: - m.mfocus = (m.mfocus - 1 + n) % n - if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent { - m.mfocus-- - } - m.focusMachine() + m.step(-1, n) return m, nil case tea.KeyEnter: if m.mfocus < 2*mfCount { - m.mfocus++ - m.focusMachine() + m.step(1, n) return m, nil } if err := m.validateMachines(); err != "" { @@ -327,6 +361,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { return m, nil } m.formErr = "" + if cmd, waiting := m.startJoins(); waiting { + m.page = pWait + return m, cmd + } m.page = pCheck cmd := m.startCheck() return m, cmd @@ -342,6 +380,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { case "right", "l", " ": d.auth = (d.auth + 1) % authCount } + if d.auth != authJoin && d.joined != nil { + _ = d.joined.Close() + d.joined, d.code, d.arrive = nil, "", nil + } d.secret.SetValue("") d.secret.Mask = d.auth == authPassword d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth] @@ -353,9 +395,60 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { return m, cmd } +// step moves the focus over the fields that are shown. +func (m *setupModel) step(d, n int) { + for range n { + m.mfocus = (m.mfocus + d + n) % n + if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) { + break + } + } + m.focusMachine() +} + +func (m setupModel) allJoined() bool { + for _, d := range m.db { + if d.auth == authJoin && d.joined == nil { + return false + } + } + return true +} + +// startJoins listens for the machines that join with a code, makes their +// codes, and waits for them; it says whether there is anything to wait for. +func (m *setupModel) startJoins() (tea.Cmd, bool) { + var cmds []tea.Cmd + for i := range m.db { + d := &m.db[i] + if d.auth != authJoin || d.joined != nil { + continue + } + if m.listener == nil { + l, err := joinpkg.Listen(fmt.Sprintf(":%d", joinpkg.DefaultPort), + fmt.Sprintf("%s:%d", m.thisAddr(), joinpkg.DefaultPort), time.Hour) + if err != nil { + m.formErr = err.Error() + return nil, false + } + m.listener = l + } + if d.arrive == nil { + c, ch := m.listener.Code("database") + d.code, d.arrive = c.String(), ch + } + ch, i := d.arrive, i + cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} }) + } + return tea.Batch(cmds...), len(cmds) > 0 +} + func (m setupModel) validateMachines() string { seen := map[string]bool{m.thisAddr(): true, m.host: true} for i, d := range m.db { + if d.auth == authJoin { + continue + } addr := strings.TrimSpace(d.addr.Value()) switch { case addr == "": @@ -375,15 +468,23 @@ func (m setupModel) validateMachines() string { // startCheck connects to the database machines and runs the checks on all // three, this one included. func (m *setupModel) startCheck() tea.Cmd { - for _, r := range m.remotes { - _ = r.Close() + for _, c := range m.conns { + if c.ssh != nil { + _ = c.ssh.Close() + } } - m.remotes = nil + m.conns = nil machines := []preflight.Machine{{ Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{}, Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness}, }} for _, d := range m.db { + if d.auth == authJoin { + m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined}) + machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)", + Addr: d.joined.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: d.joined}) + continue + } g := &onboard.GoSSH{ Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()), UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(), @@ -394,13 +495,13 @@ func (m *setupModel) startCheck() tea.Cmd { case authPassword: g.Password = d.secret.Value() } - m.remotes = append(m.remotes, g) + m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g}) machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g}) } m.checking = true m.report = preflight.Report{} - remotes := m.remotes + conns := m.conns return func() tea.Msg { rep := preflight.Run(context.Background(), machines) // A sudo that asks for a password is fine with one given: the @@ -408,9 +509,10 @@ func (m *setupModel) startCheck() tea.Cmd { for i := range rep.Findings { f := &rep.Findings[i] if f.Check == "sudo" && f.Level == preflight.Warn { - for _, g := range remotes { + for _, c := range conns { + g := c.ssh switch { - case g.Addr != f.Machine: + case g == nil || g.Addr != f.Machine: case g.SudoPassword == "": f.Level = preflight.Fail f.Fix = "give this user's sudo password on the previous screen" @@ -430,6 +532,10 @@ func (m *setupModel) startCheck() tea.Cmd { func (m setupModel) targets() []string { var out []string for _, d := range m.db { + if d.joined != nil { + out = append(out, "root@"+d.joined.Addr) + continue + } out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value())) } return out @@ -473,7 +579,7 @@ func (m *setupModel) startRun() tea.Cmd { m.in = &in m.run = newInitModel(&in, nil, false) m.run.running, m.run.width, m.run.spin = true, m.width, m.spin - ref, remotes, logp, mu := m.ref, m.remotes, m.log, m.logMu + ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener return func() tea.Msg { send := func(l string) { mu.Lock() @@ -484,27 +590,38 @@ func (m *setupModel) startRun() tea.Cmd { in.out = &lineWriter{send: send} err := func() error { ctx := context.Background() - for _, g := range remotes { - fmt.Fprintf(in.out, " .. sudo on %s\n", g.Addr) - if err := g.PrepareSudo(ctx); err != nil { - return err + for _, c := range conns { + if c.ssh != nil { + fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr) + if err := c.ssh.PrepareSudo(ctx); err != nil { + return err + } } - out, err := g.Run(ctx, "hostname", nil) + out, err := c.remote.Run(ctx, "hostname", nil) if err != nil { - return fmt.Errorf("%s: %v", g.Addr, err) + return fmt.Errorf("%s: %v", c.addr, err) } - ip, err := ipv4Of(g.Addr) + ip, err := ipv4Of(c.addr) if err != nil { return err } in.patroniNodes = append(in.patroniNodes, pgha.Node{ - Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: g}) + Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote}) } return in.run(ctx) }() - for _, g := range remotes { - // The grant is taken back whatever happened. - _ = g.Close() + // Whatever happened: a sudo grant taken back, a joined machine's + // `llmgw join` told it is done, the port for joining closed. + for _, c := range conns { + if c.ssh != nil { + _ = c.ssh.Close() + } + if c.joined != nil { + _ = c.joined.Close() + } + } + if listener != nil { + _ = listener.Close() } return initDoneMsg{err} } @@ -520,6 +637,8 @@ func (m setupModel) View() string { b.WriteString(m.adminView()) case pMachines: b.WriteString(m.machinesView()) + case pWait: + b.WriteString(m.waitView()) case pCheck: b.WriteString(m.checkView()) case pPlan: @@ -548,9 +667,16 @@ func (m setupModel) crumbs() string { if m.choice == 0 { return steps[0] } + at := int(m.page) + switch { + case m.page == pWait: + at = int(pMachines) + case m.page > pWait: + at-- + } var out []string for i, s := range steps { - if setupPage(i) == m.page { + if i == at { s = lipgloss.NewStyle().Bold(true).Render(s) } out = append(out, s) @@ -609,8 +735,10 @@ func (m setupModel) machinesView() string { return " " + s } fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))) - fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View()) - fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View()) + if d.visible(mfAddr) { + fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View()) + } var auth []string for a := 0; a < authCount; a++ { if a == d.auth { @@ -620,10 +748,16 @@ func (m setupModel) machinesView() string { } } fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " ")) - if d.auth != authAgent { + if d.visible(mfSecret) { fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View()) } - fmt.Fprintf(&b, " %s %s\n\n", label(mfSudo, "sudo password"), d.sudo.View()) + if d.visible(mfSudo) { + fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "sudo password"), d.sudo.View()) + } + if d.auth == authJoin { + b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next, and dials out to this machine.") + "\n") + } + b.WriteString("\n") } btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines") if m.mfocus == 2*mfCount { @@ -636,6 +770,29 @@ func (m setupModel) machinesView() string { return b.String() } +func (m setupModel) waitView() string { + var b strings.Builder + fmt.Fprintf(&b, "On each machine below, as root. It dials this machine on port %d; nothing dials in.\n\n", joinpkg.DefaultPort) + for i, d := range m.db { + if d.auth != authJoin { + continue + } + title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)) + if d.joined != nil { + fmt.Fprintf(&b, "%s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr) + continue + } + fmt.Fprintf(&b, "%s %s waiting\n", title, m.spin.View()) + b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n") + for _, l := range joinInstall() { + b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n") + } + b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n") + } + b.WriteString(sDim.Render("Each code is for one machine and good for an hour. esc back · ctrl+c to leave") + "\n") + return b.String() +} + func (m setupModel) checkView() string { var b strings.Builder if m.checking { @@ -691,7 +848,14 @@ func (m setupModel) planView() string { if granted { b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n") } - b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n\n") + b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n") + for _, d := range m.db { + if d.joined != nil { + b.WriteString(sDim.Render("(a machine that joined with a code needs ssh for that: the code is for these screens)") + "\n") + break + } + } + b.WriteString("\n") b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n") return b.String() } diff --git a/internal/join/join.go b/internal/join/join.go new file mode 100644 index 0000000..5e39017 --- /dev/null +++ b/internal/join/join.go @@ -0,0 +1,302 @@ +// Package join lets a machine be set up without anything logging into it. +// +// The machine doing the setup — running as root, holding the authority — +// listens for a while and hands out one code per machine it expects. On the +// other machine `sudo llmgw join ` dials out to it, checks it is talking +// to the listener the code names, proves itself with the code's secret, and +// from then on runs what it is sent, as ssh would, until the setup closes +// the connection. To the setup it is one more onboard.Remote: preflight, +// Patroni and node onboarding run over it unchanged. +// +// For sites where nothing may log into a machine from outside: the +// connection is outbound from the machine being set up, no password is +// typed anywhere, and the code is good for one connection and a limited time. +package join + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math/big" + "net" + "strings" + "sync" + "time" +) + +// DefaultPort is where a setup listens for machines joining it. +const DefaultPort = 7444 + +// codePrefix marks a join code, and its version. +const codePrefix = "llmgw-join1-" + +// Code is what `llmgw join` is given: where to dial, how to know it reached +// the right place, and the secret that admits it. +// +// Short, because somebody copies it from one screen into another: the first +// version was JSON and over 250 characters, which a terminal cut off at its +// edge. Packed it is under 90. +type Code struct { + Addr string + // Fingerprint is the first 16 bytes of the SHA-256 of the listener's + // certificate: 128 bits, more than enough to find nothing else by. + Fingerprint [16]byte + Secret [16]byte + Expires int64 + // Role says what the machine is being set up as, for its operator to + // read before it runs: "database", "node" or "control plane". + Role string +} + +var roles = []string{"", "database", "node", "control plane"} + +// String is the code as typed: one word, safe in a shell. +func (c Code) String() string { + b := make([]byte, 0, 40+len(c.Addr)) + b = append(b, c.Fingerprint[:]...) + b = append(b, c.Secret[:]...) + b = binary.BigEndian.AppendUint32(b, uint32(c.Expires)) + role := byte(0) + for i, r := range roles { + if r == c.Role { + role = byte(i) + } + } + b = append(b, role) + b = append(b, c.Addr...) + return codePrefix + base64.RawURLEncoding.EncodeToString(b) +} + +// Parse reads a code back. +func Parse(s string) (Code, error) { + raw, ok := strings.CutPrefix(strings.TrimSpace(s), codePrefix) + if !ok { + return Code{}, errors.New("not a join code: it starts " + codePrefix) + } + b, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil || len(b) < 16+16+4+1+3 { + return Code{}, errors.New("the join code is damaged: copy it whole") + } + var c Code + copy(c.Fingerprint[:], b[:16]) + copy(c.Secret[:], b[16:32]) + c.Expires = int64(binary.BigEndian.Uint32(b[32:36])) + if int(b[36]) < len(roles) { + c.Role = roles[b[36]] + } + c.Addr = string(b[37:]) + if _, _, err := net.SplitHostPort(c.Addr); err != nil { + return Code{}, errors.New("the join code is damaged: copy it whole") + } + if time.Now().Unix() > c.Expires { + return Code{}, fmt.Errorf("the join code expired at %s; ask for a new one", + time.Unix(c.Expires, 0).Format(time.RFC3339)) + } + return c, nil +} + +// The wire: one JSON object a line, each way. +type hello struct { + Secret []byte `json:"secret"` + Hostname string `json:"hostname"` + // Addr is the address the machine dialled out from: how the others + // reach it, when nothing else says. + Addr string `json:"addr"` +} + +type request struct { + Script string `json:"script,omitempty"` + Stdin []byte `json:"stdin,omitempty"` + Bye bool `json:"bye,omitempty"` +} + +type response struct { + Stdout []byte `json:"stdout,omitempty"` + Stderr []byte `json:"stderr,omitempty"` + Exit int `json:"exit"` + Error string `json:"error,omitempty"` +} + +// Listener waits for machines to join. +type Listener struct { + ln net.Listener + addr string + fingerprint [16]byte + ttl time.Duration + + mu sync.Mutex + pending map[[16]byte]chan *Remote // by secret +} + +// Listen starts listening on bind (":7444") and advertises addr ("10.0.0.5:7444") +// in its codes. Codes are good for ttl. +func Listen(bind, addr string, ttl time.Duration) (*Listener, error) { + cert, fp, err := sessionCert() + if err != nil { + return nil, err + } + ln, err := tls.Listen("tcp", bind, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}) + if err != nil { + return nil, fmt.Errorf("listening for machines to join on %s: %w", bind, err) + } + l := &Listener{ln: ln, addr: addr, fingerprint: fp, ttl: ttl, pending: map[[16]byte]chan *Remote{}} + go l.accept() + return l, nil +} + +// sessionCert is a certificate for this listener alone, trusted by the +// fingerprint in its codes rather than by any authority: it lives as long as +// the setup does. +func sessionCert() (tls.Certificate, [16]byte, error) { + var fp [16]byte + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fp, err + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: "llmgw setup"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(48 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fp, err + } + sum := sha256.Sum256(der) + copy(fp[:], sum[:16]) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, fp, nil +} + +// Code makes a code for one machine, and the channel its connection arrives +// on. +func (l *Listener) Code(role string) (Code, <-chan *Remote) { + var secret [16]byte + _, _ = rand.Read(secret[:]) + ch := make(chan *Remote, 1) + l.mu.Lock() + l.pending[secret] = ch + l.mu.Unlock() + return Code{Addr: l.addr, Fingerprint: l.fingerprint, Secret: secret, + Expires: time.Now().Add(l.ttl).Unix(), Role: role}, ch +} + +// Close stops listening; machines already joined stay connected until their +// Remote is closed. +func (l *Listener) Close() error { return l.ln.Close() } + +func (l *Listener) accept() { + for { + c, err := l.ln.Accept() + if err != nil { + return + } + go l.admit(c) + } +} + +func (l *Listener) admit(c net.Conn) { + _ = c.SetDeadline(time.Now().Add(30 * time.Second)) + r := bufio.NewReader(c) + var h hello + line, err := r.ReadBytes('\n') + if err != nil || json.Unmarshal(line, &h) != nil { + _ = c.Close() + return + } + l.mu.Lock() + var ch chan *Remote + for secret, p := range l.pending { + if subtle.ConstantTimeCompare(secret[:], h.Secret) == 1 { + ch = p + // One machine a code: whoever comes second is turned away. + delete(l.pending, secret) + } + } + l.mu.Unlock() + if ch == nil { + _, _ = c.Write([]byte(`{"error":"this code is not one the setup is waiting for, or it was used already"}` + "\n")) + _ = c.Close() + return + } + _ = c.SetDeadline(time.Time{}) + if _, err := c.Write([]byte("{}\n")); err != nil { + _ = c.Close() + return + } + addr := h.Addr + if addr == "" { + addr, _, _ = net.SplitHostPort(c.RemoteAddr().String()) + } + ch <- &Remote{conn: c, r: r, Hostname: h.Hostname, Addr: addr} +} + +// Remote is a joined machine, as the setup runs things on it. +type Remote struct { + Hostname string + Addr string + + mu sync.Mutex + conn net.Conn + r *bufio.Reader +} + +// Run runs script with sh on the machine, as root. +func (m *Remote) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.conn == nil { + return nil, fmt.Errorf("%s is no longer connected", m.Hostname) + } + if dl, ok := ctx.Deadline(); ok { + _ = m.conn.SetDeadline(dl) + defer m.conn.SetDeadline(time.Time{}) + } + b, _ := json.Marshal(request{Script: script, Stdin: stdin}) + if _, err := m.conn.Write(append(b, '\n')); err != nil { + return nil, fmt.Errorf("%s: %w", m.Hostname, err) + } + line, err := m.r.ReadBytes('\n') + if err != nil { + return nil, fmt.Errorf("%s went away: %w", m.Hostname, err) + } + var resp response + if err := json.Unmarshal(line, &resp); err != nil { + return nil, fmt.Errorf("%s answered something unreadable", m.Hostname) + } + if resp.Error != "" { + return append(resp.Stdout, resp.Stderr...), errors.New(resp.Error) + } + if resp.Exit != 0 { + return append(resp.Stdout, resp.Stderr...), fmt.Errorf("exit status %d", resp.Exit) + } + return resp.Stdout, nil +} + +// Close tells the machine the setup is done, which ends its `llmgw join`. +func (m *Remote) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + if m.conn == nil { + return nil + } + b, _ := json.Marshal(request{Bye: true}) + _, _ = m.conn.Write(append(b, '\n')) + err := m.conn.Close() + m.conn = nil + return err +} diff --git a/internal/join/join_test.go b/internal/join/join_test.go new file mode 100644 index 0000000..f8e6169 --- /dev/null +++ b/internal/join/join_test.go @@ -0,0 +1,94 @@ +package join + +import ( + "context" + "strings" + "testing" + "time" +) + +func listen(t *testing.T) *Listener { + t.Helper() + l, err := Listen("127.0.0.1:0", "", time.Hour) + if err != nil { + t.Fatal(err) + } + l.addr = l.ln.Addr().String() + t.Cleanup(func() { _ = l.Close() }) + return l +} + +func TestAJoinedMachineRunsWhatTheSetupSendsUntilItIsDone(t *testing.T) { + l := listen(t) + code, arrived := l.Code("database") + parsed, err := Parse(code.String()) + if err != nil || parsed.Role != "database" { + t.Fatalf("%v %+v", err, parsed) + } + var said []string + served := make(chan error, 1) + go func() { served <- Serve(context.Background(), parsed, func(s string) { said = append(said, s) }) }() + + var m *Remote + select { + case m = <-arrived: + case err := <-served: + t.Fatalf("join ended before arriving: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("the machine never arrived") + } + if m.Hostname == "" || m.Addr != "127.0.0.1" { + t.Errorf("arrived as %q from %q", m.Hostname, m.Addr) + } + out, err := m.Run(context.Background(), "set -e\ntr a-z A-Z", []byte("hello\n")) + if err != nil || string(out) != "HELLO\n" { + t.Fatalf("%q, %v", out, err) + } + if out, err := m.Run(context.Background(), "echo nope >&2; exit 3", nil); err == nil || !strings.Contains(string(out), "nope") { + t.Errorf("a failing script: %q, %v", out, err) + } + _ = m.Close() + if err := <-served; err != nil { + t.Errorf("join did not end cleanly when the setup was done: %v", err) + } + if len(said) < 2 || !strings.Contains(strings.Join(said, "\n"), "tr a-z A-Z") { + t.Errorf("the machine's operator was not told what ran: %q", said) + } + + // The code was for one machine. + if err := Serve(context.Background(), parsed, func(string) {}); err == nil || !strings.Contains(err.Error(), "used already") { + t.Errorf("a code used twice: %v", err) + } +} + +func TestAJoinRefusesAListenerThatIsNotTheOneInTheCode(t *testing.T) { + l := listen(t) + code, _ := l.Code("node") + other := listen(t) + code.Addr = other.addr // what the code names is elsewhere + err := Serve(context.Background(), code, func(string) {}) + if err == nil || !strings.Contains(err.Error(), "not the setup") { + t.Errorf("joined a stranger: %v", err) + } +} + +func TestAnExpiredOrDamagedCodeIsRefusedBeforeDialling(t *testing.T) { + c := Code{Addr: "10.0.0.5:7444", Role: "node", Expires: time.Now().Add(-time.Minute).Unix()} + if _, err := Parse(c.String()); err == nil || !strings.Contains(err.Error(), "expired") { + t.Errorf("expired: %v", err) + } + if _, err := Parse(c.String()[:30]); err == nil { + t.Error("a truncated code was read") + } + if _, err := Parse("ssh-rsa AAAA"); err == nil { + t.Error("something else was read as a code") + } + // Short enough to copy from a screen in one line. + c.Expires = time.Now().Add(time.Hour).Unix() + if n := len(c.String()); n > 100 { + t.Errorf("a code is %d characters", n) + } + if back, err := Parse(c.String()); err != nil || back.Addr != c.Addr || back.Role != "node" { + t.Errorf("%+v, %v", back, err) + } +} diff --git a/internal/join/serve.go b/internal/join/serve.go new file mode 100644 index 0000000..e31c324 --- /dev/null +++ b/internal/join/serve.go @@ -0,0 +1,126 @@ +package join + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "crypto/subtle" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/exec" + "strings" + "time" +) + +// Serve dials the setup a code names and runs what it sends until it says it +// is done. Each script is reported to log by its first line, so whoever ran +// `llmgw join` sees what is being done to their machine. +func Serve(ctx context.Context, c Code, log func(string)) error { + d := tls.Dialer{ + NetDialer: &net.Dialer{Timeout: 10 * time.Second}, + Config: &tls.Config{ + MinVersion: tls.VersionTLS13, + // Trusted by the fingerprint in the code, not by an authority: + // the listener's certificate is made for this setup alone. + InsecureSkipVerify: true, //nolint:gosec // verified below, by fingerprint + }, + } + want := c.Fingerprint[:] + d.Config.VerifyPeerCertificate = func(raw [][]byte, _ [][]*x509.Certificate) error { + if len(raw) == 0 { + return errors.New("the setup sent no certificate") + } + sum := sha256.Sum256(raw[0]) + if subtle.ConstantTimeCompare(sum[:16], want) != 1 { + return errors.New("the machine at that address is not the setup this code is for") + } + return nil + } + conn, err := d.DialContext(ctx, "tcp", c.Addr) + if err != nil { + return fmt.Errorf("cannot reach the setup at %s: %w; it listens there only while it waits for this machine", c.Addr, err) + } + defer conn.Close() + + host, _ := os.Hostname() + local, _, _ := net.SplitHostPort(conn.LocalAddr().String()) + b, _ := json.Marshal(hello{Secret: c.Secret[:], Hostname: host, Addr: local}) + if _, err := conn.Write(append(b, '\n')); err != nil { + return err + } + r := bufio.NewReader(conn) + line, err := r.ReadBytes('\n') + if err != nil { + return fmt.Errorf("the setup closed the connection: %w", err) + } + var ack response + if err := json.Unmarshal(line, &ack); err != nil { + return errors.New("the setup answered something unreadable") + } + if ack.Error != "" { + return errors.New(ack.Error) + } + log(fmt.Sprintf("joined the setup at %s; it is setting this machine up", c.Addr)) + + for { + line, err := r.ReadBytes('\n') + if err != nil { + return fmt.Errorf("the setup went away before it said it was done: %w", err) + } + var req request + if err := json.Unmarshal(line, &req); err != nil { + return errors.New("the setup sent something unreadable") + } + if req.Bye { + log("the setup is done with this machine") + return nil + } + log(summary(req.Script)) + resp := run(ctx, req) + out, _ := json.Marshal(resp) + if _, err := conn.Write(append(out, '\n')); err != nil { + return err + } + } +} + +// run runs one script as the setup asked. +func run(ctx context.Context, req request) response { + cmd := exec.CommandContext(ctx, "sh", "-c", req.Script) + if req.Stdin != nil { + cmd.Stdin = bytes.NewReader(req.Stdin) + } + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + resp := response{Stdout: stdout.Bytes(), Stderr: stderr.Bytes()} + var exit *exec.ExitError + switch { + case errors.As(err, &exit): + resp.Exit = exit.ExitCode() + case err != nil: + resp.Error = err.Error() + } + return resp +} + +// summary is a script's first line that says something. +func summary(script string) string { + for _, l := range strings.Split(script, "\n") { + l = strings.TrimSpace(l) + if l == "" || l == "set -e" || strings.HasPrefix(l, "#") { + continue + } + if len(l) > 100 { + l = l[:100] + "…" + } + return " " + l + } + return " (an empty script)" +}