Let a machine join a setup by code, dialling out, with nothing logging in

The setup screens can reach a database machine by a join code instead of
ssh: they listen on 7444 and show a short code; llmgw join dials out,
checks the setup by fingerprint, admits itself with the code's secret, and
runs what the setup sends until it is done. A joined machine is one more
onboard.Remote, so checks and the Patroni install run over it unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 21:49:15 +02:00
1 parent 38f2671ed9
commit f5c79052b1
8 files changed
+810 -43

No files matched your search

+61
View File
@@ -0,0 +1,61 @@
package cli
import (
"errors"
"fmt"
"os"
"runtime"
"github.com/spf13/cobra"
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
func newJoinCmd() *cobra.Command {
return &cobra.Command{
Use: "join <code>",
Short: "Let a setup on another machine set this one up, with the code it gave",
Long: "For a machine nothing may log into: the setup screens (sudo llmgw on\n" +
"the control plane) show a code for each machine they expect. Run this\n" +
"with it, as root, on that machine. It dials out to the setup — nothing\n" +
"dials in — checks it is the setup the code was made by, and runs what\n" +
"the setup sends until it is done: the same checks and installs ssh\n" +
"would carry. Each step is printed here as it runs.\n\n" +
"A code is good for one machine, for an hour, and only while the setup\n" +
"that made it is waiting.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if runtime.GOOS != "linux" {
return fmt.Errorf("join sets up a Linux machine; this is %s", runtime.GOOS)
}
if os.Geteuid() != 0 {
return errors.New("run it as root — sudo llmgw join … — since the setup installs software here")
}
code, err := joinpkg.Parse(args[0])
if err != nil {
return err
}
out := cmd.OutOrStdout()
role := code.Role
if role == "" {
role = "part of a cluster"
}
fmt.Fprintf(out, "Joining the setup at %s, as %s. It runs as root here until it is done.\n", code.Addr, role)
return joinpkg.Serve(cmd.Context(), code, func(s string) { fmt.Fprintln(out, s) })
},
}
}
// joinInstall is what to run on a machine that has no llmgw yet, to fetch
// this version for its architecture: short lines, since it is copied from a
// screen that cuts long ones off.
func joinInstall() []string {
v := version.Version
return []string{
`a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`,
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v),
"sudo chmod 755 /usr/local/bin/llmgw",
}
}
+1
View File
@@ -57,6 +57,7 @@ func Root() *cobra.Command {
newNodeCmd(),
newSelfUpgradeCmd(),
newReleaseCmd(),
newJoinCmd(),
)
return root
}
+206 -42
View File
@@ -8,10 +8,12 @@ import (
"os"
"strings"
"sync"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
@@ -34,6 +36,9 @@ const (
pWelcome setupPage = iota
pAdmin
pMachines
// pWait shows the codes for machines joining rather than logged into,
// until they have all dialled in.
pWait
pCheck
pPlan
pRun
@@ -44,10 +49,13 @@ const (
authAgent = iota
authKey
authPassword
// authJoin is no login at all: the machine runs `llmgw join` with a code
// shown here and dials out to this one.
authJoin
authCount
)
var authNames = [authCount]string{"ssh agent", "key file", "password"}
var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"}
// Fields of one database machine, in order.
const (
@@ -62,6 +70,30 @@ const (
type dbMachine struct {
addr, user, secret, sudo textField
auth int
// For a machine joining: its code, and it once it has dialled in.
code string
joined *joinpkg.Remote
arrive <-chan *joinpkg.Remote
}
// visible reports whether a field is shown, and so can be moved to: a joining
// machine is asked nothing but how it joins.
func (d dbMachine) visible(f int) bool {
switch {
case d.auth == authJoin:
return f == mfAuth
case f == mfSecret:
return d.auth != authAgent
}
return true
}
// dbConn is a database machine as it is reached, whichever way.
type dbConn struct {
remote onboard.Remote
addr string
ssh *onboard.GoSSH
joined *joinpkg.Remote
}
func newDBMachine(n int) dbMachine {
@@ -100,7 +132,10 @@ func (d *dbMachine) field(i int) *textField {
// Messages from the work behind the screens.
type (
checkDoneMsg struct{ rep preflight.Report }
setupRunMsg struct{}
joinedMsg struct {
i int
r *joinpkg.Remote
}
)
// progRef lets work started from Update send to the program it runs in.
@@ -120,7 +155,8 @@ type setupModel struct {
db [2]dbMachine
mfocus int // over both machines' fields, then the check button
remotes []*onboard.GoSSH
conns []dbConn
listener *joinpkg.Listener
report preflight.Report
checking bool
@@ -172,6 +208,13 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, cmd
case checkDoneMsg:
m.checking, m.report = false, msg.rep
case joinedMsg:
m.db[msg.i].joined = msg.r
if m.page == pWait && m.allJoined() {
m.page = pCheck
cmd := m.startCheck()
return m, cmd
}
case initLineMsg:
m.run.addLine(string(msg))
case initDoneMsg:
@@ -189,8 +232,8 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
m.page--
if m.page == pMachines && m.choice == 0 {
m.page = pAdmin
if m.page == pWait {
m.page = pMachines
}
m.formErr = ""
return m, nil
@@ -303,23 +346,14 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
n := 2*mfCount + 1
switch k.Type {
case tea.KeyTab, tea.KeyDown:
m.mfocus = (m.mfocus + 1) % n
if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent {
m.mfocus++
}
m.focusMachine()
m.step(1, n)
return m, nil
case tea.KeyShiftTab, tea.KeyUp:
m.mfocus = (m.mfocus - 1 + n) % n
if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent {
m.mfocus--
}
m.focusMachine()
m.step(-1, n)
return m, nil
case tea.KeyEnter:
if m.mfocus < 2*mfCount {
m.mfocus++
m.focusMachine()
m.step(1, n)
return m, nil
}
if err := m.validateMachines(); err != "" {
@@ -327,6 +361,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, nil
}
m.formErr = ""
if cmd, waiting := m.startJoins(); waiting {
m.page = pWait
return m, cmd
}
m.page = pCheck
cmd := m.startCheck()
return m, cmd
@@ -342,6 +380,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case "right", "l", " ":
d.auth = (d.auth + 1) % authCount
}
if d.auth != authJoin && d.joined != nil {
_ = d.joined.Close()
d.joined, d.code, d.arrive = nil, "", nil
}
d.secret.SetValue("")
d.secret.Mask = d.auth == authPassword
d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth]
@@ -353,9 +395,60 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, cmd
}
// step moves the focus over the fields that are shown.
func (m *setupModel) step(d, n int) {
for range n {
m.mfocus = (m.mfocus + d + n) % n
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
break
}
}
m.focusMachine()
}
func (m setupModel) allJoined() bool {
for _, d := range m.db {
if d.auth == authJoin && d.joined == nil {
return false
}
}
return true
}
// startJoins listens for the machines that join with a code, makes their
// codes, and waits for them; it says whether there is anything to wait for.
func (m *setupModel) startJoins() (tea.Cmd, bool) {
var cmds []tea.Cmd
for i := range m.db {
d := &m.db[i]
if d.auth != authJoin || d.joined != nil {
continue
}
if m.listener == nil {
l, err := joinpkg.Listen(fmt.Sprintf(":%d", joinpkg.DefaultPort),
fmt.Sprintf("%s:%d", m.thisAddr(), joinpkg.DefaultPort), time.Hour)
if err != nil {
m.formErr = err.Error()
return nil, false
}
m.listener = l
}
if d.arrive == nil {
c, ch := m.listener.Code("database")
d.code, d.arrive = c.String(), ch
}
ch, i := d.arrive, i
cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} })
}
return tea.Batch(cmds...), len(cmds) > 0
}
func (m setupModel) validateMachines() string {
seen := map[string]bool{m.thisAddr(): true, m.host: true}
for i, d := range m.db {
if d.auth == authJoin {
continue
}
addr := strings.TrimSpace(d.addr.Value())
switch {
case addr == "":
@@ -375,15 +468,23 @@ func (m setupModel) validateMachines() string {
// startCheck connects to the database machines and runs the checks on all
// three, this one included.
func (m *setupModel) startCheck() tea.Cmd {
for _, r := range m.remotes {
_ = r.Close()
for _, c := range m.conns {
if c.ssh != nil {
_ = c.ssh.Close()
}
}
m.remotes = nil
m.conns = nil
machines := []preflight.Machine{{
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
}}
for _, d := range m.db {
if d.auth == authJoin {
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
Addr: d.joined.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: d.joined})
continue
}
g := &onboard.GoSSH{
Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()),
UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(),
@@ -394,13 +495,13 @@ func (m *setupModel) startCheck() tea.Cmd {
case authPassword:
g.Password = d.secret.Value()
}
m.remotes = append(m.remotes, g)
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g})
}
m.checking = true
m.report = preflight.Report{}
remotes := m.remotes
conns := m.conns
return func() tea.Msg {
rep := preflight.Run(context.Background(), machines)
// A sudo that asks for a password is fine with one given: the
@@ -408,9 +509,10 @@ func (m *setupModel) startCheck() tea.Cmd {
for i := range rep.Findings {
f := &rep.Findings[i]
if f.Check == "sudo" && f.Level == preflight.Warn {
for _, g := range remotes {
for _, c := range conns {
g := c.ssh
switch {
case g.Addr != f.Machine:
case g == nil || g.Addr != f.Machine:
case g.SudoPassword == "":
f.Level = preflight.Fail
f.Fix = "give this user's sudo password on the previous screen"
@@ -430,6 +532,10 @@ func (m *setupModel) startCheck() tea.Cmd {
func (m setupModel) targets() []string {
var out []string
for _, d := range m.db {
if d.joined != nil {
out = append(out, "root@"+d.joined.Addr)
continue
}
out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value()))
}
return out
@@ -473,7 +579,7 @@ func (m *setupModel) startRun() tea.Cmd {
m.in = &in
m.run = newInitModel(&in, nil, false)
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
ref, remotes, logp, mu := m.ref, m.remotes, m.log, m.logMu
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
return func() tea.Msg {
send := func(l string) {
mu.Lock()
@@ -484,27 +590,38 @@ func (m *setupModel) startRun() tea.Cmd {
in.out = &lineWriter{send: send}
err := func() error {
ctx := context.Background()
for _, g := range remotes {
fmt.Fprintf(in.out, " .. sudo on %s\n", g.Addr)
if err := g.PrepareSudo(ctx); err != nil {
return err
for _, c := range conns {
if c.ssh != nil {
fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr)
if err := c.ssh.PrepareSudo(ctx); err != nil {
return err
}
}
out, err := g.Run(ctx, "hostname", nil)
out, err := c.remote.Run(ctx, "hostname", nil)
if err != nil {
return fmt.Errorf("%s: %v", g.Addr, err)
return fmt.Errorf("%s: %v", c.addr, err)
}
ip, err := ipv4Of(g.Addr)
ip, err := ipv4Of(c.addr)
if err != nil {
return err
}
in.patroniNodes = append(in.patroniNodes, pgha.Node{
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: g})
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote})
}
return in.run(ctx)
}()
for _, g := range remotes {
// The grant is taken back whatever happened.
_ = g.Close()
// Whatever happened: a sudo grant taken back, a joined machine's
// `llmgw join` told it is done, the port for joining closed.
for _, c := range conns {
if c.ssh != nil {
_ = c.ssh.Close()
}
if c.joined != nil {
_ = c.joined.Close()
}
}
if listener != nil {
_ = listener.Close()
}
return initDoneMsg{err}
}
@@ -520,6 +637,8 @@ func (m setupModel) View() string {
b.WriteString(m.adminView())
case pMachines:
b.WriteString(m.machinesView())
case pWait:
b.WriteString(m.waitView())
case pCheck:
b.WriteString(m.checkView())
case pPlan:
@@ -548,9 +667,16 @@ func (m setupModel) crumbs() string {
if m.choice == 0 {
return steps[0]
}
at := int(m.page)
switch {
case m.page == pWait:
at = int(pMachines)
case m.page > pWait:
at--
}
var out []string
for i, s := range steps {
if setupPage(i) == m.page {
if i == at {
s = lipgloss.NewStyle().Bold(true).Render(s)
}
out = append(out, s)
@@ -609,8 +735,10 @@ func (m setupModel) machinesView() string {
return " " + s
}
fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View())
if d.visible(mfAddr) {
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View())
}
var auth []string
for a := 0; a < authCount; a++ {
if a == d.auth {
@@ -620,10 +748,16 @@ func (m setupModel) machinesView() string {
}
}
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " "))
if d.auth != authAgent {
if d.visible(mfSecret) {
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View())
}
fmt.Fprintf(&b, " %s %s\n\n", label(mfSudo, "sudo password"), d.sudo.View())
if d.visible(mfSudo) {
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "sudo password"), d.sudo.View())
}
if d.auth == authJoin {
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next, and dials out to this machine.") + "\n")
}
b.WriteString("\n")
}
btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines")
if m.mfocus == 2*mfCount {
@@ -636,6 +770,29 @@ func (m setupModel) machinesView() string {
return b.String()
}
func (m setupModel) waitView() string {
var b strings.Builder
fmt.Fprintf(&b, "On each machine below, as root. It dials this machine on port %d; nothing dials in.\n\n", joinpkg.DefaultPort)
for i, d := range m.db {
if d.auth != authJoin {
continue
}
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
if d.joined != nil {
fmt.Fprintf(&b, "%s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
continue
}
fmt.Fprintf(&b, "%s %s waiting\n", title, m.spin.View())
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
for _, l := range joinInstall() {
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
}
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
}
b.WriteString(sDim.Render("Each code is for one machine and good for an hour. esc back · ctrl+c to leave") + "\n")
return b.String()
}
func (m setupModel) checkView() string {
var b strings.Builder
if m.checking {
@@ -691,7 +848,14 @@ func (m setupModel) planView() string {
if granted {
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n")
}
b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n\n")
b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n")
for _, d := range m.db {
if d.joined != nil {
b.WriteString(sDim.Render("(a machine that joined with a code needs ssh for that: the code is for these screens)") + "\n")
break
}
}
b.WriteString("\n")
b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n")
return b.String()
}
+302
View File
@@ -0,0 +1,302 @@
// Package join lets a machine be set up without anything logging into it.
//
// The machine doing the setup — running as root, holding the authority —
// listens for a while and hands out one code per machine it expects. On the
// other machine `sudo llmgw join <code>` dials out to it, checks it is talking
// to the listener the code names, proves itself with the code's secret, and
// from then on runs what it is sent, as ssh would, until the setup closes
// the connection. To the setup it is one more onboard.Remote: preflight,
// Patroni and node onboarding run over it unchanged.
//
// For sites where nothing may log into a machine from outside: the
// connection is outbound from the machine being set up, no password is
// typed anywhere, and the code is good for one connection and a limited time.
package join
import (
"bufio"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math/big"
"net"
"strings"
"sync"
"time"
)
// DefaultPort is where a setup listens for machines joining it.
const DefaultPort = 7444
// codePrefix marks a join code, and its version.
const codePrefix = "llmgw-join1-"
// Code is what `llmgw join` is given: where to dial, how to know it reached
// the right place, and the secret that admits it.
//
// Short, because somebody copies it from one screen into another: the first
// version was JSON and over 250 characters, which a terminal cut off at its
// edge. Packed it is under 90.
type Code struct {
Addr string
// Fingerprint is the first 16 bytes of the SHA-256 of the listener's
// certificate: 128 bits, more than enough to find nothing else by.
Fingerprint [16]byte
Secret [16]byte
Expires int64
// Role says what the machine is being set up as, for its operator to
// read before it runs: "database", "node" or "control plane".
Role string
}
var roles = []string{"", "database", "node", "control plane"}
// String is the code as typed: one word, safe in a shell.
func (c Code) String() string {
b := make([]byte, 0, 40+len(c.Addr))
b = append(b, c.Fingerprint[:]...)
b = append(b, c.Secret[:]...)
b = binary.BigEndian.AppendUint32(b, uint32(c.Expires))
role := byte(0)
for i, r := range roles {
if r == c.Role {
role = byte(i)
}
}
b = append(b, role)
b = append(b, c.Addr...)
return codePrefix + base64.RawURLEncoding.EncodeToString(b)
}
// Parse reads a code back.
func Parse(s string) (Code, error) {
raw, ok := strings.CutPrefix(strings.TrimSpace(s), codePrefix)
if !ok {
return Code{}, errors.New("not a join code: it starts " + codePrefix)
}
b, err := base64.RawURLEncoding.DecodeString(raw)
if err != nil || len(b) < 16+16+4+1+3 {
return Code{}, errors.New("the join code is damaged: copy it whole")
}
var c Code
copy(c.Fingerprint[:], b[:16])
copy(c.Secret[:], b[16:32])
c.Expires = int64(binary.BigEndian.Uint32(b[32:36]))
if int(b[36]) < len(roles) {
c.Role = roles[b[36]]
}
c.Addr = string(b[37:])
if _, _, err := net.SplitHostPort(c.Addr); err != nil {
return Code{}, errors.New("the join code is damaged: copy it whole")
}
if time.Now().Unix() > c.Expires {
return Code{}, fmt.Errorf("the join code expired at %s; ask for a new one",
time.Unix(c.Expires, 0).Format(time.RFC3339))
}
return c, nil
}
// The wire: one JSON object a line, each way.
type hello struct {
Secret []byte `json:"secret"`
Hostname string `json:"hostname"`
// Addr is the address the machine dialled out from: how the others
// reach it, when nothing else says.
Addr string `json:"addr"`
}
type request struct {
Script string `json:"script,omitempty"`
Stdin []byte `json:"stdin,omitempty"`
Bye bool `json:"bye,omitempty"`
}
type response struct {
Stdout []byte `json:"stdout,omitempty"`
Stderr []byte `json:"stderr,omitempty"`
Exit int `json:"exit"`
Error string `json:"error,omitempty"`
}
// Listener waits for machines to join.
type Listener struct {
ln net.Listener
addr string
fingerprint [16]byte
ttl time.Duration
mu sync.Mutex
pending map[[16]byte]chan *Remote // by secret
}
// Listen starts listening on bind (":7444") and advertises addr ("10.0.0.5:7444")
// in its codes. Codes are good for ttl.
func Listen(bind, addr string, ttl time.Duration) (*Listener, error) {
cert, fp, err := sessionCert()
if err != nil {
return nil, err
}
ln, err := tls.Listen("tcp", bind, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13})
if err != nil {
return nil, fmt.Errorf("listening for machines to join on %s: %w", bind, err)
}
l := &Listener{ln: ln, addr: addr, fingerprint: fp, ttl: ttl, pending: map[[16]byte]chan *Remote{}}
go l.accept()
return l, nil
}
// sessionCert is a certificate for this listener alone, trusted by the
// fingerprint in its codes rather than by any authority: it lives as long as
// the setup does.
func sessionCert() (tls.Certificate, [16]byte, error) {
var fp [16]byte
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return tls.Certificate{}, fp, err
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(time.Now().UnixNano()),
Subject: pkix.Name{CommonName: "llmgw setup"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(48 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return tls.Certificate{}, fp, err
}
sum := sha256.Sum256(der)
copy(fp[:], sum[:16])
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, fp, nil
}
// Code makes a code for one machine, and the channel its connection arrives
// on.
func (l *Listener) Code(role string) (Code, <-chan *Remote) {
var secret [16]byte
_, _ = rand.Read(secret[:])
ch := make(chan *Remote, 1)
l.mu.Lock()
l.pending[secret] = ch
l.mu.Unlock()
return Code{Addr: l.addr, Fingerprint: l.fingerprint, Secret: secret,
Expires: time.Now().Add(l.ttl).Unix(), Role: role}, ch
}
// Close stops listening; machines already joined stay connected until their
// Remote is closed.
func (l *Listener) Close() error { return l.ln.Close() }
func (l *Listener) accept() {
for {
c, err := l.ln.Accept()
if err != nil {
return
}
go l.admit(c)
}
}
func (l *Listener) admit(c net.Conn) {
_ = c.SetDeadline(time.Now().Add(30 * time.Second))
r := bufio.NewReader(c)
var h hello
line, err := r.ReadBytes('\n')
if err != nil || json.Unmarshal(line, &h) != nil {
_ = c.Close()
return
}
l.mu.Lock()
var ch chan *Remote
for secret, p := range l.pending {
if subtle.ConstantTimeCompare(secret[:], h.Secret) == 1 {
ch = p
// One machine a code: whoever comes second is turned away.
delete(l.pending, secret)
}
}
l.mu.Unlock()
if ch == nil {
_, _ = c.Write([]byte(`{"error":"this code is not one the setup is waiting for, or it was used already"}` + "\n"))
_ = c.Close()
return
}
_ = c.SetDeadline(time.Time{})
if _, err := c.Write([]byte("{}\n")); err != nil {
_ = c.Close()
return
}
addr := h.Addr
if addr == "" {
addr, _, _ = net.SplitHostPort(c.RemoteAddr().String())
}
ch <- &Remote{conn: c, r: r, Hostname: h.Hostname, Addr: addr}
}
// Remote is a joined machine, as the setup runs things on it.
type Remote struct {
Hostname string
Addr string
mu sync.Mutex
conn net.Conn
r *bufio.Reader
}
// Run runs script with sh on the machine, as root.
func (m *Remote) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.conn == nil {
return nil, fmt.Errorf("%s is no longer connected", m.Hostname)
}
if dl, ok := ctx.Deadline(); ok {
_ = m.conn.SetDeadline(dl)
defer m.conn.SetDeadline(time.Time{})
}
b, _ := json.Marshal(request{Script: script, Stdin: stdin})
if _, err := m.conn.Write(append(b, '\n')); err != nil {
return nil, fmt.Errorf("%s: %w", m.Hostname, err)
}
line, err := m.r.ReadBytes('\n')
if err != nil {
return nil, fmt.Errorf("%s went away: %w", m.Hostname, err)
}
var resp response
if err := json.Unmarshal(line, &resp); err != nil {
return nil, fmt.Errorf("%s answered something unreadable", m.Hostname)
}
if resp.Error != "" {
return append(resp.Stdout, resp.Stderr...), errors.New(resp.Error)
}
if resp.Exit != 0 {
return append(resp.Stdout, resp.Stderr...), fmt.Errorf("exit status %d", resp.Exit)
}
return resp.Stdout, nil
}
// Close tells the machine the setup is done, which ends its `llmgw join`.
func (m *Remote) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
if m.conn == nil {
return nil
}
b, _ := json.Marshal(request{Bye: true})
_, _ = m.conn.Write(append(b, '\n'))
err := m.conn.Close()
m.conn = nil
return err
}
+94
View File
@@ -0,0 +1,94 @@
package join
import (
"context"
"strings"
"testing"
"time"
)
func listen(t *testing.T) *Listener {
t.Helper()
l, err := Listen("127.0.0.1:0", "", time.Hour)
if err != nil {
t.Fatal(err)
}
l.addr = l.ln.Addr().String()
t.Cleanup(func() { _ = l.Close() })
return l
}
func TestAJoinedMachineRunsWhatTheSetupSendsUntilItIsDone(t *testing.T) {
l := listen(t)
code, arrived := l.Code("database")
parsed, err := Parse(code.String())
if err != nil || parsed.Role != "database" {
t.Fatalf("%v %+v", err, parsed)
}
var said []string
served := make(chan error, 1)
go func() { served <- Serve(context.Background(), parsed, func(s string) { said = append(said, s) }) }()
var m *Remote
select {
case m = <-arrived:
case err := <-served:
t.Fatalf("join ended before arriving: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("the machine never arrived")
}
if m.Hostname == "" || m.Addr != "127.0.0.1" {
t.Errorf("arrived as %q from %q", m.Hostname, m.Addr)
}
out, err := m.Run(context.Background(), "set -e\ntr a-z A-Z", []byte("hello\n"))
if err != nil || string(out) != "HELLO\n" {
t.Fatalf("%q, %v", out, err)
}
if out, err := m.Run(context.Background(), "echo nope >&2; exit 3", nil); err == nil || !strings.Contains(string(out), "nope") {
t.Errorf("a failing script: %q, %v", out, err)
}
_ = m.Close()
if err := <-served; err != nil {
t.Errorf("join did not end cleanly when the setup was done: %v", err)
}
if len(said) < 2 || !strings.Contains(strings.Join(said, "\n"), "tr a-z A-Z") {
t.Errorf("the machine's operator was not told what ran: %q", said)
}
// The code was for one machine.
if err := Serve(context.Background(), parsed, func(string) {}); err == nil || !strings.Contains(err.Error(), "used already") {
t.Errorf("a code used twice: %v", err)
}
}
func TestAJoinRefusesAListenerThatIsNotTheOneInTheCode(t *testing.T) {
l := listen(t)
code, _ := l.Code("node")
other := listen(t)
code.Addr = other.addr // what the code names is elsewhere
err := Serve(context.Background(), code, func(string) {})
if err == nil || !strings.Contains(err.Error(), "not the setup") {
t.Errorf("joined a stranger: %v", err)
}
}
func TestAnExpiredOrDamagedCodeIsRefusedBeforeDialling(t *testing.T) {
c := Code{Addr: "10.0.0.5:7444", Role: "node", Expires: time.Now().Add(-time.Minute).Unix()}
if _, err := Parse(c.String()); err == nil || !strings.Contains(err.Error(), "expired") {
t.Errorf("expired: %v", err)
}
if _, err := Parse(c.String()[:30]); err == nil {
t.Error("a truncated code was read")
}
if _, err := Parse("ssh-rsa AAAA"); err == nil {
t.Error("something else was read as a code")
}
// Short enough to copy from a screen in one line.
c.Expires = time.Now().Add(time.Hour).Unix()
if n := len(c.String()); n > 100 {
t.Errorf("a code is %d characters", n)
}
if back, err := Parse(c.String()); err != nil || back.Addr != c.Addr || back.Role != "node" {
t.Errorf("%+v, %v", back, err)
}
}
+126
View File
@@ -0,0 +1,126 @@
package join
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"strings"
"time"
)
// Serve dials the setup a code names and runs what it sends until it says it
// is done. Each script is reported to log by its first line, so whoever ran
// `llmgw join` sees what is being done to their machine.
func Serve(ctx context.Context, c Code, log func(string)) error {
d := tls.Dialer{
NetDialer: &net.Dialer{Timeout: 10 * time.Second},
Config: &tls.Config{
MinVersion: tls.VersionTLS13,
// Trusted by the fingerprint in the code, not by an authority:
// the listener's certificate is made for this setup alone.
InsecureSkipVerify: true, //nolint:gosec // verified below, by fingerprint
},
}
want := c.Fingerprint[:]
d.Config.VerifyPeerCertificate = func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("the setup sent no certificate")
}
sum := sha256.Sum256(raw[0])
if subtle.ConstantTimeCompare(sum[:16], want) != 1 {
return errors.New("the machine at that address is not the setup this code is for")
}
return nil
}
conn, err := d.DialContext(ctx, "tcp", c.Addr)
if err != nil {
return fmt.Errorf("cannot reach the setup at %s: %w; it listens there only while it waits for this machine", c.Addr, err)
}
defer conn.Close()
host, _ := os.Hostname()
local, _, _ := net.SplitHostPort(conn.LocalAddr().String())
b, _ := json.Marshal(hello{Secret: c.Secret[:], Hostname: host, Addr: local})
if _, err := conn.Write(append(b, '\n')); err != nil {
return err
}
r := bufio.NewReader(conn)
line, err := r.ReadBytes('\n')
if err != nil {
return fmt.Errorf("the setup closed the connection: %w", err)
}
var ack response
if err := json.Unmarshal(line, &ack); err != nil {
return errors.New("the setup answered something unreadable")
}
if ack.Error != "" {
return errors.New(ack.Error)
}
log(fmt.Sprintf("joined the setup at %s; it is setting this machine up", c.Addr))
for {
line, err := r.ReadBytes('\n')
if err != nil {
return fmt.Errorf("the setup went away before it said it was done: %w", err)
}
var req request
if err := json.Unmarshal(line, &req); err != nil {
return errors.New("the setup sent something unreadable")
}
if req.Bye {
log("the setup is done with this machine")
return nil
}
log(summary(req.Script))
resp := run(ctx, req)
out, _ := json.Marshal(resp)
if _, err := conn.Write(append(out, '\n')); err != nil {
return err
}
}
}
// run runs one script as the setup asked.
func run(ctx context.Context, req request) response {
cmd := exec.CommandContext(ctx, "sh", "-c", req.Script)
if req.Stdin != nil {
cmd.Stdin = bytes.NewReader(req.Stdin)
}
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err := cmd.Run()
resp := response{Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}
var exit *exec.ExitError
switch {
case errors.As(err, &exit):
resp.Exit = exit.ExitCode()
case err != nil:
resp.Error = err.Error()
}
return resp
}
// summary is a script's first line that says something.
func summary(script string) string {
for _, l := range strings.Split(script, "\n") {
l = strings.TrimSpace(l)
if l == "" || l == "set -e" || strings.HasPrefix(l, "#") {
continue
}
if len(l) > 100 {
l = l[:100] + "…"
}
return " " + l
}
return " (an empty script)"
}