Let a machine join a setup by code, dialling out, with nothing logging in

The setup screens can reach a database machine by a join code instead of
ssh: they listen on 7444 and show a short code; llmgw join dials out,
checks the setup by fingerprint, admits itself with the code's secret, and
runs what the setup sends until it is done. A joined machine is one more
onboard.Remote, so checks and the Patroni install run over it unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 21:49:15 +02:00
1 parent 38f2671ed9
commit f5c79052b1
8 files changed
+810 -43

No files matched your search

+17
View File
@@ -75,6 +75,23 @@ and a figure the driver does not give is left out rather than shown as zero.
each card and whose quota it counts against, for joining the GPU series to
models and tenants.
### Machines nothing may log into: `llmgw join`
For sites that allow no ssh into a machine, the setup screens offer a fourth
way to reach one: a join code. The setup listens on port 7444 while it waits
and shows a code for each such machine, with the commands that fetch llmgw
onto it; there, `sudo llmgw join <code>` dials out — nothing dials in —
checks by the certificate fingerprint in the code that it reached the setup
that made it, admits itself with the code's secret, and runs what the setup
sends, printing each step, until the setup says it is done. A code is good
for one machine and an hour, and is under 90 characters so it can be copied
whole from one screen into another.
To the setup a joined machine is one more way of running a script there:
the checks, PostgreSQL, Patroni and etcd go over it unchanged. Tried with one
machine joining by code and another logged into with a password, in one
setup.
### Setting up from nothing in screens: `sudo llmgw`
Run with nothing after it in a terminal, as root, `llmgw` opens full-screen
+3 -1
View File
@@ -91,7 +91,9 @@ CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw
plane, run `sudo llmgw` with nothing after it. With nothing installed it asks
whether this is one machine or three for high availability; for three, it asks
for the other two — address, user, and an ssh key, the ssh agent, or a
password, and the sudo password if sudo asks for one — checks all three before
password, and the sudo password if sudo asks for one; or, for a machine
nothing may log into, a join code to run there with `sudo llmgw join <code>`,
which dials out to this one on port 7444 — checks all three before
installing anything (system, room, clock, ports, internet, whether they reach
one another), shows the plan, and installs. Passwords are held in memory for
the run only; a sudo password lets the user sudo without one until the install
+61
View File
@@ -0,0 +1,61 @@
package cli
import (
"errors"
"fmt"
"os"
"runtime"
"github.com/spf13/cobra"
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
func newJoinCmd() *cobra.Command {
return &cobra.Command{
Use: "join <code>",
Short: "Let a setup on another machine set this one up, with the code it gave",
Long: "For a machine nothing may log into: the setup screens (sudo llmgw on\n" +
"the control plane) show a code for each machine they expect. Run this\n" +
"with it, as root, on that machine. It dials out to the setup — nothing\n" +
"dials in — checks it is the setup the code was made by, and runs what\n" +
"the setup sends until it is done: the same checks and installs ssh\n" +
"would carry. Each step is printed here as it runs.\n\n" +
"A code is good for one machine, for an hour, and only while the setup\n" +
"that made it is waiting.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if runtime.GOOS != "linux" {
return fmt.Errorf("join sets up a Linux machine; this is %s", runtime.GOOS)
}
if os.Geteuid() != 0 {
return errors.New("run it as root — sudo llmgw join … — since the setup installs software here")
}
code, err := joinpkg.Parse(args[0])
if err != nil {
return err
}
out := cmd.OutOrStdout()
role := code.Role
if role == "" {
role = "part of a cluster"
}
fmt.Fprintf(out, "Joining the setup at %s, as %s. It runs as root here until it is done.\n", code.Addr, role)
return joinpkg.Serve(cmd.Context(), code, func(s string) { fmt.Fprintln(out, s) })
},
}
}
// joinInstall is what to run on a machine that has no llmgw yet, to fetch
// this version for its architecture: short lines, since it is copied from a
// screen that cuts long ones off.
func joinInstall() []string {
v := version.Version
return []string{
`a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`,
fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v),
"sudo chmod 755 /usr/local/bin/llmgw",
}
}
+1
View File
@@ -57,6 +57,7 @@ func Root() *cobra.Command {
newNodeCmd(),
newSelfUpgradeCmd(),
newReleaseCmd(),
newJoinCmd(),
)
return root
}
+206 -42
View File
@@ -8,10 +8,12 @@ import (
"os"
"strings"
"sync"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
@@ -34,6 +36,9 @@ const (
pWelcome setupPage = iota
pAdmin
pMachines
// pWait shows the codes for machines joining rather than logged into,
// until they have all dialled in.
pWait
pCheck
pPlan
pRun
@@ -44,10 +49,13 @@ const (
authAgent = iota
authKey
authPassword
// authJoin is no login at all: the machine runs `llmgw join` with a code
// shown here and dials out to this one.
authJoin
authCount
)
var authNames = [authCount]string{"ssh agent", "key file", "password"}
var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"}
// Fields of one database machine, in order.
const (
@@ -62,6 +70,30 @@ const (
type dbMachine struct {
addr, user, secret, sudo textField
auth int
// For a machine joining: its code, and it once it has dialled in.
code string
joined *joinpkg.Remote
arrive <-chan *joinpkg.Remote
}
// visible reports whether a field is shown, and so can be moved to: a joining
// machine is asked nothing but how it joins.
func (d dbMachine) visible(f int) bool {
switch {
case d.auth == authJoin:
return f == mfAuth
case f == mfSecret:
return d.auth != authAgent
}
return true
}
// dbConn is a database machine as it is reached, whichever way.
type dbConn struct {
remote onboard.Remote
addr string
ssh *onboard.GoSSH
joined *joinpkg.Remote
}
func newDBMachine(n int) dbMachine {
@@ -100,7 +132,10 @@ func (d *dbMachine) field(i int) *textField {
// Messages from the work behind the screens.
type (
checkDoneMsg struct{ rep preflight.Report }
setupRunMsg struct{}
joinedMsg struct {
i int
r *joinpkg.Remote
}
)
// progRef lets work started from Update send to the program it runs in.
@@ -120,7 +155,8 @@ type setupModel struct {
db [2]dbMachine
mfocus int // over both machines' fields, then the check button
remotes []*onboard.GoSSH
conns []dbConn
listener *joinpkg.Listener
report preflight.Report
checking bool
@@ -172,6 +208,13 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, cmd
case checkDoneMsg:
m.checking, m.report = false, msg.rep
case joinedMsg:
m.db[msg.i].joined = msg.r
if m.page == pWait && m.allJoined() {
m.page = pCheck
cmd := m.startCheck()
return m, cmd
}
case initLineMsg:
m.run.addLine(string(msg))
case initDoneMsg:
@@ -189,8 +232,8 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
m.page--
if m.page == pMachines && m.choice == 0 {
m.page = pAdmin
if m.page == pWait {
m.page = pMachines
}
m.formErr = ""
return m, nil
@@ -303,23 +346,14 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
n := 2*mfCount + 1
switch k.Type {
case tea.KeyTab, tea.KeyDown:
m.mfocus = (m.mfocus + 1) % n
if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent {
m.mfocus++
}
m.focusMachine()
m.step(1, n)
return m, nil
case tea.KeyShiftTab, tea.KeyUp:
m.mfocus = (m.mfocus - 1 + n) % n
if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent {
m.mfocus--
}
m.focusMachine()
m.step(-1, n)
return m, nil
case tea.KeyEnter:
if m.mfocus < 2*mfCount {
m.mfocus++
m.focusMachine()
m.step(1, n)
return m, nil
}
if err := m.validateMachines(); err != "" {
@@ -327,6 +361,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, nil
}
m.formErr = ""
if cmd, waiting := m.startJoins(); waiting {
m.page = pWait
return m, cmd
}
m.page = pCheck
cmd := m.startCheck()
return m, cmd
@@ -342,6 +380,10 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case "right", "l", " ":
d.auth = (d.auth + 1) % authCount
}
if d.auth != authJoin && d.joined != nil {
_ = d.joined.Close()
d.joined, d.code, d.arrive = nil, "", nil
}
d.secret.SetValue("")
d.secret.Mask = d.auth == authPassword
d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth]
@@ -353,9 +395,60 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, cmd
}
// step moves the focus over the fields that are shown.
func (m *setupModel) step(d, n int) {
for range n {
m.mfocus = (m.mfocus + d + n) % n
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
break
}
}
m.focusMachine()
}
func (m setupModel) allJoined() bool {
for _, d := range m.db {
if d.auth == authJoin && d.joined == nil {
return false
}
}
return true
}
// startJoins listens for the machines that join with a code, makes their
// codes, and waits for them; it says whether there is anything to wait for.
func (m *setupModel) startJoins() (tea.Cmd, bool) {
var cmds []tea.Cmd
for i := range m.db {
d := &m.db[i]
if d.auth != authJoin || d.joined != nil {
continue
}
if m.listener == nil {
l, err := joinpkg.Listen(fmt.Sprintf(":%d", joinpkg.DefaultPort),
fmt.Sprintf("%s:%d", m.thisAddr(), joinpkg.DefaultPort), time.Hour)
if err != nil {
m.formErr = err.Error()
return nil, false
}
m.listener = l
}
if d.arrive == nil {
c, ch := m.listener.Code("database")
d.code, d.arrive = c.String(), ch
}
ch, i := d.arrive, i
cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} })
}
return tea.Batch(cmds...), len(cmds) > 0
}
func (m setupModel) validateMachines() string {
seen := map[string]bool{m.thisAddr(): true, m.host: true}
for i, d := range m.db {
if d.auth == authJoin {
continue
}
addr := strings.TrimSpace(d.addr.Value())
switch {
case addr == "":
@@ -375,15 +468,23 @@ func (m setupModel) validateMachines() string {
// startCheck connects to the database machines and runs the checks on all
// three, this one included.
func (m *setupModel) startCheck() tea.Cmd {
for _, r := range m.remotes {
_ = r.Close()
for _, c := range m.conns {
if c.ssh != nil {
_ = c.ssh.Close()
}
}
m.remotes = nil
m.conns = nil
machines := []preflight.Machine{{
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
}}
for _, d := range m.db {
if d.auth == authJoin {
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
Addr: d.joined.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: d.joined})
continue
}
g := &onboard.GoSSH{
Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()),
UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(),
@@ -394,13 +495,13 @@ func (m *setupModel) startCheck() tea.Cmd {
case authPassword:
g.Password = d.secret.Value()
}
m.remotes = append(m.remotes, g)
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g})
}
m.checking = true
m.report = preflight.Report{}
remotes := m.remotes
conns := m.conns
return func() tea.Msg {
rep := preflight.Run(context.Background(), machines)
// A sudo that asks for a password is fine with one given: the
@@ -408,9 +509,10 @@ func (m *setupModel) startCheck() tea.Cmd {
for i := range rep.Findings {
f := &rep.Findings[i]
if f.Check == "sudo" && f.Level == preflight.Warn {
for _, g := range remotes {
for _, c := range conns {
g := c.ssh
switch {
case g.Addr != f.Machine:
case g == nil || g.Addr != f.Machine:
case g.SudoPassword == "":
f.Level = preflight.Fail
f.Fix = "give this user's sudo password on the previous screen"
@@ -430,6 +532,10 @@ func (m *setupModel) startCheck() tea.Cmd {
func (m setupModel) targets() []string {
var out []string
for _, d := range m.db {
if d.joined != nil {
out = append(out, "root@"+d.joined.Addr)
continue
}
out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value()))
}
return out
@@ -473,7 +579,7 @@ func (m *setupModel) startRun() tea.Cmd {
m.in = &in
m.run = newInitModel(&in, nil, false)
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
ref, remotes, logp, mu := m.ref, m.remotes, m.log, m.logMu
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
return func() tea.Msg {
send := func(l string) {
mu.Lock()
@@ -484,27 +590,38 @@ func (m *setupModel) startRun() tea.Cmd {
in.out = &lineWriter{send: send}
err := func() error {
ctx := context.Background()
for _, g := range remotes {
fmt.Fprintf(in.out, " .. sudo on %s\n", g.Addr)
if err := g.PrepareSudo(ctx); err != nil {
return err
for _, c := range conns {
if c.ssh != nil {
fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr)
if err := c.ssh.PrepareSudo(ctx); err != nil {
return err
}
}
out, err := g.Run(ctx, "hostname", nil)
out, err := c.remote.Run(ctx, "hostname", nil)
if err != nil {
return fmt.Errorf("%s: %v", g.Addr, err)
return fmt.Errorf("%s: %v", c.addr, err)
}
ip, err := ipv4Of(g.Addr)
ip, err := ipv4Of(c.addr)
if err != nil {
return err
}
in.patroniNodes = append(in.patroniNodes, pgha.Node{
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: g})
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote})
}
return in.run(ctx)
}()
for _, g := range remotes {
// The grant is taken back whatever happened.
_ = g.Close()
// Whatever happened: a sudo grant taken back, a joined machine's
// `llmgw join` told it is done, the port for joining closed.
for _, c := range conns {
if c.ssh != nil {
_ = c.ssh.Close()
}
if c.joined != nil {
_ = c.joined.Close()
}
}
if listener != nil {
_ = listener.Close()
}
return initDoneMsg{err}
}
@@ -520,6 +637,8 @@ func (m setupModel) View() string {
b.WriteString(m.adminView())
case pMachines:
b.WriteString(m.machinesView())
case pWait:
b.WriteString(m.waitView())
case pCheck:
b.WriteString(m.checkView())
case pPlan:
@@ -548,9 +667,16 @@ func (m setupModel) crumbs() string {
if m.choice == 0 {
return steps[0]
}
at := int(m.page)
switch {
case m.page == pWait:
at = int(pMachines)
case m.page > pWait:
at--
}
var out []string
for i, s := range steps {
if setupPage(i) == m.page {
if i == at {
s = lipgloss.NewStyle().Bold(true).Render(s)
}
out = append(out, s)
@@ -609,8 +735,10 @@ func (m setupModel) machinesView() string {
return " " + s
}
fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View())
if d.visible(mfAddr) {
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View())
}
var auth []string
for a := 0; a < authCount; a++ {
if a == d.auth {
@@ -620,10 +748,16 @@ func (m setupModel) machinesView() string {
}
}
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " "))
if d.auth != authAgent {
if d.visible(mfSecret) {
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View())
}
fmt.Fprintf(&b, " %s %s\n\n", label(mfSudo, "sudo password"), d.sudo.View())
if d.visible(mfSudo) {
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "sudo password"), d.sudo.View())
}
if d.auth == authJoin {
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next, and dials out to this machine.") + "\n")
}
b.WriteString("\n")
}
btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines")
if m.mfocus == 2*mfCount {
@@ -636,6 +770,29 @@ func (m setupModel) machinesView() string {
return b.String()
}
func (m setupModel) waitView() string {
var b strings.Builder
fmt.Fprintf(&b, "On each machine below, as root. It dials this machine on port %d; nothing dials in.\n\n", joinpkg.DefaultPort)
for i, d := range m.db {
if d.auth != authJoin {
continue
}
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
if d.joined != nil {
fmt.Fprintf(&b, "%s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
continue
}
fmt.Fprintf(&b, "%s %s waiting\n", title, m.spin.View())
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
for _, l := range joinInstall() {
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
}
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
}
b.WriteString(sDim.Render("Each code is for one machine and good for an hour. esc back · ctrl+c to leave") + "\n")
return b.String()
}
func (m setupModel) checkView() string {
var b strings.Builder
if m.checking {
@@ -691,7 +848,14 @@ func (m setupModel) planView() string {
if granted {
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n")
}
b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n\n")
b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n")
for _, d := range m.db {
if d.joined != nil {
b.WriteString(sDim.Render("(a machine that joined with a code needs ssh for that: the code is for these screens)") + "\n")
break
}
}
b.WriteString("\n")
b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n")
return b.String()
}
+302
View File
@@ -0,0 +1,302 @@
// Package join lets a machine be set up without anything logging into it.
//
// The machine doing the setup — running as root, holding the authority —
// listens for a while and hands out one code per machine it expects. On the
// other machine `sudo llmgw join <code>` dials out to it, checks it is talking
// to the listener the code names, proves itself with the code's secret, and
// from then on runs what it is sent, as ssh would, until the setup closes
// the connection. To the setup it is one more onboard.Remote: preflight,
// Patroni and node onboarding run over it unchanged.
//
// For sites where nothing may log into a machine from outside: the
// connection is outbound from the machine being set up, no password is
// typed anywhere, and the code is good for one connection and a limited time.
package join
import (
"bufio"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math/big"
"net"
"strings"
"sync"
"time"
)
// DefaultPort is where a setup listens for machines joining it.
const DefaultPort = 7444
// codePrefix marks a join code, and its version.
const codePrefix = "llmgw-join1-"
// Code is what `llmgw join` is given: where to dial, how to know it reached
// the right place, and the secret that admits it.
//
// Short, because somebody copies it from one screen into another: the first
// version was JSON and over 250 characters, which a terminal cut off at its
// edge. Packed it is under 90.
type Code struct {
Addr string
// Fingerprint is the first 16 bytes of the SHA-256 of the listener's
// certificate: 128 bits, more than enough to find nothing else by.
Fingerprint [16]byte
Secret [16]byte
Expires int64
// Role says what the machine is being set up as, for its operator to
// read before it runs: "database", "node" or "control plane".
Role string
}
var roles = []string{"", "database", "node", "control plane"}
// String is the code as typed: one word, safe in a shell.
func (c Code) String() string {
b := make([]byte, 0, 40+len(c.Addr))
b = append(b, c.Fingerprint[:]...)
b = append(b, c.Secret[:]...)
b = binary.BigEndian.AppendUint32(b, uint32(c.Expires))
role := byte(0)
for i, r := range roles {
if r == c.Role {
role = byte(i)
}
}
b = append(b, role)
b = append(b, c.Addr...)
return codePrefix + base64.RawURLEncoding.EncodeToString(b)
}
// Parse reads a code back.
func Parse(s string) (Code, error) {
raw, ok := strings.CutPrefix(strings.TrimSpace(s), codePrefix)
if !ok {
return Code{}, errors.New("not a join code: it starts " + codePrefix)
}
b, err := base64.RawURLEncoding.DecodeString(raw)
if err != nil || len(b) < 16+16+4+1+3 {
return Code{}, errors.New("the join code is damaged: copy it whole")
}
var c Code
copy(c.Fingerprint[:], b[:16])
copy(c.Secret[:], b[16:32])
c.Expires = int64(binary.BigEndian.Uint32(b[32:36]))
if int(b[36]) < len(roles) {
c.Role = roles[b[36]]
}
c.Addr = string(b[37:])
if _, _, err := net.SplitHostPort(c.Addr); err != nil {
return Code{}, errors.New("the join code is damaged: copy it whole")
}
if time.Now().Unix() > c.Expires {
return Code{}, fmt.Errorf("the join code expired at %s; ask for a new one",
time.Unix(c.Expires, 0).Format(time.RFC3339))
}
return c, nil
}
// The wire: one JSON object a line, each way.
type hello struct {
Secret []byte `json:"secret"`
Hostname string `json:"hostname"`
// Addr is the address the machine dialled out from: how the others
// reach it, when nothing else says.
Addr string `json:"addr"`
}
type request struct {
Script string `json:"script,omitempty"`
Stdin []byte `json:"stdin,omitempty"`
Bye bool `json:"bye,omitempty"`
}
type response struct {
Stdout []byte `json:"stdout,omitempty"`
Stderr []byte `json:"stderr,omitempty"`
Exit int `json:"exit"`
Error string `json:"error,omitempty"`
}
// Listener waits for machines to join.
type Listener struct {
ln net.Listener
addr string
fingerprint [16]byte
ttl time.Duration
mu sync.Mutex
pending map[[16]byte]chan *Remote // by secret
}
// Listen starts listening on bind (":7444") and advertises addr ("10.0.0.5:7444")
// in its codes. Codes are good for ttl.
func Listen(bind, addr string, ttl time.Duration) (*Listener, error) {
cert, fp, err := sessionCert()
if err != nil {
return nil, err
}
ln, err := tls.Listen("tcp", bind, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13})
if err != nil {
return nil, fmt.Errorf("listening for machines to join on %s: %w", bind, err)
}
l := &Listener{ln: ln, addr: addr, fingerprint: fp, ttl: ttl, pending: map[[16]byte]chan *Remote{}}
go l.accept()
return l, nil
}
// sessionCert is a certificate for this listener alone, trusted by the
// fingerprint in its codes rather than by any authority: it lives as long as
// the setup does.
func sessionCert() (tls.Certificate, [16]byte, error) {
var fp [16]byte
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return tls.Certificate{}, fp, err
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(time.Now().UnixNano()),
Subject: pkix.Name{CommonName: "llmgw setup"},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().Add(48 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return tls.Certificate{}, fp, err
}
sum := sha256.Sum256(der)
copy(fp[:], sum[:16])
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, fp, nil
}
// Code makes a code for one machine, and the channel its connection arrives
// on.
func (l *Listener) Code(role string) (Code, <-chan *Remote) {
var secret [16]byte
_, _ = rand.Read(secret[:])
ch := make(chan *Remote, 1)
l.mu.Lock()
l.pending[secret] = ch
l.mu.Unlock()
return Code{Addr: l.addr, Fingerprint: l.fingerprint, Secret: secret,
Expires: time.Now().Add(l.ttl).Unix(), Role: role}, ch
}
// Close stops listening; machines already joined stay connected until their
// Remote is closed.
func (l *Listener) Close() error { return l.ln.Close() }
func (l *Listener) accept() {
for {
c, err := l.ln.Accept()
if err != nil {
return
}
go l.admit(c)
}
}
func (l *Listener) admit(c net.Conn) {
_ = c.SetDeadline(time.Now().Add(30 * time.Second))
r := bufio.NewReader(c)
var h hello
line, err := r.ReadBytes('\n')
if err != nil || json.Unmarshal(line, &h) != nil {
_ = c.Close()
return
}
l.mu.Lock()
var ch chan *Remote
for secret, p := range l.pending {
if subtle.ConstantTimeCompare(secret[:], h.Secret) == 1 {
ch = p
// One machine a code: whoever comes second is turned away.
delete(l.pending, secret)
}
}
l.mu.Unlock()
if ch == nil {
_, _ = c.Write([]byte(`{"error":"this code is not one the setup is waiting for, or it was used already"}` + "\n"))
_ = c.Close()
return
}
_ = c.SetDeadline(time.Time{})
if _, err := c.Write([]byte("{}\n")); err != nil {
_ = c.Close()
return
}
addr := h.Addr
if addr == "" {
addr, _, _ = net.SplitHostPort(c.RemoteAddr().String())
}
ch <- &Remote{conn: c, r: r, Hostname: h.Hostname, Addr: addr}
}
// Remote is a joined machine, as the setup runs things on it.
type Remote struct {
Hostname string
Addr string
mu sync.Mutex
conn net.Conn
r *bufio.Reader
}
// Run runs script with sh on the machine, as root.
func (m *Remote) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.conn == nil {
return nil, fmt.Errorf("%s is no longer connected", m.Hostname)
}
if dl, ok := ctx.Deadline(); ok {
_ = m.conn.SetDeadline(dl)
defer m.conn.SetDeadline(time.Time{})
}
b, _ := json.Marshal(request{Script: script, Stdin: stdin})
if _, err := m.conn.Write(append(b, '\n')); err != nil {
return nil, fmt.Errorf("%s: %w", m.Hostname, err)
}
line, err := m.r.ReadBytes('\n')
if err != nil {
return nil, fmt.Errorf("%s went away: %w", m.Hostname, err)
}
var resp response
if err := json.Unmarshal(line, &resp); err != nil {
return nil, fmt.Errorf("%s answered something unreadable", m.Hostname)
}
if resp.Error != "" {
return append(resp.Stdout, resp.Stderr...), errors.New(resp.Error)
}
if resp.Exit != 0 {
return append(resp.Stdout, resp.Stderr...), fmt.Errorf("exit status %d", resp.Exit)
}
return resp.Stdout, nil
}
// Close tells the machine the setup is done, which ends its `llmgw join`.
func (m *Remote) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
if m.conn == nil {
return nil
}
b, _ := json.Marshal(request{Bye: true})
_, _ = m.conn.Write(append(b, '\n'))
err := m.conn.Close()
m.conn = nil
return err
}
+94
View File
@@ -0,0 +1,94 @@
package join
import (
"context"
"strings"
"testing"
"time"
)
func listen(t *testing.T) *Listener {
t.Helper()
l, err := Listen("127.0.0.1:0", "", time.Hour)
if err != nil {
t.Fatal(err)
}
l.addr = l.ln.Addr().String()
t.Cleanup(func() { _ = l.Close() })
return l
}
func TestAJoinedMachineRunsWhatTheSetupSendsUntilItIsDone(t *testing.T) {
l := listen(t)
code, arrived := l.Code("database")
parsed, err := Parse(code.String())
if err != nil || parsed.Role != "database" {
t.Fatalf("%v %+v", err, parsed)
}
var said []string
served := make(chan error, 1)
go func() { served <- Serve(context.Background(), parsed, func(s string) { said = append(said, s) }) }()
var m *Remote
select {
case m = <-arrived:
case err := <-served:
t.Fatalf("join ended before arriving: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("the machine never arrived")
}
if m.Hostname == "" || m.Addr != "127.0.0.1" {
t.Errorf("arrived as %q from %q", m.Hostname, m.Addr)
}
out, err := m.Run(context.Background(), "set -e\ntr a-z A-Z", []byte("hello\n"))
if err != nil || string(out) != "HELLO\n" {
t.Fatalf("%q, %v", out, err)
}
if out, err := m.Run(context.Background(), "echo nope >&2; exit 3", nil); err == nil || !strings.Contains(string(out), "nope") {
t.Errorf("a failing script: %q, %v", out, err)
}
_ = m.Close()
if err := <-served; err != nil {
t.Errorf("join did not end cleanly when the setup was done: %v", err)
}
if len(said) < 2 || !strings.Contains(strings.Join(said, "\n"), "tr a-z A-Z") {
t.Errorf("the machine's operator was not told what ran: %q", said)
}
// The code was for one machine.
if err := Serve(context.Background(), parsed, func(string) {}); err == nil || !strings.Contains(err.Error(), "used already") {
t.Errorf("a code used twice: %v", err)
}
}
func TestAJoinRefusesAListenerThatIsNotTheOneInTheCode(t *testing.T) {
l := listen(t)
code, _ := l.Code("node")
other := listen(t)
code.Addr = other.addr // what the code names is elsewhere
err := Serve(context.Background(), code, func(string) {})
if err == nil || !strings.Contains(err.Error(), "not the setup") {
t.Errorf("joined a stranger: %v", err)
}
}
func TestAnExpiredOrDamagedCodeIsRefusedBeforeDialling(t *testing.T) {
c := Code{Addr: "10.0.0.5:7444", Role: "node", Expires: time.Now().Add(-time.Minute).Unix()}
if _, err := Parse(c.String()); err == nil || !strings.Contains(err.Error(), "expired") {
t.Errorf("expired: %v", err)
}
if _, err := Parse(c.String()[:30]); err == nil {
t.Error("a truncated code was read")
}
if _, err := Parse("ssh-rsa AAAA"); err == nil {
t.Error("something else was read as a code")
}
// Short enough to copy from a screen in one line.
c.Expires = time.Now().Add(time.Hour).Unix()
if n := len(c.String()); n > 100 {
t.Errorf("a code is %d characters", n)
}
if back, err := Parse(c.String()); err != nil || back.Addr != c.Addr || back.Role != "node" {
t.Errorf("%+v, %v", back, err)
}
}
+126
View File
@@ -0,0 +1,126 @@
package join
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"strings"
"time"
)
// Serve dials the setup a code names and runs what it sends until it says it
// is done. Each script is reported to log by its first line, so whoever ran
// `llmgw join` sees what is being done to their machine.
func Serve(ctx context.Context, c Code, log func(string)) error {
d := tls.Dialer{
NetDialer: &net.Dialer{Timeout: 10 * time.Second},
Config: &tls.Config{
MinVersion: tls.VersionTLS13,
// Trusted by the fingerprint in the code, not by an authority:
// the listener's certificate is made for this setup alone.
InsecureSkipVerify: true, //nolint:gosec // verified below, by fingerprint
},
}
want := c.Fingerprint[:]
d.Config.VerifyPeerCertificate = func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("the setup sent no certificate")
}
sum := sha256.Sum256(raw[0])
if subtle.ConstantTimeCompare(sum[:16], want) != 1 {
return errors.New("the machine at that address is not the setup this code is for")
}
return nil
}
conn, err := d.DialContext(ctx, "tcp", c.Addr)
if err != nil {
return fmt.Errorf("cannot reach the setup at %s: %w; it listens there only while it waits for this machine", c.Addr, err)
}
defer conn.Close()
host, _ := os.Hostname()
local, _, _ := net.SplitHostPort(conn.LocalAddr().String())
b, _ := json.Marshal(hello{Secret: c.Secret[:], Hostname: host, Addr: local})
if _, err := conn.Write(append(b, '\n')); err != nil {
return err
}
r := bufio.NewReader(conn)
line, err := r.ReadBytes('\n')
if err != nil {
return fmt.Errorf("the setup closed the connection: %w", err)
}
var ack response
if err := json.Unmarshal(line, &ack); err != nil {
return errors.New("the setup answered something unreadable")
}
if ack.Error != "" {
return errors.New(ack.Error)
}
log(fmt.Sprintf("joined the setup at %s; it is setting this machine up", c.Addr))
for {
line, err := r.ReadBytes('\n')
if err != nil {
return fmt.Errorf("the setup went away before it said it was done: %w", err)
}
var req request
if err := json.Unmarshal(line, &req); err != nil {
return errors.New("the setup sent something unreadable")
}
if req.Bye {
log("the setup is done with this machine")
return nil
}
log(summary(req.Script))
resp := run(ctx, req)
out, _ := json.Marshal(resp)
if _, err := conn.Write(append(out, '\n')); err != nil {
return err
}
}
}
// run runs one script as the setup asked.
func run(ctx context.Context, req request) response {
cmd := exec.CommandContext(ctx, "sh", "-c", req.Script)
if req.Stdin != nil {
cmd.Stdin = bytes.NewReader(req.Stdin)
}
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err := cmd.Run()
resp := response{Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}
var exit *exec.ExitError
switch {
case errors.As(err, &exit):
resp.Exit = exit.ExitCode()
case err != nil:
resp.Error = err.Error()
}
return resp
}
// summary is a script's first line that says something.
func summary(script string) string {
for _, l := range strings.Split(script, "\n") {
l = strings.TrimSpace(l)
if l == "" || l == "set -e" || strings.HasPrefix(l, "#") {
continue
}
if len(l) > 100 {
l = l[:100] + "…"
}
return " " + l
}
return " (an empty script)"
}