Let a machine join a setup by code, dialling out, with nothing logging in
The setup screens can reach a database machine by a join code instead of ssh: they listen on 7444 and show a short code; llmgw join dials out, checks the setup by fingerprint, admits itself with the code's secret, and runs what the setup sends until it is done. A joined machine is one more onboard.Remote, so checks and the Patroni install run over it unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
38f2671ed9
commit
f5c79052b1
8 files changed
+810
-43
No files matched your search
@@ -0,0 +1,302 @@
|
||||
// Package join lets a machine be set up without anything logging into it.
|
||||
//
|
||||
// The machine doing the setup — running as root, holding the authority —
|
||||
// listens for a while and hands out one code per machine it expects. On the
|
||||
// other machine `sudo llmgw join <code>` dials out to it, checks it is talking
|
||||
// to the listener the code names, proves itself with the code's secret, and
|
||||
// from then on runs what it is sent, as ssh would, until the setup closes
|
||||
// the connection. To the setup it is one more onboard.Remote: preflight,
|
||||
// Patroni and node onboarding run over it unchanged.
|
||||
//
|
||||
// For sites where nothing may log into a machine from outside: the
|
||||
// connection is outbound from the machine being set up, no password is
|
||||
// typed anywhere, and the code is good for one connection and a limited time.
|
||||
package join
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultPort is where a setup listens for machines joining it.
|
||||
const DefaultPort = 7444
|
||||
|
||||
// codePrefix marks a join code, and its version.
|
||||
const codePrefix = "llmgw-join1-"
|
||||
|
||||
// Code is what `llmgw join` is given: where to dial, how to know it reached
|
||||
// the right place, and the secret that admits it.
|
||||
//
|
||||
// Short, because somebody copies it from one screen into another: the first
|
||||
// version was JSON and over 250 characters, which a terminal cut off at its
|
||||
// edge. Packed it is under 90.
|
||||
type Code struct {
|
||||
Addr string
|
||||
// Fingerprint is the first 16 bytes of the SHA-256 of the listener's
|
||||
// certificate: 128 bits, more than enough to find nothing else by.
|
||||
Fingerprint [16]byte
|
||||
Secret [16]byte
|
||||
Expires int64
|
||||
// Role says what the machine is being set up as, for its operator to
|
||||
// read before it runs: "database", "node" or "control plane".
|
||||
Role string
|
||||
}
|
||||
|
||||
var roles = []string{"", "database", "node", "control plane"}
|
||||
|
||||
// String is the code as typed: one word, safe in a shell.
|
||||
func (c Code) String() string {
|
||||
b := make([]byte, 0, 40+len(c.Addr))
|
||||
b = append(b, c.Fingerprint[:]...)
|
||||
b = append(b, c.Secret[:]...)
|
||||
b = binary.BigEndian.AppendUint32(b, uint32(c.Expires))
|
||||
role := byte(0)
|
||||
for i, r := range roles {
|
||||
if r == c.Role {
|
||||
role = byte(i)
|
||||
}
|
||||
}
|
||||
b = append(b, role)
|
||||
b = append(b, c.Addr...)
|
||||
return codePrefix + base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
// Parse reads a code back.
|
||||
func Parse(s string) (Code, error) {
|
||||
raw, ok := strings.CutPrefix(strings.TrimSpace(s), codePrefix)
|
||||
if !ok {
|
||||
return Code{}, errors.New("not a join code: it starts " + codePrefix)
|
||||
}
|
||||
b, err := base64.RawURLEncoding.DecodeString(raw)
|
||||
if err != nil || len(b) < 16+16+4+1+3 {
|
||||
return Code{}, errors.New("the join code is damaged: copy it whole")
|
||||
}
|
||||
var c Code
|
||||
copy(c.Fingerprint[:], b[:16])
|
||||
copy(c.Secret[:], b[16:32])
|
||||
c.Expires = int64(binary.BigEndian.Uint32(b[32:36]))
|
||||
if int(b[36]) < len(roles) {
|
||||
c.Role = roles[b[36]]
|
||||
}
|
||||
c.Addr = string(b[37:])
|
||||
if _, _, err := net.SplitHostPort(c.Addr); err != nil {
|
||||
return Code{}, errors.New("the join code is damaged: copy it whole")
|
||||
}
|
||||
if time.Now().Unix() > c.Expires {
|
||||
return Code{}, fmt.Errorf("the join code expired at %s; ask for a new one",
|
||||
time.Unix(c.Expires, 0).Format(time.RFC3339))
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// The wire: one JSON object a line, each way.
|
||||
type hello struct {
|
||||
Secret []byte `json:"secret"`
|
||||
Hostname string `json:"hostname"`
|
||||
// Addr is the address the machine dialled out from: how the others
|
||||
// reach it, when nothing else says.
|
||||
Addr string `json:"addr"`
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Script string `json:"script,omitempty"`
|
||||
Stdin []byte `json:"stdin,omitempty"`
|
||||
Bye bool `json:"bye,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Stdout []byte `json:"stdout,omitempty"`
|
||||
Stderr []byte `json:"stderr,omitempty"`
|
||||
Exit int `json:"exit"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Listener waits for machines to join.
|
||||
type Listener struct {
|
||||
ln net.Listener
|
||||
addr string
|
||||
fingerprint [16]byte
|
||||
ttl time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[[16]byte]chan *Remote // by secret
|
||||
}
|
||||
|
||||
// Listen starts listening on bind (":7444") and advertises addr ("10.0.0.5:7444")
|
||||
// in its codes. Codes are good for ttl.
|
||||
func Listen(bind, addr string, ttl time.Duration) (*Listener, error) {
|
||||
cert, fp, err := sessionCert()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ln, err := tls.Listen("tcp", bind, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listening for machines to join on %s: %w", bind, err)
|
||||
}
|
||||
l := &Listener{ln: ln, addr: addr, fingerprint: fp, ttl: ttl, pending: map[[16]byte]chan *Remote{}}
|
||||
go l.accept()
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// sessionCert is a certificate for this listener alone, trusted by the
|
||||
// fingerprint in its codes rather than by any authority: it lives as long as
|
||||
// the setup does.
|
||||
func sessionCert() (tls.Certificate, [16]byte, error) {
|
||||
var fp [16]byte
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fp, err
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: "llmgw setup"},
|
||||
NotBefore: time.Now().Add(-time.Minute),
|
||||
NotAfter: time.Now().Add(48 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fp, err
|
||||
}
|
||||
sum := sha256.Sum256(der)
|
||||
copy(fp[:], sum[:16])
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, fp, nil
|
||||
}
|
||||
|
||||
// Code makes a code for one machine, and the channel its connection arrives
|
||||
// on.
|
||||
func (l *Listener) Code(role string) (Code, <-chan *Remote) {
|
||||
var secret [16]byte
|
||||
_, _ = rand.Read(secret[:])
|
||||
ch := make(chan *Remote, 1)
|
||||
l.mu.Lock()
|
||||
l.pending[secret] = ch
|
||||
l.mu.Unlock()
|
||||
return Code{Addr: l.addr, Fingerprint: l.fingerprint, Secret: secret,
|
||||
Expires: time.Now().Add(l.ttl).Unix(), Role: role}, ch
|
||||
}
|
||||
|
||||
// Close stops listening; machines already joined stay connected until their
|
||||
// Remote is closed.
|
||||
func (l *Listener) Close() error { return l.ln.Close() }
|
||||
|
||||
func (l *Listener) accept() {
|
||||
for {
|
||||
c, err := l.ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go l.admit(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Listener) admit(c net.Conn) {
|
||||
_ = c.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
r := bufio.NewReader(c)
|
||||
var h hello
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil || json.Unmarshal(line, &h) != nil {
|
||||
_ = c.Close()
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
var ch chan *Remote
|
||||
for secret, p := range l.pending {
|
||||
if subtle.ConstantTimeCompare(secret[:], h.Secret) == 1 {
|
||||
ch = p
|
||||
// One machine a code: whoever comes second is turned away.
|
||||
delete(l.pending, secret)
|
||||
}
|
||||
}
|
||||
l.mu.Unlock()
|
||||
if ch == nil {
|
||||
_, _ = c.Write([]byte(`{"error":"this code is not one the setup is waiting for, or it was used already"}` + "\n"))
|
||||
_ = c.Close()
|
||||
return
|
||||
}
|
||||
_ = c.SetDeadline(time.Time{})
|
||||
if _, err := c.Write([]byte("{}\n")); err != nil {
|
||||
_ = c.Close()
|
||||
return
|
||||
}
|
||||
addr := h.Addr
|
||||
if addr == "" {
|
||||
addr, _, _ = net.SplitHostPort(c.RemoteAddr().String())
|
||||
}
|
||||
ch <- &Remote{conn: c, r: r, Hostname: h.Hostname, Addr: addr}
|
||||
}
|
||||
|
||||
// Remote is a joined machine, as the setup runs things on it.
|
||||
type Remote struct {
|
||||
Hostname string
|
||||
Addr string
|
||||
|
||||
mu sync.Mutex
|
||||
conn net.Conn
|
||||
r *bufio.Reader
|
||||
}
|
||||
|
||||
// Run runs script with sh on the machine, as root.
|
||||
func (m *Remote) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.conn == nil {
|
||||
return nil, fmt.Errorf("%s is no longer connected", m.Hostname)
|
||||
}
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = m.conn.SetDeadline(dl)
|
||||
defer m.conn.SetDeadline(time.Time{})
|
||||
}
|
||||
b, _ := json.Marshal(request{Script: script, Stdin: stdin})
|
||||
if _, err := m.conn.Write(append(b, '\n')); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", m.Hostname, err)
|
||||
}
|
||||
line, err := m.r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s went away: %w", m.Hostname, err)
|
||||
}
|
||||
var resp response
|
||||
if err := json.Unmarshal(line, &resp); err != nil {
|
||||
return nil, fmt.Errorf("%s answered something unreadable", m.Hostname)
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return append(resp.Stdout, resp.Stderr...), errors.New(resp.Error)
|
||||
}
|
||||
if resp.Exit != 0 {
|
||||
return append(resp.Stdout, resp.Stderr...), fmt.Errorf("exit status %d", resp.Exit)
|
||||
}
|
||||
return resp.Stdout, nil
|
||||
}
|
||||
|
||||
// Close tells the machine the setup is done, which ends its `llmgw join`.
|
||||
func (m *Remote) Close() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.conn == nil {
|
||||
return nil
|
||||
}
|
||||
b, _ := json.Marshal(request{Bye: true})
|
||||
_, _ = m.conn.Write(append(b, '\n'))
|
||||
err := m.conn.Close()
|
||||
m.conn = nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package join
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func listen(t *testing.T) *Listener {
|
||||
t.Helper()
|
||||
l, err := Listen("127.0.0.1:0", "", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l.addr = l.ln.Addr().String()
|
||||
t.Cleanup(func() { _ = l.Close() })
|
||||
return l
|
||||
}
|
||||
|
||||
func TestAJoinedMachineRunsWhatTheSetupSendsUntilItIsDone(t *testing.T) {
|
||||
l := listen(t)
|
||||
code, arrived := l.Code("database")
|
||||
parsed, err := Parse(code.String())
|
||||
if err != nil || parsed.Role != "database" {
|
||||
t.Fatalf("%v %+v", err, parsed)
|
||||
}
|
||||
var said []string
|
||||
served := make(chan error, 1)
|
||||
go func() { served <- Serve(context.Background(), parsed, func(s string) { said = append(said, s) }) }()
|
||||
|
||||
var m *Remote
|
||||
select {
|
||||
case m = <-arrived:
|
||||
case err := <-served:
|
||||
t.Fatalf("join ended before arriving: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the machine never arrived")
|
||||
}
|
||||
if m.Hostname == "" || m.Addr != "127.0.0.1" {
|
||||
t.Errorf("arrived as %q from %q", m.Hostname, m.Addr)
|
||||
}
|
||||
out, err := m.Run(context.Background(), "set -e\ntr a-z A-Z", []byte("hello\n"))
|
||||
if err != nil || string(out) != "HELLO\n" {
|
||||
t.Fatalf("%q, %v", out, err)
|
||||
}
|
||||
if out, err := m.Run(context.Background(), "echo nope >&2; exit 3", nil); err == nil || !strings.Contains(string(out), "nope") {
|
||||
t.Errorf("a failing script: %q, %v", out, err)
|
||||
}
|
||||
_ = m.Close()
|
||||
if err := <-served; err != nil {
|
||||
t.Errorf("join did not end cleanly when the setup was done: %v", err)
|
||||
}
|
||||
if len(said) < 2 || !strings.Contains(strings.Join(said, "\n"), "tr a-z A-Z") {
|
||||
t.Errorf("the machine's operator was not told what ran: %q", said)
|
||||
}
|
||||
|
||||
// The code was for one machine.
|
||||
if err := Serve(context.Background(), parsed, func(string) {}); err == nil || !strings.Contains(err.Error(), "used already") {
|
||||
t.Errorf("a code used twice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJoinRefusesAListenerThatIsNotTheOneInTheCode(t *testing.T) {
|
||||
l := listen(t)
|
||||
code, _ := l.Code("node")
|
||||
other := listen(t)
|
||||
code.Addr = other.addr // what the code names is elsewhere
|
||||
err := Serve(context.Background(), code, func(string) {})
|
||||
if err == nil || !strings.Contains(err.Error(), "not the setup") {
|
||||
t.Errorf("joined a stranger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExpiredOrDamagedCodeIsRefusedBeforeDialling(t *testing.T) {
|
||||
c := Code{Addr: "10.0.0.5:7444", Role: "node", Expires: time.Now().Add(-time.Minute).Unix()}
|
||||
if _, err := Parse(c.String()); err == nil || !strings.Contains(err.Error(), "expired") {
|
||||
t.Errorf("expired: %v", err)
|
||||
}
|
||||
if _, err := Parse(c.String()[:30]); err == nil {
|
||||
t.Error("a truncated code was read")
|
||||
}
|
||||
if _, err := Parse("ssh-rsa AAAA"); err == nil {
|
||||
t.Error("something else was read as a code")
|
||||
}
|
||||
// Short enough to copy from a screen in one line.
|
||||
c.Expires = time.Now().Add(time.Hour).Unix()
|
||||
if n := len(c.String()); n > 100 {
|
||||
t.Errorf("a code is %d characters", n)
|
||||
}
|
||||
if back, err := Parse(c.String()); err != nil || back.Addr != c.Addr || back.Role != "node" {
|
||||
t.Errorf("%+v, %v", back, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package join
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Serve dials the setup a code names and runs what it sends until it says it
|
||||
// is done. Each script is reported to log by its first line, so whoever ran
|
||||
// `llmgw join` sees what is being done to their machine.
|
||||
func Serve(ctx context.Context, c Code, log func(string)) error {
|
||||
d := tls.Dialer{
|
||||
NetDialer: &net.Dialer{Timeout: 10 * time.Second},
|
||||
Config: &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
// Trusted by the fingerprint in the code, not by an authority:
|
||||
// the listener's certificate is made for this setup alone.
|
||||
InsecureSkipVerify: true, //nolint:gosec // verified below, by fingerprint
|
||||
},
|
||||
}
|
||||
want := c.Fingerprint[:]
|
||||
d.Config.VerifyPeerCertificate = func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(raw) == 0 {
|
||||
return errors.New("the setup sent no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(raw[0])
|
||||
if subtle.ConstantTimeCompare(sum[:16], want) != 1 {
|
||||
return errors.New("the machine at that address is not the setup this code is for")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
conn, err := d.DialContext(ctx, "tcp", c.Addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the setup at %s: %w; it listens there only while it waits for this machine", c.Addr, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
host, _ := os.Hostname()
|
||||
local, _, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
b, _ := json.Marshal(hello{Secret: c.Secret[:], Hostname: host, Addr: local})
|
||||
if _, err := conn.Write(append(b, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
r := bufio.NewReader(conn)
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return fmt.Errorf("the setup closed the connection: %w", err)
|
||||
}
|
||||
var ack response
|
||||
if err := json.Unmarshal(line, &ack); err != nil {
|
||||
return errors.New("the setup answered something unreadable")
|
||||
}
|
||||
if ack.Error != "" {
|
||||
return errors.New(ack.Error)
|
||||
}
|
||||
log(fmt.Sprintf("joined the setup at %s; it is setting this machine up", c.Addr))
|
||||
|
||||
for {
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return fmt.Errorf("the setup went away before it said it was done: %w", err)
|
||||
}
|
||||
var req request
|
||||
if err := json.Unmarshal(line, &req); err != nil {
|
||||
return errors.New("the setup sent something unreadable")
|
||||
}
|
||||
if req.Bye {
|
||||
log("the setup is done with this machine")
|
||||
return nil
|
||||
}
|
||||
log(summary(req.Script))
|
||||
resp := run(ctx, req)
|
||||
out, _ := json.Marshal(resp)
|
||||
if _, err := conn.Write(append(out, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run runs one script as the setup asked.
|
||||
func run(ctx context.Context, req request) response {
|
||||
cmd := exec.CommandContext(ctx, "sh", "-c", req.Script)
|
||||
if req.Stdin != nil {
|
||||
cmd.Stdin = bytes.NewReader(req.Stdin)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
resp := response{Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}
|
||||
var exit *exec.ExitError
|
||||
switch {
|
||||
case errors.As(err, &exit):
|
||||
resp.Exit = exit.ExitCode()
|
||||
case err != nil:
|
||||
resp.Error = err.Error()
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// summary is a script's first line that says something.
|
||||
func summary(script string) string {
|
||||
for _, l := range strings.Split(script, "\n") {
|
||||
l = strings.TrimSpace(l)
|
||||
if l == "" || l == "set -e" || strings.HasPrefix(l, "#") {
|
||||
continue
|
||||
}
|
||||
if len(l) > 100 {
|
||||
l = l[:100] + "…"
|
||||
}
|
||||
return " " + l
|
||||
}
|
||||
return " (an empty script)"
|
||||
}
|
||||
Reference in new issue
Block a user