Machine tokens only from the machine's own address

Job and node tokens travel in the per-MAC pxelinux file, readable by any
host on the provisioning network over TFTP; with a job token one could
fetch the deploy seed (user-data, tag cloud-config) or report the job,
with a node token the stateless overlay (e.g. munge.key).

The region knows the address it gives each machine (static DHCP leases
from the network model): /jobs/self, job progress, phone-home and the
node overlay are now refused from anywhere else (403, an event on the
machine). Through a remote rack controller the address is the rack's
own X-Forwarded-For entry, believed only from an authenticated rack. A
machine without a fixed address (proxy DHCP) binds a job to the first
address that uses it. agent_source_check: false turns this off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/rollcall/agentsrc.go

@@ -0,0 +1,101 @@
+package main
+
+// Where machine-side tokens may be used from. Job tokens (rollcall.job=) and
+// node tokens (rollcall.node=) travel in the per-MAC pxelinux file, which any
+// host on the provisioning network can fetch over TFTP; without a hardware
+// root of trust the machine holds no secret of its own. What the region does
+// know is the address it gives the machine (a static DHCP lease from the
+// network model), so a token is only accepted from there. A machine without
+// a fixed address (proxy DHCP, a site's DHCP server) binds a job to the
+// first address that uses it. Refusals are events on the machine.
+//
+// agent_source_check: false in config.json turns this off (NAT between the
+// machines and Rollcall).
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+)
+
+// clientIP is the machine's address: a remote rack controller proxies the
+// node API and appends it to X-Forwarded-For (the last entry is the rack's).
+func (a *App) clientIP(r *http.Request) string {
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host = r.RemoteAddr
+	}
+	if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName {
+		if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+			parts := strings.Split(xff, ",")
+			host = strings.TrimSpace(parts[len(parts)-1])
+		}
+	}
+	if ad, ok := parseAddr(host); ok {
+		return ad.String()
+	}
+	return host
+}
+
+// expectedIPs are the addresses the region hands this machine itself.
+func (a *App) expectedIPs(ctx context.Context, m *Machine) []string {
+	var out []string
+	add := func(ip string) {
+		if ad, ok := parseAddr(ip); ok && !hasTag(out, ad.String()) {
+			out = append(out, ad.String())
+		}
+	}
+	subnets := a.subnetsSnapshot()
+	if ips, err := a.store.IPs(ctx, "", m.ID); err == nil {
+		for _, x := range ips {
+			if sn := subnetFor(subnets, x.IP); sn == nil || sn.DHCPMode != "proxy" {
+				add(x.IP)
+			}
+		}
+	}
+	if mg := deref(m.MgmtIP); mg != "" {
+		if sn := subnetFor(subnets, mg); sn == nil || sn.DHCPMode != "proxy" {
+			add(mg)
+		}
+	}
+	return out
+}
+
+func (s *Store) bindJobIP(ctx context.Context, id, ip string) (string, error) {
+	var bound string
+	err := s.pool.QueryRow(ctx, `UPDATE jobs SET bound_ip=coalesce(bound_ip, $2) WHERE id=$1 RETURNING bound_ip`, id, ip).Scan(&bound)
+	return bound, err
+}
+
+// tokenSource checks where a machine token is used from. job is nil for node
+// tokens and phone-home, which are checked against fixed addresses only.
+func (a *App) tokenSource(ctx context.Context, r *http.Request, m *Machine, j *Job, what string) error {
+	if a.cfg.AgentSourceCheck != nil && !*a.cfg.AgentSourceCheck {
+		return nil
+	}
+	ip := a.clientIP(r)
+	want := a.expectedIPs(ctx, m)
+	var err error
+	switch {
+	case len(want) > 0:
+		if !hasTag(want, ip) {
+			err = fmt.Errorf("%s token used from %s; this machine is %s", what, ip, strings.Join(want, ", "))
+		}
+	case j != nil:
+		bound, berr := a.store.bindJobIP(ctx, j.ID, ip)
+		if berr != nil {
+			return berr
+		}
+		if bound != ip {
+			err = fmt.Errorf("%s token used from %s; job %s is bound to %s, where it was first used", what, ip, j.ID, bound)
+		}
+	}
+	if err != nil {
+		_ = a.store.AddEvent(ctx, m.ID, "token refused", err.Error())
+		log.Printf("%s: %v", m.ID, err)
+	}
+	return err
+}

cmd/rollcall/config.go

@@ -47,7 +47,9 @@ type Config struct {
 	CADir     string   `json:"ca_dir"`     // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca)
 	TLSListen string   `json:"tls_listen"` // HTTPS listener, default ":443"; "off" disables
 	TLSNames  []string `json:"tls_names"`  // extra names/addresses for the region certificate
-	AdminHTTP string   `json:"admin_http"` // admin API over plain HTTP: loopback (default) | allow | deny
+	// AgentSourceCheck: machine tokens only from the machine's own address (default true; see agentsrc.go)
+	AgentSourceCheck *bool  `json:"agent_source_check,omitempty"`
+	AdminHTTP        string `json:"admin_http"` // admin API over plain HTTP: loopback (default) | allow | deny
 
 	// a deploy/release job the machine has not started by then fails (default 30m;
 	// servers can take 4-5 minutes to POST, longer when many boot together)

cmd/rollcall/deploy.go

@@ -689,11 +689,28 @@ func (a *App) jobFromRequest(w http.ResponseWriter, r *http.Request) (*Job, stri
 	return j, tok
 }
 
+// jobMachine checks where the job token came from; nil when refused (answered).
+func (a *App) jobMachine(w http.ResponseWriter, r *http.Request, j *Job) *Machine {
+	m, err := a.store.Get(r.Context(), j.MachineID)
+	if err != nil {
+		fail(w, 500, err)
+		return nil
+	}
+	if err := a.tokenSource(r.Context(), r, m, j, "job"); err != nil {
+		fail(w, 403, err)
+		return nil
+	}
+	return m
+}
+
 func (a *App) handleJobSelf(w http.ResponseWriter, r *http.Request) {
 	j, tok := a.jobFromRequest(w, r)
 	if j == nil {
 		return
 	}
+	if a.jobMachine(w, r, j) == nil {
+		return
+	}
 	if j.State != "pending" && j.State != "running" {
 		fail(w, 409, fmt.Errorf("job %s is %s", j.ID, j.State))
 		return
@@ -731,13 +748,12 @@ func (a *App) handleJobProgress(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, 200, obj{"job": j.ID, "state": j.State, "next": "poweroff"})
 		return
 	}
-	prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()})
-	_ = a.store.JobProgress(r.Context(), j.ID, prog)
-	m, err := a.store.Get(r.Context(), j.MachineID)
-	if err != nil {
-		fail(w, 500, err)
+	m := a.jobMachine(w, r, j)
+	if m == nil {
 		return
 	}
+	prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()})
+	_ = a.store.JobProgress(r.Context(), j.ID, prog)
 	next := "continue"
 	switch p.State {
 	case "done":
@@ -766,6 +782,10 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) {
 		fail(w, 404, errNotFound)
 		return
 	}
+	if m, err := a.store.Get(r.Context(), j.MachineID); err != nil || a.tokenSource(r.Context(), r, m, nil, "phone-home") != nil {
+		fail(w, 403, errors.New("not from this machine"))
+		return
+	}
 	_ = r.ParseForm()
 	_ = a.store.MarkPhoneHome(r.Context(), j.MachineID)
 	detail := fmt.Sprintf("job=%s from %s hostname=%s fqdn=%s", j.ID, r.RemoteAddr, r.Form.Get("hostname"), r.Form.Get("fqdn"))

cmd/rollcall/security_test.go

@@ -124,3 +124,21 @@ func TestBMCHintHasNoPassword(t *testing.T) {
 		t.Error("garbage matched a hint")
 	}
 }
+
+// X-Forwarded-For counts only from a rack controller, and only its own entry.
+func TestClientIP(t *testing.T) {
+	a := &App{cfg: &Config{RackName: "region", RackSecret: "rs"}}
+	r := httptest.NewRequest("GET", "/api/v1/jobs/self", nil)
+	r.RemoteAddr = "10.213.77.60:41000"
+	r.Header.Set("X-Forwarded-For", "10.213.77.50")
+	if ip := a.clientIP(r); ip != "10.213.77.60" {
+		t.Errorf("a machine's own X-Forwarded-For was believed: %s", ip)
+	}
+	r.RemoteAddr = "10.0.0.151:5000"
+	r.Header.Set("X-Rollcall-Rack", "hall-b")
+	r.Header.Set("X-Rollcall-Rack-Secret", "rs")
+	r.Header.Set("X-Forwarded-For", "10.213.78.99, 10.213.78.50")
+	if ip := a.clientIP(r); ip != "10.213.78.50" {
+		t.Errorf("through a rack: %s", ip)
+	}
+}

cmd/rollcall/stateless.go

@@ -876,6 +876,10 @@ func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) {
 		fail(w, 401, errors.New("unknown node token"))
 		return
 	}
+	if err := a.tokenSource(r.Context(), r, m, nil, "node"); err != nil {
+		fail(w, 403, err)
+		return
+	}
 	if m.State != "booted" {
 		_ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr)
 		log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr)

cmd/rollcall/store.go

@@ -73,6 +73,7 @@ CREATE TABLE IF NOT EXISTS jobs (
   finished   timestamptz,
   error      text
 );
+ALTER TABLE jobs ADD COLUMN IF NOT EXISTS bound_ip text;
 CREATE INDEX IF NOT EXISTS jobs_machine ON jobs(machine_id, created);
 CREATE INDEX IF NOT EXISTS jobs_token ON jobs(token_hash);
 CREATE TABLE IF NOT EXISTS tags (