diff --git a/cmd/rollcall/agentsrc.go b/cmd/rollcall/agentsrc.go new file mode 100644 index 0000000..7197f1d --- /dev/null +++ b/cmd/rollcall/agentsrc.go @@ -0,0 +1,101 @@ +package main + +// Where machine-side tokens may be used from. Job tokens (rollcall.job=) and +// node tokens (rollcall.node=) travel in the per-MAC pxelinux file, which any +// host on the provisioning network can fetch over TFTP; without a hardware +// root of trust the machine holds no secret of its own. What the region does +// know is the address it gives the machine (a static DHCP lease from the +// network model), so a token is only accepted from there. A machine without +// a fixed address (proxy DHCP, a site's DHCP server) binds a job to the +// first address that uses it. Refusals are events on the machine. +// +// agent_source_check: false in config.json turns this off (NAT between the +// machines and Rollcall). + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "strings" +) + +// clientIP is the machine's address: a remote rack controller proxies the +// node API and appends it to X-Forwarded-For (the last entry is the rack's). +func (a *App) clientIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + parts := strings.Split(xff, ",") + host = strings.TrimSpace(parts[len(parts)-1]) + } + } + if ad, ok := parseAddr(host); ok { + return ad.String() + } + return host +} + +// expectedIPs are the addresses the region hands this machine itself. +func (a *App) expectedIPs(ctx context.Context, m *Machine) []string { + var out []string + add := func(ip string) { + if ad, ok := parseAddr(ip); ok && !hasTag(out, ad.String()) { + out = append(out, ad.String()) + } + } + subnets := a.subnetsSnapshot() + if ips, err := a.store.IPs(ctx, "", m.ID); err == nil { + for _, x := range ips { + if sn := subnetFor(subnets, x.IP); sn == nil || sn.DHCPMode != "proxy" { + add(x.IP) + } + } + } + if mg := deref(m.MgmtIP); mg != "" { + if sn := subnetFor(subnets, mg); sn == nil || sn.DHCPMode != "proxy" { + add(mg) + } + } + return out +} + +func (s *Store) bindJobIP(ctx context.Context, id, ip string) (string, error) { + var bound string + err := s.pool.QueryRow(ctx, `UPDATE jobs SET bound_ip=coalesce(bound_ip, $2) WHERE id=$1 RETURNING bound_ip`, id, ip).Scan(&bound) + return bound, err +} + +// tokenSource checks where a machine token is used from. job is nil for node +// tokens and phone-home, which are checked against fixed addresses only. +func (a *App) tokenSource(ctx context.Context, r *http.Request, m *Machine, j *Job, what string) error { + if a.cfg.AgentSourceCheck != nil && !*a.cfg.AgentSourceCheck { + return nil + } + ip := a.clientIP(r) + want := a.expectedIPs(ctx, m) + var err error + switch { + case len(want) > 0: + if !hasTag(want, ip) { + err = fmt.Errorf("%s token used from %s; this machine is %s", what, ip, strings.Join(want, ", ")) + } + case j != nil: + bound, berr := a.store.bindJobIP(ctx, j.ID, ip) + if berr != nil { + return berr + } + if bound != ip { + err = fmt.Errorf("%s token used from %s; job %s is bound to %s, where it was first used", what, ip, j.ID, bound) + } + } + if err != nil { + _ = a.store.AddEvent(ctx, m.ID, "token refused", err.Error()) + log.Printf("%s: %v", m.ID, err) + } + return err +} diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index d60e34f..3073e23 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -47,7 +47,9 @@ type Config struct { CADir string `json:"ca_dir"` // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca) TLSListen string `json:"tls_listen"` // HTTPS listener, default ":443"; "off" disables TLSNames []string `json:"tls_names"` // extra names/addresses for the region certificate - AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: loopback (default) | allow | deny + // AgentSourceCheck: machine tokens only from the machine's own address (default true; see agentsrc.go) + AgentSourceCheck *bool `json:"agent_source_check,omitempty"` + AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: loopback (default) | allow | deny // a deploy/release job the machine has not started by then fails (default 30m; // servers can take 4-5 minutes to POST, longer when many boot together) diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index ffb3793..4e1a3d5 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -689,11 +689,28 @@ func (a *App) jobFromRequest(w http.ResponseWriter, r *http.Request) (*Job, stri return j, tok } +// jobMachine checks where the job token came from; nil when refused (answered). +func (a *App) jobMachine(w http.ResponseWriter, r *http.Request, j *Job) *Machine { + m, err := a.store.Get(r.Context(), j.MachineID) + if err != nil { + fail(w, 500, err) + return nil + } + if err := a.tokenSource(r.Context(), r, m, j, "job"); err != nil { + fail(w, 403, err) + return nil + } + return m +} + func (a *App) handleJobSelf(w http.ResponseWriter, r *http.Request) { j, tok := a.jobFromRequest(w, r) if j == nil { return } + if a.jobMachine(w, r, j) == nil { + return + } if j.State != "pending" && j.State != "running" { fail(w, 409, fmt.Errorf("job %s is %s", j.ID, j.State)) return @@ -731,13 +748,12 @@ func (a *App) handleJobProgress(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, obj{"job": j.ID, "state": j.State, "next": "poweroff"}) return } - prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()}) - _ = a.store.JobProgress(r.Context(), j.ID, prog) - m, err := a.store.Get(r.Context(), j.MachineID) - if err != nil { - fail(w, 500, err) + m := a.jobMachine(w, r, j) + if m == nil { return } + prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()}) + _ = a.store.JobProgress(r.Context(), j.ID, prog) next := "continue" switch p.State { case "done": @@ -766,6 +782,10 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) { fail(w, 404, errNotFound) return } + if m, err := a.store.Get(r.Context(), j.MachineID); err != nil || a.tokenSource(r.Context(), r, m, nil, "phone-home") != nil { + fail(w, 403, errors.New("not from this machine")) + return + } _ = r.ParseForm() _ = a.store.MarkPhoneHome(r.Context(), j.MachineID) detail := fmt.Sprintf("job=%s from %s hostname=%s fqdn=%s", j.ID, r.RemoteAddr, r.Form.Get("hostname"), r.Form.Get("fqdn")) diff --git a/cmd/rollcall/security_test.go b/cmd/rollcall/security_test.go index 6ae68d7..6c12c5e 100644 --- a/cmd/rollcall/security_test.go +++ b/cmd/rollcall/security_test.go @@ -124,3 +124,21 @@ func TestBMCHintHasNoPassword(t *testing.T) { t.Error("garbage matched a hint") } } + +// X-Forwarded-For counts only from a rack controller, and only its own entry. +func TestClientIP(t *testing.T) { + a := &App{cfg: &Config{RackName: "region", RackSecret: "rs"}} + r := httptest.NewRequest("GET", "/api/v1/jobs/self", nil) + r.RemoteAddr = "10.213.77.60:41000" + r.Header.Set("X-Forwarded-For", "10.213.77.50") + if ip := a.clientIP(r); ip != "10.213.77.60" { + t.Errorf("a machine's own X-Forwarded-For was believed: %s", ip) + } + r.RemoteAddr = "10.0.0.151:5000" + r.Header.Set("X-Rollcall-Rack", "hall-b") + r.Header.Set("X-Rollcall-Rack-Secret", "rs") + r.Header.Set("X-Forwarded-For", "10.213.78.99, 10.213.78.50") + if ip := a.clientIP(r); ip != "10.213.78.50" { + t.Errorf("through a rack: %s", ip) + } +} diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index 8e33551..e4faf66 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -876,6 +876,10 @@ func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) { fail(w, 401, errors.New("unknown node token")) return } + if err := a.tokenSource(r.Context(), r, m, nil, "node"); err != nil { + fail(w, 403, err) + return + } if m.State != "booted" { _ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr) log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr) diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index bb8b492..773608e 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -73,6 +73,7 @@ CREATE TABLE IF NOT EXISTS jobs ( finished timestamptz, error text ); +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS bound_ip text; CREATE INDEX IF NOT EXISTS jobs_machine ON jobs(machine_id, created); CREATE INDEX IF NOT EXISTS jobs_token ON jobs(token_hash); CREATE TABLE IF NOT EXISTS tags (