Lost power commands are visible; jobs not started in job_start_timeout fail
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Successful in 37s

pxeBoot dropped the errors of its BMC calls, so a command lost while the
rack or region restarted produced an empty, success-looking event and the
job waited forever. Errors now go into the event (pxe boot failed), and a
watchdog fails jobs the machine has not started within job_start_timeout
(default 30m: real servers take 4-5 minutes to POST, longer in bulk).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-28 19:23:38 +01:00
1 parent 4b30c91c81
commit 996ecbd89d
6 files changed
+83 -2

No files matched your search

+47
View File
@@ -731,3 +731,50 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) {
log.Printf("phone-home %s: %s", j.MachineID, detail)
writeJSON(w, 200, obj{"ok": true})
}
// jobStartTimeout: a deploy or release job the machine has not picked up by then
// is failed, so a lost power command or a machine that did not PXE-boot is
// visible instead of an endless "pending". Real servers spend 4-5 minutes in
// POST on a cold start (more after a firmware or memory change), and many
// machines booting at once share the PXE and image servers: the default leaves
// room for that; job_start_timeout in config.json changes it.
const jobStartTimeout = 30 * time.Minute
func (a *App) jobStartTimeout() time.Duration {
if d, err := time.ParseDuration(a.cfg.JobStartTimeout); err == nil && d >= time.Minute {
return d
}
return jobStartTimeout
}
// staleJobs picks the pending jobs created before cutoff.
func staleJobs(active map[string]*Job, cutoff time.Time) []*Job {
var out []*Job
for _, j := range active {
if j.State == "pending" && j.Created.Before(cutoff) {
out = append(out, j)
}
}
sort.Slice(out, func(i, k int) bool { return out[i].Created.Before(out[k].Created) })
return out
}
func (a *App) jobWatchdog() {
for range time.Tick(time.Minute) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
active, err := a.store.ActiveJobs(ctx)
if err == nil {
limit := a.jobStartTimeout()
for _, j := range staleJobs(active, time.Now().Add(-limit)) {
m, err := a.store.Get(ctx, j.MachineID)
if err != nil {
continue
}
msg := fmt.Sprintf("the machine did not start the job within %s: check its power, that it PXE-boots on the provisioning network, and its console for a hung POST (events show the power command's result; job_start_timeout raises the limit)", limit)
_ = a.store.FinishJob(ctx, j.ID, "failed", msg)
a.afterJob(ctx, m, j, "failed", msg)
}
}
cancel()
}
}