Lost power commands are visible; jobs not started in job_start_timeout fail
pxeBoot dropped the errors of its BMC calls, so a command lost while the rack or region restarted produced an empty, success-looking event and the job waited forever. Errors now go into the event (pxe boot failed), and a watchdog fails jobs the machine has not started within job_start_timeout (default 30m: real servers take 4-5 minutes to POST, longer in bulk). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4b30c91c81
commit
996ecbd89d
6 files changed
+83
-2
No files matched your search
@@ -731,3 +731,50 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("phone-home %s: %s", j.MachineID, detail)
|
||||
writeJSON(w, 200, obj{"ok": true})
|
||||
}
|
||||
|
||||
// jobStartTimeout: a deploy or release job the machine has not picked up by then
|
||||
// is failed, so a lost power command or a machine that did not PXE-boot is
|
||||
// visible instead of an endless "pending". Real servers spend 4-5 minutes in
|
||||
// POST on a cold start (more after a firmware or memory change), and many
|
||||
// machines booting at once share the PXE and image servers: the default leaves
|
||||
// room for that; job_start_timeout in config.json changes it.
|
||||
const jobStartTimeout = 30 * time.Minute
|
||||
|
||||
func (a *App) jobStartTimeout() time.Duration {
|
||||
if d, err := time.ParseDuration(a.cfg.JobStartTimeout); err == nil && d >= time.Minute {
|
||||
return d
|
||||
}
|
||||
return jobStartTimeout
|
||||
}
|
||||
|
||||
// staleJobs picks the pending jobs created before cutoff.
|
||||
func staleJobs(active map[string]*Job, cutoff time.Time) []*Job {
|
||||
var out []*Job
|
||||
for _, j := range active {
|
||||
if j.State == "pending" && j.Created.Before(cutoff) {
|
||||
out = append(out, j)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, k int) bool { return out[i].Created.Before(out[k].Created) })
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) jobWatchdog() {
|
||||
for range time.Tick(time.Minute) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
active, err := a.store.ActiveJobs(ctx)
|
||||
if err == nil {
|
||||
limit := a.jobStartTimeout()
|
||||
for _, j := range staleJobs(active, time.Now().Add(-limit)) {
|
||||
m, err := a.store.Get(ctx, j.MachineID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
msg := fmt.Sprintf("the machine did not start the job within %s: check its power, that it PXE-boots on the provisioning network, and its console for a hung POST (events show the power command's result; job_start_timeout raises the limit)", limit)
|
||||
_ = a.store.FinishJob(ctx, j.ID, "failed", msg)
|
||||
a.afterJob(ctx, m, j, "failed", msg)
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user