Wait for a new machine's own apt, and retry a failed install sooner
Found by building a node from nothing. Its agent tried to install libgomp1 while cloud-init and unattended-upgrades were running apt, and failed with "Unable to lock directory /var/lib/apt/lists/": DPkg::Lock::Timeout covers dpkg's lock, not the one `apt-get update` takes on its lists. The node was correctly kept out of placement and said why, and then sat idle for the ten minutes until the next pass. A package command that fails only on a held lock is now retried for up to ten minutes, reported as "waiting for another apt-get to finish" rather than as a failure. A pass that does fail is retried after a minute, backing off to ten. The weights directory is created when the agent starts rather than by the first download, so a new node reports its disk from its first heartbeat. Onboarding takes several control plane addresses, comma separated, so a new node gets the same failover list as the rest. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8097a5616d
commit
a40fc53e09
4 files changed
+107
-7
No files matched your search
+21
-3
@@ -409,6 +409,11 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
a.log.Error("hardware discovery failed", "err", err)
|
||||
}
|
||||
a.recoverOrphans()
|
||||
// Made now rather than by the first download, so a new node reports its
|
||||
// disk from its first heartbeat instead of only once it holds a model.
|
||||
if err := os.MkdirAll(a.cfg.WeightsCache, 0o755); err != nil {
|
||||
a.log.Warn("cannot create the weights directory", "dir", a.cfg.WeightsCache, "err", err)
|
||||
}
|
||||
go a.provisionLoop(ctx)
|
||||
|
||||
attempt := 0
|
||||
@@ -1265,14 +1270,17 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) {
|
||||
|
||||
// provisionLoop installs and re-checks the wanted engines, one pass at a time.
|
||||
func (a *Agent) provisionLoop(ctx context.Context) {
|
||||
t := time.NewTicker(provisionEvery)
|
||||
defer t.Stop()
|
||||
// After a failure the next pass comes sooner, and backs off: a lock held
|
||||
// by a new machine's own updates clears in a minute or two, while ten
|
||||
// minutes' wait left a node idle for no reason.
|
||||
next := provisionEvery
|
||||
retry := time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-a.provKick:
|
||||
case <-t.C:
|
||||
case <-time.After(next):
|
||||
}
|
||||
a.mu.Lock()
|
||||
wants := slices.Clone(a.provWants)
|
||||
@@ -1283,6 +1291,16 @@ func (a *Agent) provisionLoop(ctx context.Context) {
|
||||
}
|
||||
|
||||
statuses := a.prov.Converge(ctx, wants)
|
||||
next = provisionEvery
|
||||
for _, st := range statuses {
|
||||
if st.State == provision.Failed {
|
||||
next, retry = retry, min(retry*2, provisionEvery)
|
||||
break
|
||||
}
|
||||
}
|
||||
if next == provisionEvery {
|
||||
retry = time.Minute
|
||||
}
|
||||
installed := map[string]string{}
|
||||
keep := map[string]bool{}
|
||||
for _, st := range statuses {
|
||||
|
||||
Reference in new issue
Block a user