Wait for a new machine's own apt, and retry a failed install sooner

Found by building a node from nothing. Its agent tried to install libgomp1
while cloud-init and unattended-upgrades were running apt, and failed with
"Unable to lock directory /var/lib/apt/lists/": DPkg::Lock::Timeout covers
dpkg's lock, not the one `apt-get update` takes on its lists. The node was
correctly kept out of placement and said why, and then sat idle for the ten
minutes until the next pass.

A package command that fails only on a held lock is now retried for up to
ten minutes, reported as "waiting for another apt-get to finish" rather than
as a failure. A pass that does fail is retried after a minute, backing off to
ten.

The weights directory is created when the agent starts rather than by the
first download, so a new node reports its disk from its first heartbeat.

Onboarding takes several control plane addresses, comma separated, so a new
node gets the same failover list as the rest.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 12:14:22 +02:00
1 parent 8097a5616d
commit a40fc53e09
4 files changed
+107 -7

No files matched your search

+21 -3
View File
@@ -409,6 +409,11 @@ func (a *Agent) Run(ctx context.Context) error {
a.log.Error("hardware discovery failed", "err", err)
}
a.recoverOrphans()
// Made now rather than by the first download, so a new node reports its
// disk from its first heartbeat instead of only once it holds a model.
if err := os.MkdirAll(a.cfg.WeightsCache, 0o755); err != nil {
a.log.Warn("cannot create the weights directory", "dir", a.cfg.WeightsCache, "err", err)
}
go a.provisionLoop(ctx)
attempt := 0
@@ -1265,14 +1270,17 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) {
// provisionLoop installs and re-checks the wanted engines, one pass at a time.
func (a *Agent) provisionLoop(ctx context.Context) {
t := time.NewTicker(provisionEvery)
defer t.Stop()
// After a failure the next pass comes sooner, and backs off: a lock held
// by a new machine's own updates clears in a minute or two, while ten
// minutes' wait left a node idle for no reason.
next := provisionEvery
retry := time.Minute
for {
select {
case <-ctx.Done():
return
case <-a.provKick:
case <-t.C:
case <-time.After(next):
}
a.mu.Lock()
wants := slices.Clone(a.provWants)
@@ -1283,6 +1291,16 @@ func (a *Agent) provisionLoop(ctx context.Context) {
}
statuses := a.prov.Converge(ctx, wants)
next = provisionEvery
for _, st := range statuses {
if st.State == provision.Failed {
next, retry = retry, min(retry*2, provisionEvery)
break
}
}
if next == provisionEvery {
retry = time.Minute
}
installed := map[string]string{}
keep := map[string]bool{}
for _, st := range statuses {