Wait for a new machine's own apt, and retry a failed install sooner

Found by building a node from nothing. Its agent tried to install libgomp1
while cloud-init and unattended-upgrades were running apt, and failed with
"Unable to lock directory /var/lib/apt/lists/": DPkg::Lock::Timeout covers
dpkg's lock, not the one `apt-get update` takes on its lists. The node was
correctly kept out of placement and said why, and then sat idle for the ten
minutes until the next pass.

A package command that fails only on a held lock is now retried for up to
ten minutes, reported as "waiting for another apt-get to finish" rather than
as a failure. A pass that does fail is retried after a minute, backing off to
ten.

The weights directory is created when the agent starts rather than by the
first download, so a new node reports its disk from its first heartbeat.

Onboarding takes several control plane addresses, comma separated, so a new
node gets the same failover list as the rest.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 12:14:22 +02:00
1 parent 8097a5616d
commit a40fc53e09
4 files changed
+107 -7

No files matched your search

+46 -2
View File
@@ -283,13 +283,57 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
}
in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
for _, c := range cmds {
if out, err := in.run(ctx, pm.env, c[0], c[1:]...); err != nil {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
return err
}
}
return nil
}
// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute
// lockRetry is how often a locked package manager is tried again.
var lockRetry = 10 * time.Second
// runWaitingForLock runs a package command, waiting while another holds the
// package manager's lock.
//
// A machine a minute old is running cloud-init's and unattended-upgrades' own
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
// update` takes on its lists, so the first install on a new node failed with
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
deadline := time.Now().Add(lockWait)
for {
out, err := in.run(ctx, env, c[0], c[1:]...)
if err == nil {
return nil
}
if !lockHeld(string(out)) || time.Now().After(deadline) {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
}
in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(lockRetry):
}
}
}
// lockHeld reports whether a package command failed only because another
// held the lock.
func lockHeld(out string) bool {
for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
"waiting for process with pid", "System management is locked"} {
if strings.Contains(out, s) {
return true
}
}
return false
}
// packageManager is how one packaging family asks and installs.
type packageManager struct {
// query exits zero when the package named after it is installed.