Wait for a new machine's own apt, and retry a failed install sooner
Found by building a node from nothing. Its agent tried to install libgomp1 while cloud-init and unattended-upgrades were running apt, and failed with "Unable to lock directory /var/lib/apt/lists/": DPkg::Lock::Timeout covers dpkg's lock, not the one `apt-get update` takes on its lists. The node was correctly kept out of placement and said why, and then sat idle for the ten minutes until the next pass. A package command that fails only on a held lock is now retried for up to ten minutes, reported as "waiting for another apt-get to finish" rather than as a failure. A pass that does fail is retried after a minute, backing off to ten. The weights directory is created when the agent starts rather than by the first download, so a new node reports its disk from its first heartbeat. Onboarding takes several control plane addresses, comma separated, so a new node gets the same failover list as the rest. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8097a5616d
commit
a40fc53e09
4 files changed
+107
-7
No files matched your search
@@ -283,13 +283,57 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
|
||||
}
|
||||
in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
|
||||
for _, c := range cmds {
|
||||
if out, err := in.run(ctx, pm.env, c[0], c[1:]...); err != nil {
|
||||
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
|
||||
if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockWait bounds how long a package command waits for another to finish.
|
||||
const lockWait = 10 * time.Minute
|
||||
|
||||
// lockRetry is how often a locked package manager is tried again.
|
||||
var lockRetry = 10 * time.Second
|
||||
|
||||
// runWaitingForLock runs a package command, waiting while another holds the
|
||||
// package manager's lock.
|
||||
//
|
||||
// A machine a minute old is running cloud-init's and unattended-upgrades' own
|
||||
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
|
||||
// update` takes on its lists, so the first install on a new node failed with
|
||||
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
|
||||
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
|
||||
deadline := time.Now().Add(lockWait)
|
||||
for {
|
||||
out, err := in.run(ctx, env, c[0], c[1:]...)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !lockHeld(string(out)) || time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
|
||||
}
|
||||
in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(lockRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lockHeld reports whether a package command failed only because another
|
||||
// held the lock.
|
||||
func lockHeld(out string) bool {
|
||||
for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
|
||||
"waiting for process with pid", "System management is locked"} {
|
||||
if strings.Contains(out, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// packageManager is how one packaging family asks and installs.
|
||||
type packageManager struct {
|
||||
// query exits zero when the package named after it is installed.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
@@ -89,6 +90,8 @@ type fakeSystem struct {
|
||||
tools map[string]bool
|
||||
lddOut string
|
||||
calls []string
|
||||
// locked is how many more package commands find the lock held.
|
||||
locked int
|
||||
}
|
||||
|
||||
var errExit = errors.New("exit 1")
|
||||
@@ -110,6 +113,10 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
|
||||
}
|
||||
return nil, errExit
|
||||
case name == "apt-get" || name == "dnf":
|
||||
if f.locked > 0 {
|
||||
f.locked--
|
||||
return []byte("E: Unable to lock directory /var/lib/apt/lists/\n"), errExit
|
||||
}
|
||||
for i, a := range args {
|
||||
if a == "install" {
|
||||
for _, p := range args[i+1:] {
|
||||
@@ -300,3 +307,26 @@ func TestASystemTooOldForTheBuildSaysWhatItLacks(t *testing.T) {
|
||||
t.Errorf("the same shortfall was listed more than once: %q", got[0].Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
|
||||
// A new node's first minutes: cloud-init and unattended-upgrades hold
|
||||
// apt's lists lock, which DPkg::Lock::Timeout does not cover.
|
||||
lockRetry = time.Millisecond
|
||||
t.Cleanup(func() { lockRetry = 10 * time.Second })
|
||||
|
||||
rel := llamaRelease(t)
|
||||
sys := &fakeSystem{installed: map[string]bool{}, locked: 3}
|
||||
in, seen := installer(t, rel.serve(t), sys, ubuntu)
|
||||
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("gave up on a lock another apt held: %+v", got[0])
|
||||
}
|
||||
waited := false
|
||||
for _, s := range *seen {
|
||||
waited = waited || strings.Contains(s.Detail, "waiting for another apt-get")
|
||||
}
|
||||
if !waited {
|
||||
t.Error("the wait was not reported")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user