Wait for a new machine's own apt, and retry a failed install sooner
Found by building a node from nothing. Its agent tried to install libgomp1 while cloud-init and unattended-upgrades were running apt, and failed with "Unable to lock directory /var/lib/apt/lists/": DPkg::Lock::Timeout covers dpkg's lock, not the one `apt-get update` takes on its lists. The node was correctly kept out of placement and said why, and then sat idle for the ten minutes until the next pass. A package command that fails only on a held lock is now retried for up to ten minutes, reported as "waiting for another apt-get to finish" rather than as a failure. A pass that does fail is retried after a minute, backing off to ten. The weights directory is created when the agent starts rather than by the first download, so a new node reports its disk from its first heartbeat. Onboarding takes several control plane addresses, comma separated, so a new node gets the same failover list as the rest. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8097a5616d
commit
a40fc53e09
4 files changed
+107
-7
No files matched your search
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
@@ -89,6 +90,8 @@ type fakeSystem struct {
|
||||
tools map[string]bool
|
||||
lddOut string
|
||||
calls []string
|
||||
// locked is how many more package commands find the lock held.
|
||||
locked int
|
||||
}
|
||||
|
||||
var errExit = errors.New("exit 1")
|
||||
@@ -110,6 +113,10 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
|
||||
}
|
||||
return nil, errExit
|
||||
case name == "apt-get" || name == "dnf":
|
||||
if f.locked > 0 {
|
||||
f.locked--
|
||||
return []byte("E: Unable to lock directory /var/lib/apt/lists/\n"), errExit
|
||||
}
|
||||
for i, a := range args {
|
||||
if a == "install" {
|
||||
for _, p := range args[i+1:] {
|
||||
@@ -300,3 +307,26 @@ func TestASystemTooOldForTheBuildSaysWhatItLacks(t *testing.T) {
|
||||
t.Errorf("the same shortfall was listed more than once: %q", got[0].Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
|
||||
// A new node's first minutes: cloud-init and unattended-upgrades hold
|
||||
// apt's lists lock, which DPkg::Lock::Timeout does not cover.
|
||||
lockRetry = time.Millisecond
|
||||
t.Cleanup(func() { lockRetry = 10 * time.Second })
|
||||
|
||||
rel := llamaRelease(t)
|
||||
sys := &fakeSystem{installed: map[string]bool{}, locked: 3}
|
||||
in, seen := installer(t, rel.serve(t), sys, ubuntu)
|
||||
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("gave up on a lock another apt held: %+v", got[0])
|
||||
}
|
||||
waited := false
|
||||
for _, s := range *seen {
|
||||
waited = waited || strings.Contains(s.Detail, "waiting for another apt-get")
|
||||
}
|
||||
if !waited {
|
||||
t.Error("the wait was not reported")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user