Upgrade the nodes one at a time on the bus, with a package record, a kernel hold, an after-upgrade check, a rollback and a fallback kernel (upgrade safety, issues 1840 and 1841) #450

Merged
mesh-tools merged 47 commits from feat/upgrade-safety into main 2026-10-22 18:34:16 +00:00
3 changed files with 54 additions and 5 deletions
Showing only changes of commit 76a0a726f0 - Show all commits
+1 -1
View File
@@ -46,7 +46,7 @@ built for the fallback kernel too) after its boot-entries service, so the entry
|---|---|---|
| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly |
| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched |
| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names; the default is left as it is |
| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names, with `GRUB_TOP_LEVEL` naming the kernel the machine booted (`BOOT_IMAGE`; else the non-fallback kernel), since GRUB lists kernels in reverse version order and would otherwise make the fallback kernel the default; `/etc/default/grub` still overrides it when it sets one |
| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) |
| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks |
| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network |
@@ -78,7 +78,8 @@ func newBootBed(t *testing.T) bootBed {
}
write(t, b.script, content, 0o755)
write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644)
write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"$@\" >>"+b.mkconfig+"\n", 0o755)
// the stand-in grub-mkconfig records the kernel it was told to put first, then its arguments.
write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"top=$GRUB_TOP_LEVEL $@\" >>"+b.mkconfig+"\n", 0o755)
return b
}
@@ -164,11 +165,37 @@ func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) {
t.Fatalf("changed: %v %s", err, out)
}
raw, _ := os.ReadFile(b.mkconfig)
if string(raw) != "-o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" {
if string(raw) != "top=/boot/vmlinuz-linux -o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" {
t.Fatalf("grub-mkconfig was asked %q", raw)
}
}
// ADR 0508 §9: GRUB's menu lists kernels in reverse version order, so linux-lts would come first and become the
// default entry (GRUB_DEFAULT=0); the kernel the machine booted is put first, and the fallback kernel never.
func TestGrubKeepsTheBootedKernelFirstAndNeverTheFallback(t *testing.T) {
for name, c := range map[string]struct{ cmdline, want string }{
"booted from the default kernel": {"BOOT_IMAGE=/vmlinuz-linux root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
"booted from a grub device path": {"BOOT_IMAGE=(hd0,msdos1)/vmlinuz-linux-zen root=UUID=x rw\n", "top=/boot/vmlinuz-linux-zen"},
"booted from the fallback kernel": {"BOOT_IMAGE=/vmlinuz-linux-lts root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
"no image on the command line": {"root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
"an image no longer in /boot": {"BOOT_IMAGE=/vmlinuz-linux-gone root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
} {
b := newBootBed(t)
b.put(t, "boot/grub/grub.cfg", grubMenu)
b.put(t, "proc/cmdline", c.cmdline)
for _, k := range []string{"vmlinuz-linux", "vmlinuz-linux-lts", "vmlinuz-linux-zen"} {
b.put(t, "boot/"+k, "k")
}
if out, err := b.run(t); err != nil {
t.Fatalf("%s: %v %s", name, err, out)
}
raw, _ := os.ReadFile(b.mkconfig)
if !strings.HasPrefix(string(raw), c.want+" -o ") {
t.Errorf("%s: grub-mkconfig was asked %q, want %s first", name, raw, c.want)
}
}
}
func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) {
b := newBootBed(t)
out, err := b.run(t)
+24 -2
View File
@@ -104,8 +104,30 @@ grub)
if [ "$menu" = "$have" ]; then
echo "grub: the menu lists the kernels in /boot, left as it is"
else
grub-mkconfig -o "$cfg"
echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again; the default entry is as it was"
# GRUB lists kernels in reverse version order, so the fallback kernel would come first and become the
# default entry: the kernel the machine booted is put first (GRUB_TOP_LEVEL; /etc/default/grub still
# overrides it when it sets one), the fallback kernel never.
top=$(tr ' ' '\n' <"$root/proc/cmdline" | sed -n 's|^BOOT_IMAGE=\(.*\)$|\1|p' | head -n 1)
top=$(printf '%s' "$top" | sed 's|.*/||')
case "$top" in
vmlinuz-linux-lts | "") top= ;;
esac
[ -n "$top" ] && [ -e "$boot/$top" ] || top=
if [ -z "$top" ]; then
if [ -e "$boot/vmlinuz-linux" ]; then
top=vmlinuz-linux
else
top=$(printf '%s\n' $have | grep -v '^vmlinuz-linux-lts$' | head -n 1 || true)
fi
fi
if [ -n "$top" ]; then
GRUB_TOP_LEVEL=/boot/$top grub-mkconfig -o "$cfg"
else
grub-mkconfig -o "$cfg"
fi
first=$top
[ -n "$first" ] || first="the menu's own order"
echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again with $first first; the default entry is as it was"
fi
;;
esac