Merge branch 'feat/upgrade-safety' into p1

2026-10-22 14:29:29 +02:00
parent efac1b4856 bc77afa6bb
commit c0bc0d4826
11 changed files with 670 additions and 12 deletions
@@ -1,8 +1,9 @@
 # kernel

-The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
-**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
-entries) is not built yet.
+The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). It holds two things:
+**transparent huge pages**, set per machine, and **the fallback kernel** (`linux-lts` with its headers) and
+its place on the boot menu (novox/hq ADR 0508 §9 and §10). The default kernel's package, and microcode, are
+not built yet.

 ## Why (novox/hq issue 746)

@@ -35,16 +36,34 @@ under the new mode. A failed rotation is said in the unit's journal and does not
 rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
 or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.

+## The fallback kernel (novox/hq ADR 0508, issue 1841)
+
+Every machine keeps a second, older kernel to start from, so a kernel upgrade that does not boot is not the
+end of the machine. The module declares the packages `linux-lts` and `linux-lts-headers` (so every DKMS module is
+built for the fallback kernel too) after its boot-entries service, so the entry is in place before the kernel is.
+
+| | what | notes |
+|---|---|---|
+| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly |
+| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched |
+| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names; the default is left as it is |
+| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) |
+| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks |
+| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network |
+| file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed |
+| packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook |
+
 ## Tools

 | tool | | answers |
 |---|---|---|
+| `kernel_boot` | r | the loader (`systemd-boot`, `grub` or `none`), the menu's entries, the kernels in `/boot`, the entry booted (systemd-boot) or the kernel image booted (GRUB), whether the fallback kernel is on the menu with its kernel in `/boot`, and the room free on `/boot` in KiB |
 | `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |

 ## Tests

 `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
-manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
+manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh
 writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
 rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
 reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
@@ -0,0 +1,89 @@
+package main
+
+// kernel_boot: which loader booted the machine, its entries, the kernels in /boot, the entry it booted and
+// whether the fallback kernel is on the menu (novox/hq ADR 0508 §9, issue 1841). Reads only.
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+)
+
+const loaderSelected = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
+
+// BootState is what kernel_boot answers.
+type BootState struct {
+	Loader          string   `json:"loader"`
+	Entries         []string `json:"entries"`
+	Kernels         []string `json:"kernels"`
+	Booted          string   `json:"booted"`
+	FallbackPresent bool     `json:"fallback_present"`
+	BootFreeKiB     int64    `json:"boot_free_kib"`
+}
+
+var grubKernel = regexp.MustCompile(`(?m)^\s*linux\s+(?:/boot)?/(vmlinuz-\S+)`)
+var grubTitle = regexp.MustCompile(`(?m)^\s*menuentry\s+'([^']*)'`)
+var bootImage = regexp.MustCompile(`BOOT_IMAGE=(?:/boot)?/?(\S+)`)
+
+// Boot reads the machine, the loader the way the boot-entries script does: the loader's EFI variable means
+// systemd-boot, a grub.cfg without it GRUB.
+func (m *Machine) Boot() (*BootState, error) {
+	b := &BootState{Loader: "none", Entries: []string{}, Kernels: []string{}}
+	files, _ := filepath.Glob(m.path("boot/vmlinuz-*"))
+	for _, f := range files {
+		b.Kernels = append(b.Kernels, filepath.Base(f))
+	}
+	sort.Strings(b.Kernels)
+	hasKernel := func(name string) bool {
+		for _, k := range b.Kernels {
+			if k == name {
+				return true
+			}
+		}
+		return false
+	}
+	var s syscall.Statfs_t
+	if err := syscall.Statfs(m.path("boot"), &s); err == nil {
+		b.BootFreeKiB = int64(s.Bavail) * int64(s.Bsize) / 1024
+	}
+	if raw, err := os.ReadFile(m.path(loaderSelected)); err == nil {
+		b.Loader = "systemd-boot"
+		if len(raw) > 4 {
+			var u []uint16
+			for i := 4; i+1 < len(raw); i += 2 {
+				u = append(u, uint16(raw[i])|uint16(raw[i+1])<<8)
+			}
+			b.Booted = strings.TrimRight(string(utf16.Decode(u)), "\x00")
+		}
+		files, _ := filepath.Glob(m.path("boot/loader/entries/*.conf"))
+		for _, f := range files {
+			b.Entries = append(b.Entries, filepath.Base(f))
+		}
+		sort.Strings(b.Entries)
+		if raw, err := os.ReadFile(m.path("boot/loader/entries/mesh-linux-lts.conf")); err == nil {
+			b.FallbackPresent = strings.Contains(string(raw), "/vmlinuz-linux-lts") && hasKernel("vmlinuz-linux-lts")
+		}
+		return b, nil
+	}
+	if raw, err := os.ReadFile(m.path("boot/grub/grub.cfg")); err == nil {
+		b.Loader = "grub"
+		for _, t := range grubTitle.FindAllStringSubmatch(string(raw), -1) {
+			b.Entries = append(b.Entries, t[1])
+		}
+		for _, k := range grubKernel.FindAllStringSubmatch(string(raw), -1) {
+			if k[1] == "vmlinuz-linux-lts" && hasKernel("vmlinuz-linux-lts") {
+				b.FallbackPresent = true
+			}
+		}
+		if cmd, err := m.read("proc/cmdline"); err == nil {
+			if i := bootImage.FindStringSubmatch(cmd); i != nil {
+				b.Booted = i[1]
+			}
+		}
+	}
+	return b, nil
+}
@@ -0,0 +1,90 @@
+package main
+
+// kernel_boot over stand-in roots: a systemd-boot machine with and without the fallback entry, a GRUB machine
+// with and without the fallback kernel in its menu, and a machine with neither (novox/hq ADR 0508 §9, issue 1841).
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const selectedVar = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
+
+func put(t *testing.T, root, p, content string) {
+	t.Helper()
+	full := filepath.Join(root, p)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	write(t, full, content, 0o644)
+}
+
+func bootOf(t *testing.T, root string) *BootState {
+	t.Helper()
+	got, err := (&Machine{Root: root}).Boot()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return got
+}
+
+func TestKernelBootReadsASystemdBootMachineWithItsFallbackEntry(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
+	put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
+	put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\n")
+	put(t, root, "boot/vmlinuz-linux", "k")
+	put(t, root, "boot/vmlinuz-linux-lts", "k")
+	got := bootOf(t, root)
+	if got.Loader != "systemd-boot" || got.Booted != "arch.conf" || !got.FallbackPresent {
+		t.Fatalf("%+v", got)
+	}
+	if len(got.Entries) != 2 || got.Entries[0] != "arch.conf" || got.Entries[1] != "mesh-linux-lts.conf" {
+		t.Fatalf("entries: %v", got.Entries)
+	}
+	if len(got.Kernels) != 2 || got.Kernels[0] != "vmlinuz-linux" || got.Kernels[1] != "vmlinuz-linux-lts" {
+		t.Fatalf("kernels: %v", got.Kernels)
+	}
+	if got.BootFreeKiB <= 0 {
+		t.Fatalf("free: %d", got.BootFreeKiB)
+	}
+}
+
+func TestKernelBootSaysASystemdBootMachineHasNoFallbackYet(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
+	put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
+	put(t, root, "boot/vmlinuz-linux", "k")
+	// The entry file alone is not the fallback: the kernel it names must be in /boot.
+	put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "linux /vmlinuz-linux-lts\n")
+	if got := bootOf(t, root); got.FallbackPresent {
+		t.Fatalf("a fallback entry without its kernel: %+v", got)
+	}
+}
+
+func TestKernelBootReadsAGrubMachine(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, "boot/grub/grub.cfg", grubMenu)
+	put(t, root, "boot/vmlinuz-linux", "k")
+	put(t, root, "proc/cmdline", bootedCmdline)
+	got := bootOf(t, root)
+	if got.Loader != "grub" || got.FallbackPresent || got.Booted != "vmlinuz-linux" {
+		t.Fatalf("%+v", got)
+	}
+	if len(got.Entries) != 2 || got.Entries[0] != "Arch Linux" {
+		t.Fatalf("entries: %v", got.Entries)
+	}
+	put(t, root, "boot/vmlinuz-linux-lts", "k")
+	put(t, root, "boot/grub/grub.cfg", grubMenu+"menuentry 'Arch Linux LTS' {\n\tlinux\t/boot/vmlinuz-linux-lts root=x rw\n}\n")
+	if got := bootOf(t, root); !got.FallbackPresent {
+		t.Fatalf("%+v", got)
+	}
+}
+
+func TestKernelBootSaysWhenThereIsNoLoader(t *testing.T) {
+	got := bootOf(t, t.TempDir())
+	if got.Loader != "none" || got.FallbackPresent || len(got.Entries) != 0 {
+		t.Fatalf("%+v", got)
+	}
+}
@@ -0,0 +1,257 @@
+package main
+
+// The boot-entries script as the mesh writes it, run with sh over a stand-in root (novox/hq ADR 0508 §9 and
+// §10, issue 1841). The fixtures are the shapes read from machines: a systemd-boot entries directory, a GRUB
+// menu, a hook that copies every kernel image to the default kernel's name, and the command line a machine
+// booted with.
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const bootedCmdline = "initrd=\\initramfs-linux.img BOOT_IMAGE=/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3 initrd=\\intel-ucode.img\n"
+
+const defaultEntry = "title Arch Linux\nlinux /vmlinuz-linux\ninitrd /intel-ucode.img\ninitrd /initramfs-linux.img\noptions root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet\n"
+
+const loaderConf = "default arch.conf\ntimeout 3\nconsole-mode max\n"
+
+// grubMenu is the part of a generated grub.cfg that names kernels, for a machine holding one kernel.
+const grubMenu = `menuentry 'Arch Linux' --class arch {
+	set root='hd0,gpt2'
+	linux	/boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
+	initrd	/boot/initramfs-linux.img
+}
+menuentry 'Arch Linux, with fallback initramfs' --class arch {
+	linux	/boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
+	initrd	/boot/initramfs-linux-fallback.img
+}
+`
+
+// kernelCopyingHook copies every installed kernel image to the default kernel's name on the boot partition.
+const kernelCopyingHook = `[Trigger]
+Operation = Install
+Operation = Upgrade
+Type = Path
+Target = usr/lib/modules/*/vmlinuz
+
+[Action]
+Description = Copying the kernel to the default name
+When = PostTransaction
+Exec = /usr/bin/sh -c '/usr/bin/cp /usr/lib/modules/$(uname -r)/vmlinuz /boot/vmlinuz-linux'
+`
+
+// otherHook is a hook of the distribution's shape that has nothing to do with kernel images.
+const otherHook = `[Trigger]
+Operation = Upgrade
+Type = Package
+Target = pacman-mirrorlist
+
+[Action]
+Description = Updating the mirror list
+When = PostTransaction
+Exec = /usr/bin/cp /etc/pacman.d/mirrorlist.pacnew /etc/pacman.d/mirrorlist
+`
+
+type bootBed struct {
+	script, root, state, bin, mkconfig string
+	need                               string
+}
+
+func newBootBed(t *testing.T) bootBed {
+	t.Helper()
+	dir := t.TempDir()
+	b := bootBed{script: filepath.Join(dir, "boot-entries"), root: filepath.Join(dir, "root"), state: filepath.Join(dir, "state"),
+		bin: filepath.Join(dir, "bin"), mkconfig: filepath.Join(dir, "mkconfig")}
+	content := manifest(t).resource(t, "boot-entries").text(t)
+	content = strings.ReplaceAll(content, "${dir:state}", b.state)
+	if strings.Contains(content, "${") {
+		t.Fatalf("a substitution the test does not make is left in the script")
+	}
+	for _, d := range []string{b.root + "/boot", b.root + "/proc", b.root + "/etc/pacman.d/hooks", b.state, b.bin} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write(t, b.script, content, 0o755)
+	write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644)
+	write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"$@\" >>"+b.mkconfig+"\n", 0o755)
+	return b
+}
+
+func (b bootBed) put(t *testing.T, p, content string) {
+	t.Helper()
+	full := filepath.Join(b.root, p)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	write(t, full, content, 0o644)
+}
+
+func (b bootBed) systemdBoot(t *testing.T) {
+	b.put(t, "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f", "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
+	b.put(t, "boot/loader/loader.conf", loaderConf)
+	b.put(t, "boot/loader/entries/arch.conf", defaultEntry)
+	b.put(t, "boot/vmlinuz-linux", "k")
+}
+
+func (b bootBed) run(t *testing.T) (string, error) {
+	t.Helper()
+	args := []string{b.script, "run", b.root, b.state}
+	if b.need != "" {
+		args = append(args, b.need)
+	}
+	cmd := exec.Command("sh", args...)
+	cmd.Env = append(os.Environ(), "PATH="+b.bin+string(os.PathListSeparator)+os.Getenv("PATH"))
+	out, err := cmd.CombinedOutput()
+	return string(out), err
+}
+
+func (b bootBed) read(t *testing.T, p string) string {
+	t.Helper()
+	raw, _ := os.ReadFile(filepath.Join(b.root, p))
+	return string(raw)
+}
+
+func TestASystemdBootMachineGetsTheFallbackEntryFromItsBootedCommandLine(t *testing.T) {
+	b := newBootBed(t)
+	b.systemdBoot(t)
+	b.put(t, "boot/intel-ucode.img", "u")
+	out, err := b.run(t)
+	if err != nil || !strings.Contains(out, "wrote") {
+		t.Fatalf("%v %s", err, out)
+	}
+	want := "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\ninitrd /intel-ucode.img\ninitrd /initramfs-linux-lts.img\n" +
+		"options root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3\n"
+	if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); got != want {
+		t.Fatalf("entry:\n%s\nwant:\n%s", got, want)
+	}
+	// The default entry and the loader's own file are as they were.
+	if b.read(t, "boot/loader/entries/arch.conf") != defaultEntry || b.read(t, "boot/loader/loader.conf") != loaderConf {
+		t.Fatalf("the default entry or loader.conf was touched")
+	}
+	if out, err := b.run(t); err != nil || !strings.Contains(out, "as it was") {
+		t.Fatalf("again: %v %s", err, out)
+	}
+}
+
+func TestWithoutMicrocodeThereIsNoMicrocodeLine(t *testing.T) {
+	b := newBootBed(t)
+	b.systemdBoot(t)
+	if out, err := b.run(t); err != nil {
+		t.Fatalf("%v %s", err, out)
+	}
+	if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); strings.Contains(got, "ucode") || !strings.Contains(got, "initrd /initramfs-linux-lts.img") {
+		t.Fatalf("entry: %s", got)
+	}
+}
+
+func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) {
+	b := newBootBed(t)
+	b.put(t, "boot/grub/grub.cfg", grubMenu)
+	b.put(t, "boot/vmlinuz-linux", "k")
+	if out, err := b.run(t); err != nil || !strings.Contains(out, "left as it is") {
+		t.Fatalf("same: %v %s", err, out)
+	}
+	if _, err := os.Stat(b.mkconfig); err == nil {
+		t.Fatalf("the menu was generated again with the same kernels")
+	}
+	b.put(t, "boot/vmlinuz-linux-lts", "k")
+	if out, err := b.run(t); err != nil || !strings.Contains(out, "generated again") {
+		t.Fatalf("changed: %v %s", err, out)
+	}
+	raw, _ := os.ReadFile(b.mkconfig)
+	if string(raw) != "-o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" {
+		t.Fatalf("grub-mkconfig was asked %q", raw)
+	}
+}
+
+func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) {
+	b := newBootBed(t)
+	out, err := b.run(t)
+	if err == nil || !strings.Contains(out, "no boot loader found") {
+		t.Fatalf("%v %s", err, out)
+	}
+}
+
+func TestABootPartitionWithoutRoomFailsBeforeAnythingIsWritten(t *testing.T) {
+	b := newBootBed(t)
+	b.systemdBoot(t)
+	b.need = "999999999999"
+	out, err := b.run(t)
+	if err == nil || !strings.Contains(out, "free room there before the kernel is installed") {
+		t.Fatalf("%v %s", err, out)
+	}
+	if b.read(t, "boot/loader/entries/mesh-linux-lts.conf") != "" {
+		t.Fatalf("an entry was written")
+	}
+	// Once the fallback kernel is in place the room is not asked for again.
+	b.put(t, "boot/vmlinuz-linux-lts", "k")
+	if out, err := b.run(t); err != nil {
+		t.Fatalf("%v %s", err, out)
+	}
+}
+
+func TestAHookThatCopiesKernelsToOneFixedNameIsMovedIntoTheStateAndNamed(t *testing.T) {
+	b := newBootBed(t)
+	b.systemdBoot(t)
+	b.put(t, "etc/pacman.d/hooks/60-copy-kernel.hook", kernelCopyingHook)
+	b.put(t, "etc/pacman.d/hooks/70-mirrorlist.hook", otherHook)
+	b.put(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", "[Trigger]\nTarget = usr/lib/modules/*/vmlinuz\n[Action]\nExec = /bin/cp x /boot/y\n")
+	out, err := b.run(t)
+	if err != nil || !strings.Contains(out, "taken over: the pacman hook 60-copy-kernel.hook") {
+		t.Fatalf("%v %s", err, out)
+	}
+	if b.read(t, "etc/pacman.d/hooks/60-copy-kernel.hook") != "" {
+		t.Fatalf("the copying hook is still in the hook directory")
+	}
+	if raw, _ := os.ReadFile(filepath.Join(b.state, "taken-over", "60-copy-kernel.hook")); string(raw) != kernelCopyingHook {
+		t.Fatalf("moved hook: %q", raw)
+	}
+	if b.read(t, "etc/pacman.d/hooks/70-mirrorlist.hook") != otherHook || b.read(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook") == "" {
+		t.Fatalf("a hook that copies no kernel was moved")
+	}
+}
+
+func TestTheBootEntriesUnitAndHookAreTheModules(t *testing.T) {
+	m := manifest(t)
+	unit := m.resource(t, "boot-unit").text(t)
+	for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/boot-entries run", "ProtectSystem=strict",
+		"ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks", "NoNewPrivileges=yes", "WantedBy=multi-user.target"} {
+		if !strings.Contains(unit, line+"\n") {
+			t.Errorf("the unit lacks %q", line)
+		}
+	}
+	hook := m.resource(t, "boot-hook")
+	text := hook.text(t)
+	if hook["path"] != "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook" || !strings.Contains(text, "Target = usr/lib/modules/*/vmlinuz\n") ||
+		!strings.Contains(text, "When = PostTransaction\n") || !strings.Contains(text, "Exec = ${dir:state}/boot-entries run\n") {
+		t.Fatalf("the hook: %v\n%s", hook["path"], text)
+	}
+	apply := m.resource(t, "boot-apply")
+	if apply["unit"] != "mesh-kernel-boot-entries.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
+		t.Fatalf("apply: %v", apply)
+	}
+}
+
+// The fallback kernel and its headers are declared after the boot-entries service, so the entry is placed
+// before the kernel is installed and every DKMS module is built for it (ADR 0508 §9).
+func TestTheFallbackKernelAndItsHeadersAreDeclaredAfterTheBootEntriesUnit(t *testing.T) {
+	m := manifest(t)
+	at := map[string]int{}
+	for i, r := range m.Resources {
+		at[r["id"].(string)] = i
+	}
+	for id, pkg := range map[string]string{"linux-lts": "linux-lts", "linux-lts-headers": "linux-lts-headers"} {
+		r := m.resource(t, id)
+		if r["type"] != "package" || r["package"] != pkg || r["absent"] == true {
+			t.Errorf("%s: %v", id, r)
+		}
+		if at[id] < at["boot-apply"] || at[id] < at["boot-hook"] {
+			t.Errorf("%s is declared before the boot-entries unit and hook", id)
+		}
+	}
+}
@@ -1,6 +1,6 @@
 package main

-// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for
+// Over a stand-in tree holding the files of a real machine of 2026-10-14 23:25 (read through node-reads for
 // novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and
 // journald's memory.stat and io.stat, cut to the lines the tool reads.

@@ -26,7 +26,7 @@ func tree(t *testing.T, files map[string]string) *Machine {
 	return &Machine{Root: root}
 }

-func novox(t *testing.T) *Machine {
+func capturedMachine(t *testing.T) *Machine {
 	thp := "sys/kernel/mm/transparent_hugepage/"
 	cg := "sys/fs/cgroup/system.slice/"
 	files := map[string]string{
@@ -53,8 +53,8 @@ func novox(t *testing.T) *Machine {
 	return tree(t, files)
 }

-func TestNovoxBeforeTheChange(t *testing.T) {
-	a, err := novox(t).HugePages(10)
+func TestTheCapturedMachineBeforeTheChange(t *testing.T) {
+	a, err := capturedMachine(t).HugePages(10)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestNovoxBeforeTheChange(t *testing.T) {
 }

 func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) {
-	a, err := novox(t).HugePages(1)
+	a, err := capturedMachine(t).HugePages(1)
 	if err != nil || len(a.Units) != 1 {
 		t.Fatalf("%v %+v", err, a)
 	}
@@ -46,5 +46,15 @@ func tools(m *Machine) []stdio.Tool {
 				return m.HugePages(n)
 			},
 		},
+		{
+			Name: "kernel_boot",
+			Description: "How this machine boots and whether the fallback kernel is on its menu (novox/hq ADR 0508 §9): the loader that booted it " +
+				"(systemd-boot, read from the EFI variable it sets, grub, or none), the menu's entries, the kernels in /boot, the entry " +
+				"the machine booted (systemd-boot) or the kernel image it booted (grub), whether the fallback kernel linux-lts is on " +
+				"the menu with its kernel in /boot, and the room free on /boot in KiB (the fallback kernel needs about 72 MB). " +
+				"Replaces ls /boot, bootctl list, reading grub.cfg and df /boot. (r)",
+			Input: schema(map[string]any{}),
+			Run:   func(args map[string]any) (any, error) { return m.Boot() },
+		},
 	}
 }
@@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
 	for _, r := range m.Resources {
 		ids = append(ids, r["id"].(string))
 	}
-	if strings.Join(ids, " ") != "state huge-pages unit apply" {
+	if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" {
 		t.Fatalf("order: %v", ids)
 	}
 	script := m.resource(t, "huge-pages")
@@ -129,7 +129,7 @@ func (b bed) file(t *testing.T, p string) string {
 }

 func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) {
-	// novox's file on 2026-10-14, read through node-reads.
+	// a real machine's file on 2026-10-14, read through node-reads.
 	b := newBed(t, "madvise", "[always] madvise never\n")
 	out, err := b.run(t, "apply")
 	if err != nil || !strings.Contains(out, "always -> madvise") {
@@ -0,0 +1,13 @@
+# Module kernel, novox/hq ADR 0508 §9, issue 1841. After a kernel image is installed, upgraded or removed, the
+# fallback kernel's boot entry (or the GRUB menu) is placed again. Written by the mesh; change the module.
+[Trigger]
+Operation = Install
+Operation = Upgrade
+Operation = Remove
+Type = Path
+Target = usr/lib/modules/*/vmlinuz
+
+[Action]
+Description = Placing the fallback kernel on the boot menu
+When = PostTransaction
+Exec = ${dir:state}/boot-entries run
@@ -0,0 +1,111 @@
+#!/bin/sh
+# The fallback kernel's place on this machine's boot menu (module kernel, novox/hq ADR 0508 §9 and §10,
+# issue 1841). Written by the mesh and run as root by mesh-kernel-boot-entries.service at boot and when the
+# module is applied, and by the pacman hook 90-mesh-kernel-boot-entries.hook after a kernel is installed,
+# upgraded or removed. Replaced on every push; change the module, never this file.
+#
+#   boot-entries run [<root> [<this module's state> [<room needed in /boot, KiB>]]]
+#
+# The three are arguments only so the module's tests can run this against a stand-in tree.
+#
+# It reads from the machine which loader booted it, never from a setting:
+#   systemd-boot  the EFI variable LoaderEntrySelected is present: the entry mesh-linux-lts.conf is written
+#                 from the command line the machine booted with.
+#   GRUB          /boot/grub/grub.cfg exists and that variable does not: the menu is generated again, only
+#                 when the set of kernels in /boot differs from the menu's.
+#   neither       it fails loudly. The default entry is never touched.
+# And a pacman hook of someone else's that copies kernel images to one fixed name in /boot is moved into this
+# module's state first (ADR 0508 §10): it would put the fallback kernel in the default kernel's place.
+set -eu
+
+root=/
+state='${dir:state}'
+[ $# -ge 2 ] && root=$(printf '%s' "$2" | sed 's|/$||')
+[ $# -ge 3 ] && state=$3
+boot=$root/boot
+hooks=$root/etc/pacman.d/hooks
+efivar=$root/sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f
+ours=90-mesh-kernel-boot-entries.hook
+# The fallback kernel and its initramfs are about 72 MB in /boot (issue 1836): less room fails the unit
+# before a package writes half of them.
+need_kib=98304
+[ $# -ge 4 ] && need_kib=$4
+
+[ $# -ge 1 ] && [ "$1" = run ] || {
+	echo "usage: boot-entries run [<root> [<state directory> [<KiB needed>]]]" >&2
+	exit 2
+}
+
+# A hook that is not ours, triggered by kernel images, whose action copies to one fixed name in /boot.
+copies_kernels() {
+	grep -Eqi '^[[:space:]]*Target[[:space:]]*=.*(vmlinuz|usr/lib/modules)' "$1" &&
+		grep -Eq '^[[:space:]]*Exec[[:space:]]*=.*(cp|install|ln|rsync)[[:space:]].*/boot/[A-Za-z0-9._+-]+([^A-Za-z0-9._+/%$*-]|$)' "$1"
+}
+
+if [ -d "$hooks" ]; then
+	for hook in "$hooks"/*.hook; do
+		[ -f "$hook" ] || continue
+		[ "$(basename "$hook")" = "$ours" ] && continue
+		copies_kernels "$hook" || continue
+		mkdir -p "$state/taken-over"
+		mv -f "$hook" "$state/taken-over/$(basename "$hook")"
+		echo "taken over: the pacman hook $(basename "$hook") copied kernel images to one fixed name in /boot; moved to $state/taken-over/ so each kernel keeps its own package's name"
+	done
+fi
+
+if [ -e "$efivar" ]; then
+	loader=systemd-boot
+elif [ -f "$boot/grub/grub.cfg" ]; then
+	loader=grub
+else
+	echo "no boot loader found: the machine was not booted by systemd-boot (no LoaderEntrySelected variable) and there is no $boot/grub/grub.cfg; the fallback kernel has no menu to be placed on" >&2
+	exit 1
+fi
+
+# Room first, only while the fallback kernel is not in /boot yet.
+if [ ! -e "$boot/vmlinuz-linux-lts" ]; then
+	free=$(df -Pk "$boot" | awk 'NR==2 {print $4}')
+	[ -n "$free" ] || free=0
+	if [ "$free" -lt "$need_kib" ]; then
+		echo "$boot has $free KiB free and the fallback kernel needs $need_kib KiB: free room there before the kernel is installed" >&2
+		exit 1
+	fi
+fi
+
+case "$loader" in
+systemd-boot)
+	entries=$boot/loader/entries
+	cmdline=$(sed -e 's/BOOT_IMAGE=[^ ]*//g' -e 's/initrd=[^ ]*//g' "$root/proc/cmdline" | tr -s ' \n' ' ' | sed -e 's/^ //' -e 's/ $//')
+	[ -n "$cmdline" ] || {
+		echo "$root/proc/cmdline holds no command line to boot the fallback kernel with" >&2
+		exit 1
+	}
+	want=$(
+		echo "title Arch Linux (linux-lts, fallback)"
+		echo "linux /vmlinuz-linux-lts"
+		for ucode in intel-ucode.img amd-ucode.img; do
+			[ -e "$boot/$ucode" ] && echo "initrd /$ucode"
+		done
+		echo "initrd /initramfs-linux-lts.img"
+		echo "options $cmdline"
+	)
+	mkdir -p "$entries"
+	if [ "$(cat "$entries/mesh-linux-lts.conf" 2>/dev/null || true)" = "$want" ]; then
+		echo "systemd-boot: mesh-linux-lts.conf, as it was"
+	else
+		printf '%s\n' "$want" >"$entries/mesh-linux-lts.conf"
+		echo "systemd-boot: wrote $entries/mesh-linux-lts.conf from the command line the machine booted with; the default entry is as it was"
+	fi
+	;;
+grub)
+	cfg=$boot/grub/grub.cfg
+	menu=$(sed -n 's|^[[:space:]]*linux[[:space:]]*\(/boot\)\{0,1\}/\(vmlinuz-[^[:space:]]*\).*|\2|p' "$cfg" | sort -u)
+	have=$(cd "$boot" && ls -d vmlinuz-* 2>/dev/null | sort -u || true)
+	if [ "$menu" = "$have" ]; then
+		echo "grub: the menu lists the kernels in /boot, left as it is"
+	else
+		grub-mkconfig -o "$cfg"
+		echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again; the default entry is as it was"
+	fi
+	;;
+esac
@@ -0,0 +1,22 @@
+[Unit]
+Description=The fallback kernel's place on this machine's boot menu
+# Module kernel, novox/hq ADR 0508 §9, issue 1841. Starting this unit reads which loader booted the machine and
+# places the fallback kernel's boot entry (systemd-boot) or generates the menu again (GRUB); it fails, and says
+# why, when there is no loader or the boot partition has no room. The kernel packages are declared after this
+# unit, so the entry is in place before they are installed.
+After=local-fs.target
+Wants=local-fs.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=${dir:state}/boot-entries run
+# It writes the boot partition, the pacman hook directory and its own state, and nothing else.
+ProtectSystem=strict
+ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks
+ProtectHome=yes
+PrivateNetwork=yes
+NoNewPrivileges=yes
+
+[Install]
+WantedBy=multi-user.target
@@ -13,7 +13,8 @@
    }
  },
  "tools": [
-    "kernel_huge_pages"
+    "kernel_huge_pages",
+    "kernel_boot"
  ],
  "resources": [
    {
@@ -53,6 +54,52 @@
      "health": {
        "kind": "unit"
      }
+    },
+    {
+      "id": "boot-entries",
+      "type": "file",
+      "path": "${dir:state}/boot-entries",
+      "mode": "0755",
+      "trusted": true,
+      "template": "files/boot-entries.sh"
+    },
+    {
+      "id": "boot-unit",
+      "type": "file",
+      "path": "/etc/systemd/system/mesh-kernel-boot-entries.service",
+      "mode": "0644",
+      "template": "files/mesh-kernel-boot-entries.service"
+    },
+    {
+      "id": "boot-hook",
+      "type": "file",
+      "path": "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook",
+      "mode": "0644",
+      "template": "files/90-mesh-kernel-boot-entries.hook"
+    },
+    {
+      "id": "boot-apply",
+      "type": "service",
+      "unit": "mesh-kernel-boot-entries.service",
+      "state": "running",
+      "boot": "enabled",
+      "restart-on": [
+        "boot-unit",
+        "boot-entries"
+      ],
+      "health": {
+        "kind": "unit"
+      }
+    },
+    {
+      "id": "linux-lts",
+      "type": "package",
+      "package": "linux-lts"
+    },
+    {
+      "id": "linux-lts-headers",
+      "type": "package",
+      "package": "linux-lts-headers"
    }
  ],
  "build": {