diff --git a/modules/kernel/README.md b/modules/kernel/README.md index 6acb6a37..2f04aac7 100644 --- a/modules/kernel/README.md +++ b/modules/kernel/README.md @@ -1,8 +1,9 @@ # kernel -The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one: -**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot -entries) is not built yet. +The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). It holds two things: +**transparent huge pages**, set per machine, and **the fallback kernel** (`linux-lts` with its headers) and +its place on the boot menu (novox/hq ADR 0508 §9 and §10). The default kernel's package, and microcode, are +not built yet. ## Why (novox/hq issue 746) @@ -35,16 +36,34 @@ under the new mode. A failed rotation is said in the unit's journal and does not rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing` or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit. +## The fallback kernel (novox/hq ADR 0508, issue 1841) + +Every machine keeps a second, older kernel to start from, so a kernel upgrade that does not boot is not the +end of the machine. The module declares the packages `linux-lts` and `linux-lts-headers` (so every DKMS module is +built for the fallback kernel too) after its boot-entries service, so the entry is in place before the kernel is. + +| | what | notes | +|---|---|---| +| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly | +| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched | +| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names; the default is left as it is | +| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) | +| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks | +| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network | +| file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed | +| packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook | + ## Tools | tool | | answers | |---|---|---| +| `kernel_boot` | r | the loader (`systemd-boot`, `grub` or `none`), the menu's entries, the kernels in `/boot`, the entry booted (systemd-boot) or the kernel image booted (GRUB), whether the fallback kernel is on the menu with its kernel in `/boot`, and the room free on `/boot` in KiB | | `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) | ## Tests `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the -manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel diff --git a/modules/kernel/cmd/kernel-tools/boot.go b/modules/kernel/cmd/kernel-tools/boot.go new file mode 100644 index 00000000..10a895a1 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/boot.go @@ -0,0 +1,89 @@ +package main + +// kernel_boot: which loader booted the machine, its entries, the kernels in /boot, the entry it booted and +// whether the fallback kernel is on the menu (novox/hq ADR 0508 §9, issue 1841). Reads only. + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "syscall" + "unicode/utf16" +) + +const loaderSelected = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + +// BootState is what kernel_boot answers. +type BootState struct { + Loader string `json:"loader"` + Entries []string `json:"entries"` + Kernels []string `json:"kernels"` + Booted string `json:"booted"` + FallbackPresent bool `json:"fallback_present"` + BootFreeKiB int64 `json:"boot_free_kib"` +} + +var grubKernel = regexp.MustCompile(`(?m)^\s*linux\s+(?:/boot)?/(vmlinuz-\S+)`) +var grubTitle = regexp.MustCompile(`(?m)^\s*menuentry\s+'([^']*)'`) +var bootImage = regexp.MustCompile(`BOOT_IMAGE=(?:/boot)?/?(\S+)`) + +// Boot reads the machine, the loader the way the boot-entries script does: the loader's EFI variable means +// systemd-boot, a grub.cfg without it GRUB. +func (m *Machine) Boot() (*BootState, error) { + b := &BootState{Loader: "none", Entries: []string{}, Kernels: []string{}} + files, _ := filepath.Glob(m.path("boot/vmlinuz-*")) + for _, f := range files { + b.Kernels = append(b.Kernels, filepath.Base(f)) + } + sort.Strings(b.Kernels) + hasKernel := func(name string) bool { + for _, k := range b.Kernels { + if k == name { + return true + } + } + return false + } + var s syscall.Statfs_t + if err := syscall.Statfs(m.path("boot"), &s); err == nil { + b.BootFreeKiB = int64(s.Bavail) * int64(s.Bsize) / 1024 + } + if raw, err := os.ReadFile(m.path(loaderSelected)); err == nil { + b.Loader = "systemd-boot" + if len(raw) > 4 { + var u []uint16 + for i := 4; i+1 < len(raw); i += 2 { + u = append(u, uint16(raw[i])|uint16(raw[i+1])<<8) + } + b.Booted = strings.TrimRight(string(utf16.Decode(u)), "\x00") + } + files, _ := filepath.Glob(m.path("boot/loader/entries/*.conf")) + for _, f := range files { + b.Entries = append(b.Entries, filepath.Base(f)) + } + sort.Strings(b.Entries) + if raw, err := os.ReadFile(m.path("boot/loader/entries/mesh-linux-lts.conf")); err == nil { + b.FallbackPresent = strings.Contains(string(raw), "/vmlinuz-linux-lts") && hasKernel("vmlinuz-linux-lts") + } + return b, nil + } + if raw, err := os.ReadFile(m.path("boot/grub/grub.cfg")); err == nil { + b.Loader = "grub" + for _, t := range grubTitle.FindAllStringSubmatch(string(raw), -1) { + b.Entries = append(b.Entries, t[1]) + } + for _, k := range grubKernel.FindAllStringSubmatch(string(raw), -1) { + if k[1] == "vmlinuz-linux-lts" && hasKernel("vmlinuz-linux-lts") { + b.FallbackPresent = true + } + } + if cmd, err := m.read("proc/cmdline"); err == nil { + if i := bootImage.FindStringSubmatch(cmd); i != nil { + b.Booted = i[1] + } + } + } + return b, nil +} diff --git a/modules/kernel/cmd/kernel-tools/boot_test.go b/modules/kernel/cmd/kernel-tools/boot_test.go new file mode 100644 index 00000000..9c8d964f --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/boot_test.go @@ -0,0 +1,90 @@ +package main + +// kernel_boot over stand-in roots: a systemd-boot machine with and without the fallback entry, a GRUB machine +// with and without the fallback kernel in its menu, and a machine with neither (novox/hq ADR 0508 §9, issue 1841). + +import ( + "os" + "path/filepath" + "testing" +) + +const selectedVar = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + +func put(t *testing.T, root, p, content string) { + t.Helper() + full := filepath.Join(root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + write(t, full, content, 0o644) +} + +func bootOf(t *testing.T, root string) *BootState { + t.Helper() + got, err := (&Machine{Root: root}).Boot() + if err != nil { + t.Fatal(err) + } + return got +} + +func TestKernelBootReadsASystemdBootMachineWithItsFallbackEntry(t *testing.T) { + root := t.TempDir() + put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + put(t, root, "boot/loader/entries/arch.conf", defaultEntry) + put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\n") + put(t, root, "boot/vmlinuz-linux", "k") + put(t, root, "boot/vmlinuz-linux-lts", "k") + got := bootOf(t, root) + if got.Loader != "systemd-boot" || got.Booted != "arch.conf" || !got.FallbackPresent { + t.Fatalf("%+v", got) + } + if len(got.Entries) != 2 || got.Entries[0] != "arch.conf" || got.Entries[1] != "mesh-linux-lts.conf" { + t.Fatalf("entries: %v", got.Entries) + } + if len(got.Kernels) != 2 || got.Kernels[0] != "vmlinuz-linux" || got.Kernels[1] != "vmlinuz-linux-lts" { + t.Fatalf("kernels: %v", got.Kernels) + } + if got.BootFreeKiB <= 0 { + t.Fatalf("free: %d", got.BootFreeKiB) + } +} + +func TestKernelBootSaysASystemdBootMachineHasNoFallbackYet(t *testing.T) { + root := t.TempDir() + put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + put(t, root, "boot/loader/entries/arch.conf", defaultEntry) + put(t, root, "boot/vmlinuz-linux", "k") + // The entry file alone is not the fallback: the kernel it names must be in /boot. + put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "linux /vmlinuz-linux-lts\n") + if got := bootOf(t, root); got.FallbackPresent { + t.Fatalf("a fallback entry without its kernel: %+v", got) + } +} + +func TestKernelBootReadsAGrubMachine(t *testing.T) { + root := t.TempDir() + put(t, root, "boot/grub/grub.cfg", grubMenu) + put(t, root, "boot/vmlinuz-linux", "k") + put(t, root, "proc/cmdline", bootedCmdline) + got := bootOf(t, root) + if got.Loader != "grub" || got.FallbackPresent || got.Booted != "vmlinuz-linux" { + t.Fatalf("%+v", got) + } + if len(got.Entries) != 2 || got.Entries[0] != "Arch Linux" { + t.Fatalf("entries: %v", got.Entries) + } + put(t, root, "boot/vmlinuz-linux-lts", "k") + put(t, root, "boot/grub/grub.cfg", grubMenu+"menuentry 'Arch Linux LTS' {\n\tlinux\t/boot/vmlinuz-linux-lts root=x rw\n}\n") + if got := bootOf(t, root); !got.FallbackPresent { + t.Fatalf("%+v", got) + } +} + +func TestKernelBootSaysWhenThereIsNoLoader(t *testing.T) { + got := bootOf(t, t.TempDir()) + if got.Loader != "none" || got.FallbackPresent || len(got.Entries) != 0 { + t.Fatalf("%+v", got) + } +} diff --git a/modules/kernel/cmd/kernel-tools/bootentries_test.go b/modules/kernel/cmd/kernel-tools/bootentries_test.go new file mode 100644 index 00000000..f6df9c78 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/bootentries_test.go @@ -0,0 +1,257 @@ +package main + +// The boot-entries script as the mesh writes it, run with sh over a stand-in root (novox/hq ADR 0508 §9 and +// §10, issue 1841). The fixtures are the shapes read from machines: a systemd-boot entries directory, a GRUB +// menu, a hook that copies every kernel image to the default kernel's name, and the command line a machine +// booted with. + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const bootedCmdline = "initrd=\\initramfs-linux.img BOOT_IMAGE=/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3 initrd=\\intel-ucode.img\n" + +const defaultEntry = "title Arch Linux\nlinux /vmlinuz-linux\ninitrd /intel-ucode.img\ninitrd /initramfs-linux.img\noptions root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet\n" + +const loaderConf = "default arch.conf\ntimeout 3\nconsole-mode max\n" + +// grubMenu is the part of a generated grub.cfg that names kernels, for a machine holding one kernel. +const grubMenu = `menuentry 'Arch Linux' --class arch { + set root='hd0,gpt2' + linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet + initrd /boot/initramfs-linux.img +} +menuentry 'Arch Linux, with fallback initramfs' --class arch { + linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet + initrd /boot/initramfs-linux-fallback.img +} +` + +// kernelCopyingHook copies every installed kernel image to the default kernel's name on the boot partition. +const kernelCopyingHook = `[Trigger] +Operation = Install +Operation = Upgrade +Type = Path +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Copying the kernel to the default name +When = PostTransaction +Exec = /usr/bin/sh -c '/usr/bin/cp /usr/lib/modules/$(uname -r)/vmlinuz /boot/vmlinuz-linux' +` + +// otherHook is a hook of the distribution's shape that has nothing to do with kernel images. +const otherHook = `[Trigger] +Operation = Upgrade +Type = Package +Target = pacman-mirrorlist + +[Action] +Description = Updating the mirror list +When = PostTransaction +Exec = /usr/bin/cp /etc/pacman.d/mirrorlist.pacnew /etc/pacman.d/mirrorlist +` + +type bootBed struct { + script, root, state, bin, mkconfig string + need string +} + +func newBootBed(t *testing.T) bootBed { + t.Helper() + dir := t.TempDir() + b := bootBed{script: filepath.Join(dir, "boot-entries"), root: filepath.Join(dir, "root"), state: filepath.Join(dir, "state"), + bin: filepath.Join(dir, "bin"), mkconfig: filepath.Join(dir, "mkconfig")} + content := manifest(t).resource(t, "boot-entries").text(t) + content = strings.ReplaceAll(content, "${dir:state}", b.state) + if strings.Contains(content, "${") { + t.Fatalf("a substitution the test does not make is left in the script") + } + for _, d := range []string{b.root + "/boot", b.root + "/proc", b.root + "/etc/pacman.d/hooks", b.state, b.bin} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write(t, b.script, content, 0o755) + write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644) + write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"$@\" >>"+b.mkconfig+"\n", 0o755) + return b +} + +func (b bootBed) put(t *testing.T, p, content string) { + t.Helper() + full := filepath.Join(b.root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + write(t, full, content, 0o644) +} + +func (b bootBed) systemdBoot(t *testing.T) { + b.put(t, "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f", "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + b.put(t, "boot/loader/loader.conf", loaderConf) + b.put(t, "boot/loader/entries/arch.conf", defaultEntry) + b.put(t, "boot/vmlinuz-linux", "k") +} + +func (b bootBed) run(t *testing.T) (string, error) { + t.Helper() + args := []string{b.script, "run", b.root, b.state} + if b.need != "" { + args = append(args, b.need) + } + cmd := exec.Command("sh", args...) + cmd.Env = append(os.Environ(), "PATH="+b.bin+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (b bootBed) read(t *testing.T, p string) string { + t.Helper() + raw, _ := os.ReadFile(filepath.Join(b.root, p)) + return string(raw) +} + +func TestASystemdBootMachineGetsTheFallbackEntryFromItsBootedCommandLine(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.put(t, "boot/intel-ucode.img", "u") + out, err := b.run(t) + if err != nil || !strings.Contains(out, "wrote") { + t.Fatalf("%v %s", err, out) + } + want := "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\ninitrd /intel-ucode.img\ninitrd /initramfs-linux-lts.img\n" + + "options root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3\n" + if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); got != want { + t.Fatalf("entry:\n%s\nwant:\n%s", got, want) + } + // The default entry and the loader's own file are as they were. + if b.read(t, "boot/loader/entries/arch.conf") != defaultEntry || b.read(t, "boot/loader/loader.conf") != loaderConf { + t.Fatalf("the default entry or loader.conf was touched") + } + if out, err := b.run(t); err != nil || !strings.Contains(out, "as it was") { + t.Fatalf("again: %v %s", err, out) + } +} + +func TestWithoutMicrocodeThereIsNoMicrocodeLine(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + if out, err := b.run(t); err != nil { + t.Fatalf("%v %s", err, out) + } + if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); strings.Contains(got, "ucode") || !strings.Contains(got, "initrd /initramfs-linux-lts.img") { + t.Fatalf("entry: %s", got) + } +} + +func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) { + b := newBootBed(t) + b.put(t, "boot/grub/grub.cfg", grubMenu) + b.put(t, "boot/vmlinuz-linux", "k") + if out, err := b.run(t); err != nil || !strings.Contains(out, "left as it is") { + t.Fatalf("same: %v %s", err, out) + } + if _, err := os.Stat(b.mkconfig); err == nil { + t.Fatalf("the menu was generated again with the same kernels") + } + b.put(t, "boot/vmlinuz-linux-lts", "k") + if out, err := b.run(t); err != nil || !strings.Contains(out, "generated again") { + t.Fatalf("changed: %v %s", err, out) + } + raw, _ := os.ReadFile(b.mkconfig) + if string(raw) != "-o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" { + t.Fatalf("grub-mkconfig was asked %q", raw) + } +} + +func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) { + b := newBootBed(t) + out, err := b.run(t) + if err == nil || !strings.Contains(out, "no boot loader found") { + t.Fatalf("%v %s", err, out) + } +} + +func TestABootPartitionWithoutRoomFailsBeforeAnythingIsWritten(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.need = "999999999999" + out, err := b.run(t) + if err == nil || !strings.Contains(out, "free room there before the kernel is installed") { + t.Fatalf("%v %s", err, out) + } + if b.read(t, "boot/loader/entries/mesh-linux-lts.conf") != "" { + t.Fatalf("an entry was written") + } + // Once the fallback kernel is in place the room is not asked for again. + b.put(t, "boot/vmlinuz-linux-lts", "k") + if out, err := b.run(t); err != nil { + t.Fatalf("%v %s", err, out) + } +} + +func TestAHookThatCopiesKernelsToOneFixedNameIsMovedIntoTheStateAndNamed(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.put(t, "etc/pacman.d/hooks/60-copy-kernel.hook", kernelCopyingHook) + b.put(t, "etc/pacman.d/hooks/70-mirrorlist.hook", otherHook) + b.put(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", "[Trigger]\nTarget = usr/lib/modules/*/vmlinuz\n[Action]\nExec = /bin/cp x /boot/y\n") + out, err := b.run(t) + if err != nil || !strings.Contains(out, "taken over: the pacman hook 60-copy-kernel.hook") { + t.Fatalf("%v %s", err, out) + } + if b.read(t, "etc/pacman.d/hooks/60-copy-kernel.hook") != "" { + t.Fatalf("the copying hook is still in the hook directory") + } + if raw, _ := os.ReadFile(filepath.Join(b.state, "taken-over", "60-copy-kernel.hook")); string(raw) != kernelCopyingHook { + t.Fatalf("moved hook: %q", raw) + } + if b.read(t, "etc/pacman.d/hooks/70-mirrorlist.hook") != otherHook || b.read(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook") == "" { + t.Fatalf("a hook that copies no kernel was moved") + } +} + +func TestTheBootEntriesUnitAndHookAreTheModules(t *testing.T) { + m := manifest(t) + unit := m.resource(t, "boot-unit").text(t) + for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/boot-entries run", "ProtectSystem=strict", + "ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks", "NoNewPrivileges=yes", "WantedBy=multi-user.target"} { + if !strings.Contains(unit, line+"\n") { + t.Errorf("the unit lacks %q", line) + } + } + hook := m.resource(t, "boot-hook") + text := hook.text(t) + if hook["path"] != "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook" || !strings.Contains(text, "Target = usr/lib/modules/*/vmlinuz\n") || + !strings.Contains(text, "When = PostTransaction\n") || !strings.Contains(text, "Exec = ${dir:state}/boot-entries run\n") { + t.Fatalf("the hook: %v\n%s", hook["path"], text) + } + apply := m.resource(t, "boot-apply") + if apply["unit"] != "mesh-kernel-boot-entries.service" || apply["state"] != "running" || apply["boot"] != "enabled" { + t.Fatalf("apply: %v", apply) + } +} + +// The fallback kernel and its headers are declared after the boot-entries service, so the entry is placed +// before the kernel is installed and every DKMS module is built for it (ADR 0508 §9). +func TestTheFallbackKernelAndItsHeadersAreDeclaredAfterTheBootEntriesUnit(t *testing.T) { + m := manifest(t) + at := map[string]int{} + for i, r := range m.Resources { + at[r["id"].(string)] = i + } + for id, pkg := range map[string]string{"linux-lts": "linux-lts", "linux-lts-headers": "linux-lts-headers"} { + r := m.resource(t, id) + if r["type"] != "package" || r["package"] != pkg || r["absent"] == true { + t.Errorf("%s: %v", id, r) + } + if at[id] < at["boot-apply"] || at[id] < at["boot-hook"] { + t.Errorf("%s is declared before the boot-entries unit and hook", id) + } + } +} diff --git a/modules/kernel/cmd/kernel-tools/hugepages_test.go b/modules/kernel/cmd/kernel-tools/hugepages_test.go index 2d6359c7..4467b628 100644 --- a/modules/kernel/cmd/kernel-tools/hugepages_test.go +++ b/modules/kernel/cmd/kernel-tools/hugepages_test.go @@ -1,6 +1,6 @@ package main -// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for +// Over a stand-in tree holding the files of a real machine of 2026-10-14 23:25 (read through node-reads for // novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and // journald's memory.stat and io.stat, cut to the lines the tool reads. @@ -26,7 +26,7 @@ func tree(t *testing.T, files map[string]string) *Machine { return &Machine{Root: root} } -func novox(t *testing.T) *Machine { +func capturedMachine(t *testing.T) *Machine { thp := "sys/kernel/mm/transparent_hugepage/" cg := "sys/fs/cgroup/system.slice/" files := map[string]string{ @@ -53,8 +53,8 @@ func novox(t *testing.T) *Machine { return tree(t, files) } -func TestNovoxBeforeTheChange(t *testing.T) { - a, err := novox(t).HugePages(10) +func TestTheCapturedMachineBeforeTheChange(t *testing.T) { + a, err := capturedMachine(t).HugePages(10) if err != nil { t.Fatal(err) } @@ -86,7 +86,7 @@ func TestNovoxBeforeTheChange(t *testing.T) { } func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) { - a, err := novox(t).HugePages(1) + a, err := capturedMachine(t).HugePages(1) if err != nil || len(a.Units) != 1 { t.Fatalf("%v %+v", err, a) } diff --git a/modules/kernel/cmd/kernel-tools/main.go b/modules/kernel/cmd/kernel-tools/main.go index ec15c916..93cdece8 100644 --- a/modules/kernel/cmd/kernel-tools/main.go +++ b/modules/kernel/cmd/kernel-tools/main.go @@ -46,5 +46,15 @@ func tools(m *Machine) []stdio.Tool { return m.HugePages(n) }, }, + { + Name: "kernel_boot", + Description: "How this machine boots and whether the fallback kernel is on its menu (novox/hq ADR 0508 §9): the loader that booted it " + + "(systemd-boot, read from the EFI variable it sets, grub, or none), the menu's entries, the kernels in /boot, the entry " + + "the machine booted (systemd-boot) or the kernel image it booted (grub), whether the fallback kernel linux-lts is on " + + "the menu with its kernel in /boot, and the room free on /boot in KiB (the fallback kernel needs about 72 MB). " + + "Replaces ls /boot, bootctl list, reading grub.cfg and df /boot. (r)", + Input: schema(map[string]any{}), + Run: func(args map[string]any) (any, error) { return m.Boot() }, + }, } } diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index b68dacf5..aad91fc6 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { for _, r := range m.Resources { ids = append(ids, r["id"].(string)) } - if strings.Join(ids, " ") != "state huge-pages unit apply" { + if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" { t.Fatalf("order: %v", ids) } script := m.resource(t, "huge-pages") @@ -129,7 +129,7 @@ func (b bed) file(t *testing.T, p string) string { } func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) { - // novox's file on 2026-10-14, read through node-reads. + // a real machine's file on 2026-10-14, read through node-reads. b := newBed(t, "madvise", "[always] madvise never\n") out, err := b.run(t, "apply") if err != nil || !strings.Contains(out, "always -> madvise") { diff --git a/modules/kernel/files/90-mesh-kernel-boot-entries.hook b/modules/kernel/files/90-mesh-kernel-boot-entries.hook new file mode 100644 index 00000000..d671caaf --- /dev/null +++ b/modules/kernel/files/90-mesh-kernel-boot-entries.hook @@ -0,0 +1,13 @@ +# Module kernel, novox/hq ADR 0508 §9, issue 1841. After a kernel image is installed, upgraded or removed, the +# fallback kernel's boot entry (or the GRUB menu) is placed again. Written by the mesh; change the module. +[Trigger] +Operation = Install +Operation = Upgrade +Operation = Remove +Type = Path +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Placing the fallback kernel on the boot menu +When = PostTransaction +Exec = ${dir:state}/boot-entries run diff --git a/modules/kernel/files/boot-entries.sh b/modules/kernel/files/boot-entries.sh new file mode 100644 index 00000000..9bcf957a --- /dev/null +++ b/modules/kernel/files/boot-entries.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# The fallback kernel's place on this machine's boot menu (module kernel, novox/hq ADR 0508 §9 and §10, +# issue 1841). Written by the mesh and run as root by mesh-kernel-boot-entries.service at boot and when the +# module is applied, and by the pacman hook 90-mesh-kernel-boot-entries.hook after a kernel is installed, +# upgraded or removed. Replaced on every push; change the module, never this file. +# +# boot-entries run [ [ []]] +# +# The three are arguments only so the module's tests can run this against a stand-in tree. +# +# It reads from the machine which loader booted it, never from a setting: +# systemd-boot the EFI variable LoaderEntrySelected is present: the entry mesh-linux-lts.conf is written +# from the command line the machine booted with. +# GRUB /boot/grub/grub.cfg exists and that variable does not: the menu is generated again, only +# when the set of kernels in /boot differs from the menu's. +# neither it fails loudly. The default entry is never touched. +# And a pacman hook of someone else's that copies kernel images to one fixed name in /boot is moved into this +# module's state first (ADR 0508 §10): it would put the fallback kernel in the default kernel's place. +set -eu + +root=/ +state='${dir:state}' +[ $# -ge 2 ] && root=$(printf '%s' "$2" | sed 's|/$||') +[ $# -ge 3 ] && state=$3 +boot=$root/boot +hooks=$root/etc/pacman.d/hooks +efivar=$root/sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f +ours=90-mesh-kernel-boot-entries.hook +# The fallback kernel and its initramfs are about 72 MB in /boot (issue 1836): less room fails the unit +# before a package writes half of them. +need_kib=98304 +[ $# -ge 4 ] && need_kib=$4 + +[ $# -ge 1 ] && [ "$1" = run ] || { + echo "usage: boot-entries run [ [ []]]" >&2 + exit 2 +} + +# A hook that is not ours, triggered by kernel images, whose action copies to one fixed name in /boot. +copies_kernels() { + grep -Eqi '^[[:space:]]*Target[[:space:]]*=.*(vmlinuz|usr/lib/modules)' "$1" && + grep -Eq '^[[:space:]]*Exec[[:space:]]*=.*(cp|install|ln|rsync)[[:space:]].*/boot/[A-Za-z0-9._+-]+([^A-Za-z0-9._+/%$*-]|$)' "$1" +} + +if [ -d "$hooks" ]; then + for hook in "$hooks"/*.hook; do + [ -f "$hook" ] || continue + [ "$(basename "$hook")" = "$ours" ] && continue + copies_kernels "$hook" || continue + mkdir -p "$state/taken-over" + mv -f "$hook" "$state/taken-over/$(basename "$hook")" + echo "taken over: the pacman hook $(basename "$hook") copied kernel images to one fixed name in /boot; moved to $state/taken-over/ so each kernel keeps its own package's name" + done +fi + +if [ -e "$efivar" ]; then + loader=systemd-boot +elif [ -f "$boot/grub/grub.cfg" ]; then + loader=grub +else + echo "no boot loader found: the machine was not booted by systemd-boot (no LoaderEntrySelected variable) and there is no $boot/grub/grub.cfg; the fallback kernel has no menu to be placed on" >&2 + exit 1 +fi + +# Room first, only while the fallback kernel is not in /boot yet. +if [ ! -e "$boot/vmlinuz-linux-lts" ]; then + free=$(df -Pk "$boot" | awk 'NR==2 {print $4}') + [ -n "$free" ] || free=0 + if [ "$free" -lt "$need_kib" ]; then + echo "$boot has $free KiB free and the fallback kernel needs $need_kib KiB: free room there before the kernel is installed" >&2 + exit 1 + fi +fi + +case "$loader" in +systemd-boot) + entries=$boot/loader/entries + cmdline=$(sed -e 's/BOOT_IMAGE=[^ ]*//g' -e 's/initrd=[^ ]*//g' "$root/proc/cmdline" | tr -s ' \n' ' ' | sed -e 's/^ //' -e 's/ $//') + [ -n "$cmdline" ] || { + echo "$root/proc/cmdline holds no command line to boot the fallback kernel with" >&2 + exit 1 + } + want=$( + echo "title Arch Linux (linux-lts, fallback)" + echo "linux /vmlinuz-linux-lts" + for ucode in intel-ucode.img amd-ucode.img; do + [ -e "$boot/$ucode" ] && echo "initrd /$ucode" + done + echo "initrd /initramfs-linux-lts.img" + echo "options $cmdline" + ) + mkdir -p "$entries" + if [ "$(cat "$entries/mesh-linux-lts.conf" 2>/dev/null || true)" = "$want" ]; then + echo "systemd-boot: mesh-linux-lts.conf, as it was" + else + printf '%s\n' "$want" >"$entries/mesh-linux-lts.conf" + echo "systemd-boot: wrote $entries/mesh-linux-lts.conf from the command line the machine booted with; the default entry is as it was" + fi + ;; +grub) + cfg=$boot/grub/grub.cfg + menu=$(sed -n 's|^[[:space:]]*linux[[:space:]]*\(/boot\)\{0,1\}/\(vmlinuz-[^[:space:]]*\).*|\2|p' "$cfg" | sort -u) + have=$(cd "$boot" && ls -d vmlinuz-* 2>/dev/null | sort -u || true) + if [ "$menu" = "$have" ]; then + echo "grub: the menu lists the kernels in /boot, left as it is" + else + grub-mkconfig -o "$cfg" + echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again; the default entry is as it was" + fi + ;; +esac diff --git a/modules/kernel/files/mesh-kernel-boot-entries.service b/modules/kernel/files/mesh-kernel-boot-entries.service new file mode 100644 index 00000000..56cfa319 --- /dev/null +++ b/modules/kernel/files/mesh-kernel-boot-entries.service @@ -0,0 +1,22 @@ +[Unit] +Description=The fallback kernel's place on this machine's boot menu +# Module kernel, novox/hq ADR 0508 §9, issue 1841. Starting this unit reads which loader booted the machine and +# places the fallback kernel's boot entry (systemd-boot) or generates the menu again (GRUB); it fails, and says +# why, when there is no loader or the boot partition has no room. The kernel packages are declared after this +# unit, so the entry is in place before they are installed. +After=local-fs.target +Wants=local-fs.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=${dir:state}/boot-entries run +# It writes the boot partition, the pacman hook directory and its own state, and nothing else. +ProtectSystem=strict +ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks +ProtectHome=yes +PrivateNetwork=yes +NoNewPrivileges=yes + +[Install] +WantedBy=multi-user.target diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 67d32156..2a63563e 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -13,7 +13,8 @@ } }, "tools": [ - "kernel_huge_pages" + "kernel_huge_pages", + "kernel_boot" ], "resources": [ { @@ -53,6 +54,52 @@ "health": { "kind": "unit" } + }, + { + "id": "boot-entries", + "type": "file", + "path": "${dir:state}/boot-entries", + "mode": "0755", + "trusted": true, + "template": "files/boot-entries.sh" + }, + { + "id": "boot-unit", + "type": "file", + "path": "/etc/systemd/system/mesh-kernel-boot-entries.service", + "mode": "0644", + "template": "files/mesh-kernel-boot-entries.service" + }, + { + "id": "boot-hook", + "type": "file", + "path": "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", + "mode": "0644", + "template": "files/90-mesh-kernel-boot-entries.hook" + }, + { + "id": "boot-apply", + "type": "service", + "unit": "mesh-kernel-boot-entries.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "boot-unit", + "boot-entries" + ], + "health": { + "kind": "unit" + } + }, + { + "id": "linux-lts", + "type": "package", + "package": "linux-lts" + }, + { + "id": "linux-lts-headers", + "type": "package", + "package": "linux-lts-headers" } ], "build": { diff --git a/modules/pacman/cmd/pacman-tools/bus_fake_test.go b/modules/pacman/cmd/pacman-tools/bus_fake_test.go new file mode 100644 index 00000000..0d90a846 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/bus_fake_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// memBucket is a state in memory, with the bus's revisions and its refusal of a lost race. +type memBucket struct { + mu sync.Mutex + values map[string][]byte + revs map[string]uint64 + rev uint64 + writes []string // "op key", in order + fail map[string]error +} + +func newBucket() *memBucket { + return &memBucket{values: map[string][]byte{}, revs: map[string]uint64{}, fail: map[string]error{}} +} + +func (b *memBucket) Get(key string) (*stdio.StateEntry, error) { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.fail["get "+key]; err != nil { + return nil, err + } + v, ok := b.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: b.revs[key]}, nil +} + +func (b *memBucket) write(op, key string, value any) (uint64, error) { + if err := b.fail[op+" "+key]; err != nil { + return 0, err + } + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + b.rev++ + b.values[key], b.revs[key] = data, b.rev + b.writes = append(b.writes, op+" "+key) + return b.rev, nil +} + +func (b *memBucket) Create(key string, value any) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if _, ok := b.values[key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return b.write("create", key, value) +} + +func (b *memBucket) Update(key string, value any, revision uint64) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.revs[key] != revision { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return b.write("update", key, value) +} + +func (b *memBucket) Put(key string, value any) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.write("put", key, value) +} + +func (b *memBucket) Delete(key string) error { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.fail["delete "+key]; err != nil { + return err + } + delete(b.values, key) + delete(b.revs, key) + b.writes = append(b.writes, "delete "+key) + return nil +} + +func (b *memBucket) Keys() ([]string, error) { + b.mu.Lock() + defer b.mu.Unlock() + keys := []string{} + for k := range b.values { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} + +// value reads a key into its shape; a missing key is the zero value and false. +func (b *memBucket) value(key string, v any) bool { + b.mu.Lock() + defer b.mu.Unlock() + raw, ok := b.values[key] + if ok { + if err := json.Unmarshal(raw, v); err != nil { + panic(err) + } + } + return ok +} + +// writesTo are the writes to keys starting with prefix. +func (b *memBucket) writesTo(prefix string) []string { + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for _, w := range b.writes { + if _, key, _ := strings.Cut(w, " "); strings.HasPrefix(key, prefix) { + out = append(out, w) + } + } + return out +} + +// sentEvent is one event a fake sink was handed. +type sentEvent struct { + Name string + Body any +} + +type eventLog struct { + mu sync.Mutex + sent []sentEvent +} + +func (e *eventLog) Emit(name string, body any, _ ...stdio.EmitOptions) error { + e.mu.Lock() + defer e.mu.Unlock() + e.sent = append(e.sent, sentEvent{name, body}) + return nil +} + +func (e *eventLog) names() []string { + e.mu.Lock() + defer e.mu.Unlock() + var out []string + for _, s := range e.sent { + out = append(out, s.Name) + } + return out +} diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index dd784305..931d625f 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -168,6 +168,34 @@ func tools(m *Machine) []stdio.Tool { return m.UpgradeHealth(), nil }, }, + { + Name: "pacman_upgrades", + Description: "Every full upgrade and rollback of every node, from the bus (ADR 0508 §1): each run's node, kind, what started it, " + + "its round, when it began and ended, its outcome (refused, finished, failed or running) and whether its after-upgrade check passed; " + + "narrowed to a node and to days, newest first. With a node and a run, that run whole, every step with its time.", + Input: schema(map[string]any{ + "node": map[string]any{"type": "string", "description": "only this node's runs"}, + "from": sinceArg, + "until": map[string]any{"type": "string", "description": "a day, YYYY-MM-DD, the last one listed"}, + "run": map[string]any{"type": "string", "description": "one run, by its id, with node"}, + "limit": limitArg(50, 1000), + }), + Run: func(args map[string]any) (any, error) { + var v [4]string + for i, k := range []string{"node", "from", "until", "run"} { + s, err := text(args, k, false) + if err != nil { + return nil, err + } + v[i] = s + } + n, err := whole(args, "limit", 50, 1, 1000) + if err != nil { + return nil, err + } + return ListUpgrades(upgradeState(StateRuns), v[0], v[1], v[2], v[3], n) + }, + }, { Name: "pacman_orphans", Description: "Packages installed as dependencies that nothing requires any more (pacman -Qdt), with versions.", diff --git a/modules/pacman/cmd/pacman-tools/record.go b/modules/pacman/cmd/pacman-tools/record.go new file mode 100644 index 00000000..25570861 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record.go @@ -0,0 +1,183 @@ +package main + +// The package record taken before every upgrade (novox/hq ADR 0508 §2): a copy of the node's `installed` key, +// the running kernel, the units failed at that moment and the installed DKMS modules, written to `records` at +// `.` and read back. A run does not start, and says why, when the node's packages are marked not +// current or the record cannot be written and read back equal. Records older than `record-days` are deleted. + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// OSRelease is the running kernel's release. +const OSRelease = "/proc/sys/kernel/osrelease" + +// defaultRecordDays is record-days' default in the manifest, for a config file that holds none. +const defaultRecordDays = 180 + +// FailedUnits are the units of the service manager failed now, by name. +func (m *Machine) FailedUnits() ([]string, error) { + out, err := m.Out("systemctl", "list-units", "--state=failed", "--plain", "--no-legend", "--no-pager") + if err != nil { + return nil, err + } + units := []string{} + for _, l := range lines(out) { + if f := strings.Fields(l); len(f) > 0 { + units = append(units, f[0]) + } + } + sort.Strings(units) + return units, nil +} + +// ParseDKMSStatus reads `dkms status`: each module at its version and the kernels it is installed for. A module +// only added (not built for any kernel) is listed with none. Both forms dkms has written are read: +// `name/version, kernel, arch: state` and the older `name, version, kernel, arch: state`. +func ParseDKMSStatus(out string) []DKMSModule { + byModule := map[string]*DKMSModule{} + var order []string + for _, l := range lines(out) { + head, state, ok := strings.Cut(l, ": ") + if !ok { + continue + } + parts := strings.Split(head, ", ") + var name, version, kernel string + if n, v, slash := strings.Cut(parts[0], "/"); slash { + name, version = n, v + if len(parts) > 1 { + kernel = parts[1] + } + } else if len(parts) >= 2 { + name, version = parts[0], parts[1] + if len(parts) > 2 { + kernel = parts[2] + } + } else { + continue + } + key := name + "/" + version + m := byModule[key] + if m == nil { + m = &DKMSModule{Module: name, Version: version, Kernels: []string{}} + byModule[key] = m + order = append(order, key) + } + if kernel != "" && strings.HasPrefix(strings.TrimSpace(state), "installed") { + m.Kernels = append(m.Kernels, kernel) + } + } + sort.Strings(order) + modules := []DKMSModule{} + for _, k := range order { + sort.Strings(byModule[k].Kernels) + modules = append(modules, *byModule[k]) + } + return modules +} + +// DKMSModules are the DKMS modules installed on the machine; none when dkms is not installed. +func (m *Machine) DKMSModules() ([]DKMSModule, error) { + r := m.Run(bg(), "dkms", "status") + if r.Err == "ENOENT" { + return []DKMSModule{}, nil + } + if r.Status != 0 || r.Err != "" { + return nil, failure("dkms", "dkms", r) + } + return ParseDKMSStatus(r.Stdout), nil +} + +// TakeRecord takes the node's record for a run, writes it and reads it back. It answers the record, or why +// it could not be taken: a run that has a why does not start. +func (u *Upgrades) TakeRecord(run string) (*Record, string) { + nc, err := u.Installed.Get(NotCurrentKey(u.Node)) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if nc != nil { + var mark NotCurrent + _ = json.Unmarshal(nc.Value, &mark) + return nil, fmt.Sprintf("the node's packages on the bus are not current since %s (%d bytes, over the bus's cap on a value): a record of them would not be what is installed", + mark.Since.Format(time.RFC3339), mark.Bytes) + } + e, err := u.Installed.Get(u.Node) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if e == nil { + return nil, "the node's packages are not on the bus yet: there is nothing to record" + } + var installed InstalledValue + if err := json.Unmarshal(e.Value, &installed); err != nil { + return nil, "the node's packages on the bus are not readable: " + err.Error() + } + kernel, err := u.Machine.ReadFile(OSRelease) + if err != nil { + return nil, "the running kernel could not be read: " + err.Error() + } + failed, err := u.Machine.FailedUnits() + if err != nil { + return nil, "the failed units could not be read: " + err.Error() + } + dkms, err := u.Machine.DKMSModules() + if err != nil { + return nil, "the DKMS modules could not be read: " + err.Error() + } + rec := Record{Node: u.Node, Run: run, At: u.Machine.Now().UTC(), Installed: installed, + Kernel: strings.TrimSpace(string(kernel)), FailedUnits: failed, DKMS: dkms} + key := NodeKey(u.Node, run) + if _, err := u.Records.Put(key, rec); err != nil { + return nil, "the record could not be written: " + err.Error() + } + back, err := u.Records.Get(key) + if err != nil { + return nil, "the record could not be read back: " + err.Error() + } + if back == nil { + return nil, "the record was written and is not there when read back" + } + var read Record + want, _ := json.Marshal(rec) + if err := json.Unmarshal(back.Value, &read); err != nil { + return nil, "the record read back is not readable: " + err.Error() + } + if got, _ := json.Marshal(read); !bytes.Equal(got, want) { + return nil, "the record read back differs from the one written" + } + return &rec, "" +} + +// PruneRecords deletes the node's records older than record-days, judged by the time of their run, and +// answers how many it deleted. +func (u *Upgrades) PruneRecords() (int, error) { + days := u.Config.RecordDays + if days < 1 { + days = defaultRecordDays + } + cutoff := u.Machine.Now().AddDate(0, 0, -days) + keys, err := u.Records.Keys() + if err != nil { + return 0, fmt.Errorf("listing records: %w", err) + } + n := 0 + for _, k := range keys { + node, id, ok := strings.Cut(k, ".") + if !ok || node != u.Node { + continue + } + if t, ok := runTime(id); ok && t.Before(cutoff) { + if err := u.Records.Delete(k); err != nil { + return n, fmt.Errorf("deleting record %s: %w", k, err) + } + n++ + } + } + return n, nil +} diff --git a/modules/pacman/cmd/pacman-tools/record_test.go b/modules/pacman/cmd/pacman-tools/record_test.go new file mode 100644 index 00000000..ac1dca95 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record_test.go @@ -0,0 +1,152 @@ +package main + +// The record before every upgrade (novox/hq ADR 0508 §2; acceptance 1841/AC3). + +import ( + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func capturedDKMSStatus(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("testdata/dkms-status-installed-and-added/status") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// recordMachine is a machine running one kernel, with one unit failed and the captured DKMS modules. +func recordMachine(t *testing.T) *Machine { + m := machine(byLine(map[string]Ran{ + "systemctl list-units --state=failed --plain --no-legend --no-pager": {Stdout: "backup.service loaded failed failed The backup\n"}, + "dkms status": {Stdout: capturedDKMSStatus(t)}, + }, nil), 1000) + m.ReadFile = func(path string) ([]byte, error) { + if path == OSRelease { + return []byte("7.2.8-arch1-2\n"), nil + } + return nil, errNoFile + } + return m +} + +var nodeAPackages = InstalledValue{At: time.Date(2026, 10, 3, 9, 0, 0, 0, time.UTC), Packages: []InstalledPackage{ + {Name: "linux", Version: "7.2.8.arch1-2", Explicit: true, Repo: "core"}, + {Name: "zfs-dkms", Version: "2.4.4-1", Explicit: true, Repo: RepoForeign}, +}} + +func recordUpgrades(t *testing.T) (*Upgrades, *memBucket, *memBucket) { + installed, records := newBucket(), newBucket() + if _, err := installed.Put("node-a", nodeAPackages); err != nil { + t.Fatal(err) + } + return &Upgrades{Node: "node-a", Machine: recordMachine(t), Installed: installed, Records: records}, installed, records +} + +func TestDKMSStatusInBothFormsIsEachModuleWithTheKernelsItIsBuiltFor(t *testing.T) { + mods := ParseDKMSStatus(capturedDKMSStatus(t) + "zfs, 2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs, 2.4.4, 6.18.54-2-lts, x86_64: installed\n") + var got []string + for _, m := range mods { + got = append(got, m.Module+"/"+m.Version+"="+strings.Join(m.Kernels, "+")) + } + want := "openrazer-driver/3.12.4=7.2.8-arch1-2 vboxhost/7.2.20_OSE=7.2.8-arch1-2 xone/0.5.8=7.2.8-arch1-2 xpad/0.4= zfs/2.4.4=6.18.54-2-lts+7.2.8-arch1-2" + if strings.Join(got, " ") != want { + t.Errorf("%v", got) + } +} + +func TestARecordIsTheNodesPackagesKernelFailedUnitsAndDKMSWrittenAndReadBack(t *testing.T) { + u, _, records := recordUpgrades(t) + rec, why := u.TakeRecord("20261004T120000Z") + if why != "" || rec == nil { + t.Fatalf("refused: %s", why) + } + var kept Record + if !records.value("node-a.20261004T120000Z", &kept) { + t.Fatal("no record at .") + } + a, _ := json.Marshal(kept.Installed) + b, _ := json.Marshal(nodeAPackages) + if string(a) != string(b) { + t.Errorf("the record copies installed.: %s", a) + } + if kept.Kernel != "7.2.8-arch1-2" || strings.Join(kept.FailedUnits, " ") != "backup.service" || len(kept.DKMS) != 4 || + kept.Node != "node-a" || kept.Run != "20261004T120000Z" { + t.Errorf("record: %+v", kept) + } + if w := strings.Join(records.writes, " "); w != "put node-a.20261004T120000Z" { + t.Errorf("writes: %s", w) + } +} + +func TestNoRecordAndNoRunWhenThePackagesAreNotCurrentOrTheRecordIsNotReadBack(t *testing.T) { + u, installed, records := recordUpgrades(t) + if _, err := installed.Put(NotCurrentKey("node-a"), NotCurrent{Bytes: 300000, Since: time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)}); err != nil { + t.Fatal(err) + } + if rec, why := u.TakeRecord("r1"); rec != nil || !strings.Contains(why, "not current") || !strings.Contains(why, "300000") { + t.Errorf("not current: %v %q", rec, why) + } + if len(records.writes) != 0 { + t.Errorf("a record of packages not current was written: %v", records.writes) + } + + u, installed, _ = recordUpgrades(t) + _ = installed.Delete("node-a") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "not on the bus") { + t.Errorf("no installed key: %q", why) + } + + u, _, records = recordUpgrades(t) + records.fail["put node-a.r1"] = errors.New("value over the bucket's limit") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "could not be written") || !strings.Contains(why, "over the bucket's limit") { + t.Errorf("not written: %q", why) + } + + u, _, _ = recordUpgrades(t) + u.Records = &mangling{newBucket()} + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "differs") { + t.Errorf("read back unequal: %q", why) + } +} + +// mangling is a state that reads back other than what was written. +type mangling struct{ *memBucket } + +func (m *mangling) Get(key string) (*stdio.StateEntry, error) { + e, err := m.memBucket.Get(key) + if e != nil { + e.Value = []byte(strings.Replace(string(e.Value), "7.2.8-arch1-2", "7.2.7-arch1-1", 1)) + } + return e, err +} + +func TestRecordsOlderThanRecordDaysAreDeletedAndYoungerOnesKept(t *testing.T) { + u, _, records := recordUpgrades(t) + u.Config.RecordDays = 180 + now := u.Machine.Now() // 2026-10-04 + for _, k := range []string{ + NodeKey("node-a", RunID(now.AddDate(0, 0, -181))), + NodeKey("node-a", RunID(now.AddDate(0, 0, -179))), + NodeKey("node-b", RunID(now.AddDate(0, 0, -400))), + } { + if _, err := records.Put(k, Record{}); err != nil { + t.Fatal(err) + } + } + n, err := u.PruneRecords() + if err != nil || n != 1 { + t.Fatalf("%d deleted, %v", n, err) + } + keys, _ := records.Keys() + if strings.Join(keys, " ") != NodeKey("node-a", RunID(now.AddDate(0, 0, -179)))+" "+NodeKey("node-b", RunID(now.AddDate(0, 0, -400))) { + t.Errorf("kept: %v (the younger record, and another node's, which its own keeper prunes)", keys) + } +} diff --git a/modules/pacman/cmd/pacman-tools/runs_test.go b/modules/pacman/cmd/pacman-tools/runs_test.go new file mode 100644 index 00000000..d339b695 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/runs_test.go @@ -0,0 +1,217 @@ +package main + +// Runs on the bus and the log read once (novox/hq ADR 0508 §1; acceptance 1841/AC1 for the writing of steps, +// 1841/AC2 for the log and pacman_upgrades). + +import ( + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func capturedLog(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("testdata/pacman-log-three-full-upgrades/pacman.log") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// The captured log holds three full upgrades among other commands: one that ended before any transaction, one +// whose transaction completed, and a later one that upgraded one package. +func TestEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { + runs := PastUpgrades("node-a", capturedLog(t)) + if len(runs) != 3 { + t.Fatalf("%d runs, want the three full upgrades and none of the other commands: %+v", len(runs), runs) + } + for _, r := range runs { + if r.Node != "node-a" || r.Kind != RunUpgrade || r.From != FromLog || len(r.Steps) != 2 || r.Steps[0].Step != StepStarted { + t.Errorf("a run read from the log: %+v", r) + } + } + first, second, third := runs[0], runs[1], runs[2] + if first.Run != "20261004T130648Z" || first.Steps[1].Step != StepFinished || first.Steps[1].Why != noTransaction || + len(first.Steps[1].Changes.Changed) != 0 { + t.Errorf("an upgrade with no transaction says so: %+v", first) + } + end := second.Steps[1] + if second.Run != "20261004T130724Z" || end.Step != StepFinished || !end.At.Equal(time.Date(2026, 10, 4, 13, 22, 50, 0, time.UTC)) { + t.Errorf("the second upgrade finished when its transaction completed: %+v", second) + } + var names []string + for _, c := range end.Changes.Changed { + names = append(names, c.Name) + } + if strings.Join(names, " ") != "bash glibc iana-etc readline tzdata wireshark-qt xfsprogs xpad xterm zathura-pdf-mupdf" { + t.Errorf("what it changed, sorted: %v", names) + } + if c := end.Changes.Changed[0]; c.From != "5.3.15-1" || c.To != "5.3.20-1" { + t.Errorf("bash: %+v", c) + } + if len(end.Changes.Removed) != 0 { + t.Errorf("the removal run between them is no part of an upgrade: %+v", end.Changes.Removed) + } + if c := third.Steps[1].Changes.Changed; third.Run != "20261004T132817Z" || len(c) != 1 || c[0].Name != "dunst" { + t.Errorf("the third upgrade: %+v", third) + } +} + +func TestATransactionWithoutAnEndOrThatFailedIsAFailedRun(t *testing.T) { + log := "[2026-10-10T04:00:00+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + + "[2026-10-10T04:00:01+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-10T04:00:05+0000] [ALPM] transaction started\n" + + "[2026-10-10T04:00:06+0000] [ALPM] upgraded linux (7.2.8.arch1-2 -> 7.2.9.arch1-1)\n" + + "[2026-10-10T04:00:09+0000] [ALPM] transaction failed\n" + + "[2026-10-11T04:00:00+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + + "[2026-10-11T04:00:01+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-11T04:00:05+0000] [ALPM] transaction started\n" + runs := PastUpgrades("node-b", log) + if len(runs) != 2 { + t.Fatalf("%+v", runs) + } + if s := runs[0].Steps[1]; s.Step != StepFailed || s.Error != "transaction failed" || len(s.Changes.Changed) != 1 { + t.Errorf("a failed transaction: %+v", s) + } + if s := runs[1].Steps[1]; s.Step != StepFailed || !strings.Contains(s.Error, "no end") { + t.Errorf("a transaction the log holds no end of: %+v", s) + } +} + +func TestTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing(t *testing.T) { + m := machine(nil, 1000) + reads := 0 + log := capturedLog(t) + m.ReadFile = func(path string) ([]byte, error) { + if path != PacmanLog { + t.Fatalf("read %s", path) + } + reads++ + return []byte(log), nil + } + runs := newBucket() + other := Run{Node: "node-b", Run: "20261001T040000Z", Kind: RunUpgrade, From: FromRound} + if _, err := runs.Create(NodeKey("node-b", other.Run), other); err != nil { + t.Fatal(err) + } + u := &Upgrades{Node: "node-a", Machine: m, Runs: runs} + n, err := u.ImportLog() + if err != nil || n != 3 || reads != 1 { + t.Fatalf("first start: %d runs, %d reads, %v", n, reads, err) + } + var r Run + if !runs.value("node-a.20261004T132817Z", &r) || r.From != FromLog { + t.Errorf("the run is kept at ., marked as read from the log: %+v", r) + } + before := len(runs.writes) + n, err = u.ImportLog() + if err != nil || n != 0 || reads != 1 || len(runs.writes) != before { + t.Errorf("a second start: %d runs, %d reads, %d writes, %v", n, reads, len(runs.writes)-before, err) + } +} + +// A run's key is created at its first step and then only updated from the revision its writer holds: a step +// is written once, and a run another writer changed meanwhile is refused, never overwritten. +func TestAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten(t *testing.T) { + runs := newBucket() + u := &Upgrades{Node: "node-a", Runs: runs} + r := &openRun{Run: Run{Node: "node-a", Run: "20261024T040000Z", Kind: RunUpgrade, From: FromAsk}} + at := time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + for _, s := range []string{StepRecorded, StepCheckedBefore} { + if err := u.step(r, Step{Step: s, At: at}); err != nil { + t.Fatal(err) + } + } + if w := strings.Join(runs.writes, ", "); w != "create node-a.20261024T040000Z, update node-a.20261024T040000Z" { + t.Errorf("writes: %s", w) + } + var kept Run + runs.value("node-a.20261024T040000Z", &kept) + if len(kept.Steps) != 2 || kept.Steps[1].Step != StepCheckedBefore { + t.Errorf("kept: %+v", kept) + } + if _, err := runs.Put("node-a.20261024T040000Z", kept); err != nil { + t.Fatal(err) + } + if err := u.step(r, Step{Step: StepStarted, At: at}); !errors.Is(err, stdio.ErrChanged) || len(r.Steps) != 2 { + t.Errorf("a step over another writer's change: %v, %d steps held", err, len(r.Steps)) + } +} + +func TestPacmanUpgradesAnswersAllOneNodeDaysAndOneRun(t *testing.T) { + runs := newBucket() + put := func(node, id string, steps ...Step) { + r := Run{Node: node, Run: id, Kind: RunUpgrade, From: FromRound, Round: "2026-W43", Steps: steps} + if _, err := runs.Create(NodeKey(node, id), r); err != nil { + t.Fatal(err) + } + } + at := time.Date(2026, 10, 24, 4, 30, 0, 0, time.UTC) + passed := &Check{Passed: true} + put("node-a", "20261024T040000Z", Step{Step: StepRecorded, At: at}, Step{Step: StepFinished, At: at}, Step{Step: StepCheckedAfter, At: at, Check: passed}) + put("node-b", "20261024T050000Z", Step{Step: StepRecorded, At: at}, Step{Step: StepStarted, At: at}) + put("node-b", "20261017T050000Z", Step{Step: StepRefused, At: at, Why: "installed is not current"}) + all, err := ListUpgrades(runs, "", "", "", "", 50) + if err != nil { + t.Fatal(err) + } + listed := all["runs"].([]RunSummary) + var order []string + for _, s := range listed { + order = append(order, s.Node+"/"+s.Run+"/"+s.Outcome+"/"+s.Checked) + } + if strings.Join(order, " ") != "node-b/20261024T050000Z/running/ node-a/20261024T040000Z/finished/passed node-b/20261017T050000Z/refused/" { + t.Fatalf("all runs, newest first: %v", order) + } + if listed[2].Error != "installed is not current" { + t.Errorf("a refused run says why: %+v", listed[2]) + } + one, _ := ListUpgrades(runs, "node-b", "", "", "", 50) + if one["count"] != 2 { + t.Errorf("one node: %v", one) + } + days, _ := ListUpgrades(runs, "", "2026-10-18", "2026-10-24", "", 50) + if days["count"] != 2 { + t.Errorf("between two days, the last one included: %v", days) + } + limited, _ := ListUpgrades(runs, "", "", "", "", 1) + if limited["count"] != 1 || limited["truncated"] != true { + t.Errorf("limited: %v", limited) + } + whole, err := ListUpgrades(runs, "node-a", "", "", "20261024T040000Z", 50) + if err != nil || len(whole["run"].(Run).Steps) != 3 { + t.Errorf("one run whole: %v %v", whole, err) + } + if _, err := ListUpgrades(runs, "node-a", "", "", "20261024T050000Z", 50); err == nil { + t.Error("a run the node does not have was answered") + } + if _, err := ListUpgrades(runs, "", "", "", "20261024T040000Z", 50); err == nil { + t.Error("a run without its node was answered") + } +} + +func TestTheUpgradesToolReadsTheRunsState(t *testing.T) { + runs := newBucket() + saved := upgradeState + upgradeState = func(name string) Bucket { + if name != StateRuns { + t.Fatalf("read %s", name) + } + return runs + } + defer func() { upgradeState = saved }() + for _, tool := range tools(machine(nil, 1000)) { + if tool.Name == "pacman_upgrades" { + out, err := tool.Run(map[string]any{"node": "node-a"}) + if err != nil || out.(map[string]any)["count"] != 0 { + t.Errorf("%v %v", out, err) + } + return + } + } + t.Fatal("no pacman_upgrades") +} diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status b/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status new file mode 100644 index 00000000..d19fd59a --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status @@ -0,0 +1,4 @@ +openrazer-driver/3.12.4, 7.2.8-arch1-2, x86_64: installed +vboxhost/7.2.20_OSE, 7.2.8-arch1-2, x86_64: installed +xone/0.5.8, 7.2.8-arch1-2, x86_64: installed +xpad/0.4: added diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META new file mode 100644 index 00000000..8133bf1e --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META @@ -0,0 +1,10 @@ +Meta: 1 +Name: zfs +Branch: 1.0 +Version: 2.4.4 +Release: 1 +Release-Tags: relext +License: CDDL +Author: OpenZFS +Linux-Maximum: 7.2 +Linux-Minimum: 4.18 diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf new file mode 100644 index 00000000..50136d8a --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf @@ -0,0 +1,56 @@ +PACKAGE_NAME="zfs" +PACKAGE_VERSION="2.4.4" +PACKAGE_CONFIG="/etc/sysconfig/zfs" +NO_WEAK_MODULES="yes" +PRE_BUILD="configure + --disable-dependency-tracking + --prefix=/usr + --with-config=kernel + --with-linux="${kernel_source_dir}" + --with-linux-obj="${kernel_source_dir}" + $( + [[ -n \"${ICP_ROOT}\" ]] && \ + { + echo --with-qat=\"${ICP_ROOT}\" + } + ) + $( + [[ -r ${PACKAGE_CONFIG} ]] \ + && source ${PACKAGE_CONFIG} \ + && shopt -q -s extglob \ + && \ + { + if [[ ${ZFS_DKMS_ENABLE_DEBUG,,} == @(y|yes) ]] + then + echo --enable-debug + fi + if [[ ${ZFS_DKMS_ENABLE_DEBUGINFO,,} == @(y|yes) ]] + then + echo --enable-debuginfo + fi + } + ) +" +POST_BUILD="scripts/dkms.postbuild + -n ${PACKAGE_NAME} + -v ${PACKAGE_VERSION} + -a ${arch} + -k ${kernelver} + -t ${dkms_tree} +" +AUTOINSTALL="yes" +MAKE[0]="make -C module/" +STRIP[0]="$( + [[ -r ${PACKAGE_CONFIG} ]] \ + && source ${PACKAGE_CONFIG} \ + && shopt -q -s extglob \ + && [[ ${ZFS_DKMS_DISABLE_STRIP,,} == @(y|yes) ]] \ + && echo -n no +)" +STRIP[1]="${STRIP[0]}" +BUILT_MODULE_NAME[0]="zfs" +BUILT_MODULE_LOCATION[0]="module/" +DEST_MODULE_LOCATION[0]="/extra" +BUILT_MODULE_NAME[1]="spl" +BUILT_MODULE_LOCATION[1]="module/" +DEST_MODULE_LOCATION[1]="/extra" diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log b/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log new file mode 100644 index 00000000..e5b1b6d4 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log @@ -0,0 +1,64 @@ +[2026-10-04T14:52:51+0200] [ALPM] installed logrotate (3.22.0-1) +[2026-10-04T14:52:51+0200] [ALPM] transaction completed +[2026-10-04T14:52:51+0200] [ALPM] running '30-systemd-daemon-reload-system.hook'... +[2026-10-04T14:54:22+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:06:13+0200] [PACMAN] Running 'pacman -Syu --noconfirm' +[2026-10-04T15:06:13+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:06:48+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:07:23+0200] [PACMAN] Running 'pacman -Rns --noconfirm lib32-libpcap' +[2026-10-04T15:07:23+0200] [ALPM] transaction started +[2026-10-04T15:07:24+0200] [ALPM] removed lib32-libpcap (1.10.6-1) +[2026-10-04T15:07:24+0200] [ALPM] removed lib32-libnl (3.12.0-1) +[2026-10-04T15:07:24+0200] [ALPM] transaction completed +[2026-10-04T15:07:24+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:07:24+0200] [PACMAN] Running 'pacman -Syu --noconfirm' +[2026-10-04T15:07:24+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:07:24+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:16:51+0200] [ALPM] running '60-mkinitcpio-remove.hook'... +[2026-10-04T15:16:52+0200] [ALPM] running '71-dkms-remove.hook'... +[2026-10-04T15:16:52+0200] [ALPM-SCRIPTLET] ==> dkms remove --no-depmod xone/0.5.8 -k 7.1.9-arch1-2 +[2026-10-04T15:16:52+0200] [ALPM-SCRIPTLET] ==> depmod 7.1.9-arch1-2 +[2026-10-04T15:16:55+0200] [ALPM] running 'ghc-unregister.hook'... +[2026-10-04T15:16:57+0200] [ALPM] running 'texinfo-remove.hook'... +[2026-10-04T15:16:58+0200] [ALPM] transaction started +[2026-10-04T15:16:58+0200] [ALPM] upgraded tzdata (2026c-1 -> 2026e-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded iana-etc (20260530-1 -> 20260617-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded glibc (2.44+r24+g16be1518495f-1 -> 2.44+r50+g1848099f063e-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded readline (8.3.003-1 -> 8.3.6-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded bash (5.3.15-1 -> 5.3.20-1) +[2026-10-04T15:22:46+0200] [ALPM] upgraded wireshark-qt (4.7.2-1 -> 4.7.3-1) +[2026-10-04T15:22:46+0200] [ALPM] upgraded xfsprogs (7.1.1-1 -> 7.2.0-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded xpad (5.8.0-1 -> 5.9.0-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded xterm (410-1 -> 411-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded zathura-pdf-mupdf (2026.07.18-1 -> 2026.07.18-4) +[2026-10-04T15:22:50+0200] [ALPM] transaction completed +[2026-10-04T15:22:52+0200] [ALPM] running '10-glibc-locale-gen.hook'... +[2026-10-04T15:22:52+0200] [ALPM-SCRIPTLET] Generating locales... +[2026-10-04T15:22:53+0200] [ALPM-SCRIPTLET] en_US.UTF-8... done +[2026-10-04T15:22:53+0200] [ALPM-SCRIPTLET] Generation complete. +[2026-10-04T15:22:53+0200] [ALPM] running '11-glibc-ldconfig.hook'... +[2026-10-04T15:24:49+0200] [ALPM] running 'texinfo-install.hook'... +[2026-10-04T15:24:51+0200] [ALPM] running 'update-desktop-database.hook'... +[2026-10-04T15:24:51+0200] [ALPM] running 'update-vlc-plugin-cache.hook'... +[2026-10-04T15:24:52+0200] [ALPM] running 'xorg-mkfontscale.hook'... +[2026-10-04T15:28:15+0200] [PACMAN] Running 'pacman -Syu' +[2026-10-04T15:28:15+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:28:17+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:30:26+0200] [ALPM] transaction started +[2026-10-04T15:30:26+0200] [ALPM] upgraded dunst (1.13.2-2 -> 1.13.2-3) +[2026-10-04T15:30:26+0200] [ALPM] transaction completed +[2026-10-04T15:30:26+0200] [ALPM] running '30-systemd-daemon-reload-user.hook'... +[2026-10-04T15:30:26+0200] [ALPM-SCRIPTLET] Failed to connect to system scope bus via local transport: Connection refused +[2026-10-04T15:30:26+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:30:26+0200] [ALPM] running 'i3status-rust.hook'... +[2026-10-04T16:00:43+0200] [PACMAN] Running 'pacman -R --noconfirm ntp' +[2026-10-04T16:00:43+0200] [ALPM] transaction started +[2026-10-04T16:00:45+0200] [ALPM] removed ntp (4.2.8.p18-6) +[2026-10-04T16:00:45+0200] [ALPM] transaction completed +[2026-10-04T16:00:45+0200] [ALPM] running '30-systemd-daemon-reload-system.hook'... +[2026-10-04T16:00:45+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-05T10:04:50+0200] [PACMAN] Running 'pacman -S --noconfirm --ask 4 lemurs' +[2026-10-05T10:04:51+0200] [ALPM] transaction started +[2026-10-05T10:04:51+0200] [ALPM] warning: /etc/lemurs/config.toml saved as /etc/lemurs/config.toml.pacsave +[2026-10-05T10:04:51+0200] [ALPM] removed lemurs-git (0.4.0.r17.g7151336-2) +[2026-10-05T10:04:51+0200] [ALPM] installed lemurs (0.4.0-1) diff --git a/modules/pacman/cmd/pacman-tools/upgrades.go b/modules/pacman/cmd/pacman-tools/upgrades.go new file mode 100644 index 00000000..90a01d71 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrades.go @@ -0,0 +1,392 @@ +package main + +// Upgrades as runs on the bus (novox/hq ADR 0508 §1): every full upgrade of a node, and every rollback, is a run +// in the state `runs` at `.`, written at each of its steps and at no other time, and each step is also +// an event. The keeper's loop calls what is here; everything it reaches is behind an interface (a state, the +// events, the root acts) or the Machine's injected runner and files, so it is tested without a bus or a machine. + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Bucket is one of the module's states as a writer sees it: a process's state, or a tools bundle's +// (stdio.KeptState). A write that lost a race is stdio.ErrChanged. +type Bucket interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) + Put(key string, value any) (uint64, error) + Delete(key string) error + Keys() ([]string, error) +} + +// EventSink emits the keeper's events: the process's connection (process.Conn). +type EventSink interface { + Emit(eventType string, body any, opts ...stdio.EmitOptions) error +} + +// RootActs are the acts that need root, carried out by the module's root helper (ADR 0510): the keeper never +// runs them itself. +type RootActs interface { + // Transaction starts the run's transaction as a unit of the service manager, named TransactionUnit(kind, + // run), leaving out the packages ignore names, and answers once it started. + Transaction(kind, run string, ignore []string) error + // Journal is the last n lines a transaction unit logged. + Journal(unit string, n int) ([]string, error) +} + +// TransactionUnit is the unit a run's transaction runs as: named by the run, so a keeper started again finds it. +func TransactionUnit(kind, run string) string { + return unitPrefix + kind + "-" + strings.ToLower(run) +} + +// Upgrades is one node's upgrades: what the keeper's loop calls for its own node. +type Upgrades struct { + Node string + Config KeeperConfig + Machine *Machine + Root RootActs + Events EventSink + // The module's states. + Installed, Updates, Runs, Records, Asks Bucket + // Check is the after-upgrade check (ADR 0508 §4); nil checks only that the transaction succeeded. + Check func(run Run, record *Record) Check +} + +// openRun is a run being written, with the revision of its key. +type openRun struct { + Run + rev uint64 +} + +// step writes the run with one more step: created at its first step, then only ever updated from the revision +// this writer wrote, so a second writer is refused rather than overwritten. +func (u *Upgrades) step(r *openRun, s Step) error { + r.Steps = append(r.Steps, s) + key := NodeKey(r.Node, r.Run.Run) + var err error + if r.rev == 0 { + r.rev, err = u.Runs.Create(key, r.Run) + } else { + r.rev, err = u.Runs.Update(key, r.Run, r.rev) + } + if err != nil { + r.Steps = r.Steps[:len(r.Steps)-1] + return fmt.Errorf("writing step %s of run %s: %w", s.Step, key, err) + } + return nil +} + +// emit says a step as its event; an event that is not sent is said in the error, the step stays written. +func (u *Upgrades) emit(event string, body any) error { + if u.Events == nil { + return nil + } + if err := u.Events.Emit(event, body); err != nil { + return fmt.Errorf("emitting %s: %w", event, err) + } + return nil +} + +// The package manager's log, read as blocks: each begins at a `Running ''` line (or a full upgrade +// starting with none before it) and holds the transaction it ran and the packages it changed. +var ( + runningLine = regexp.MustCompile(`^\[([^\]]+)\] \[PACMAN\] Running '(.*)'$`) + transactionLine = regexp.MustCompile(`^\[([^\]]+)\] \[ALPM\] transaction (started|completed|failed|interrupted)$`) +) + +// logBlock is one command pacman ran, as its log tells it. +type logBlock struct { + Start time.Time + Command string + Full bool // it started a full system upgrade + FullAt time.Time // when + Began bool // a transaction started + End string // completed, failed or interrupted; empty when the log holds no end + EndAt time.Time + Changes Changes +} + +// logBlocks reads pacman's log into the commands it ran, oldest first. +func logBlocks(text string) []logBlock { + var blocks []logBlock + var cur *logBlock + open := func(at time.Time, command string) { + blocks = append(blocks, logBlock{Start: at, Command: command}) + cur = &blocks[len(blocks)-1] + } + for _, l := range strings.Split(text, "\n") { + l = strings.TrimRight(l, "\r") + if m := runningLine.FindStringSubmatch(l); m != nil { + if t, ok := logTime(m[1]); ok { + open(t, m[2]) + } + continue + } + if m := fullUpdate.FindStringSubmatch(l); m != nil { + t, ok := logTime(m[1]) + if !ok { + continue + } + if cur == nil || cur.Full || cur.Began { + open(t, "") + } + cur.Full, cur.FullAt = true, t + continue + } + if cur == nil { + continue + } + if m := transactionLine.FindStringSubmatch(l); m != nil { + t, _ := logTime(m[1]) + if m[2] == "started" { + cur.Began = true + } else if cur.Began && cur.End == "" { + cur.End, cur.EndAt = m[2], t + } + continue + } + if cur.Began && cur.End == "" { + if m := logLine.FindStringSubmatch(l); m != nil { + addChange(&cur.Changes, m[2], m[3], m[4]) + } + } + } + for i := range blocks { + sortChanges(&blocks[i].Changes) + } + return blocks +} + +func addChange(c *Changes, action, name, version string) { + switch action { + case "installed": + c.Added = append(c.Added, PackageVersion{Name: name, Version: version}) + case "removed": + c.Removed = append(c.Removed, PackageVersion{Name: name, Version: version}) + case "upgraded", "downgraded": + from, to, _ := strings.Cut(version, " -> ") + c.Changed = append(c.Changed, VersionChange{Name: name, From: from, To: to}) + } +} + +func sortChanges(c *Changes) { + if c.Added == nil { + c.Added = []PackageVersion{} + } + if c.Removed == nil { + c.Removed = []PackageVersion{} + } + if c.Changed == nil { + c.Changed = []VersionChange{} + } + sort.Slice(c.Added, func(i, j int) bool { return c.Added[i].Name < c.Added[j].Name }) + sort.Slice(c.Removed, func(i, j int) bool { return c.Removed[i].Name < c.Removed[j].Name }) + sort.Slice(c.Changed, func(i, j int) bool { return c.Changed[i].Name < c.Changed[j].Name }) +} + +// noTransaction is why a full upgrade the log tells of has no transaction: the log does not say which. +const noTransaction = "the log holds no transaction for this upgrade: there was nothing to do, or it ended before changing anything" + +// PastUpgrades is every full upgrade pacman's log tells of, as runs read from the log (ADR 0508 §1): started +// when the upgrade started, finished or failed when its transaction ended, with what it changed. One whose +// transaction has no end in the log failed, so said. +func PastUpgrades(node, text string) []Run { + var runs []Run + for _, b := range logBlocks(text) { + if !b.Full { + continue + } + r := Run{Node: node, Run: RunID(b.FullAt), Kind: RunUpgrade, From: FromLog, + Steps: []Step{{Step: StepStarted, At: b.FullAt}}} + changes := b.Changes + switch { + case !b.Began: + r.Steps = append(r.Steps, Step{Step: StepFinished, At: b.FullAt, Why: noTransaction, Changes: &changes}) + case b.End == "completed": + r.Steps = append(r.Steps, Step{Step: StepFinished, At: b.EndAt, Changes: &changes}) + case b.End == "": + r.Steps = append(r.Steps, Step{Step: StepFailed, At: b.FullAt, Error: "the log holds no end of this transaction", Changes: &changes}) + default: + r.Steps = append(r.Steps, Step{Step: StepFailed, At: b.EndAt, Error: "transaction " + b.End, Changes: &changes}) + } + runs = append(runs, r) + } + return runs +} + +// ImportLog reads the node's past full upgrades from pacman's log into `runs` once: when the node has no run +// yet. It answers how many it wrote. A run read from the log is history and emits nothing. +func (u *Upgrades) ImportLog() (int, error) { + keys, err := u.Runs.Keys() + if err != nil { + return 0, fmt.Errorf("listing runs: %w", err) + } + for _, k := range keys { + if strings.HasPrefix(k, u.Node+".") { + return 0, nil + } + } + text, err := u.Machine.ReadFile(PacmanLog) + if err != nil { + return 0, fmt.Errorf("reading %s: %w", PacmanLog, err) + } + n := 0 + for _, r := range PastUpgrades(u.Node, string(text)) { + if _, err := u.Runs.Create(NodeKey(u.Node, r.Run), r); err != nil { + if errors.Is(err, stdio.ErrChanged) { + continue // two upgrades started in the same second: the first is kept + } + return n, fmt.Errorf("writing run %s: %w", r.Run, err) + } + n++ + } + return n, nil +} + +// runTime is when a run began, from its id. +func runTime(id string) (time.Time, bool) { + t, err := time.Parse("20060102T150405Z", id) + return t, err == nil +} + +// RunSummary is one run as pacman_upgrades lists it. +type RunSummary struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Began time.Time `json:"began"` + Outcome string `json:"outcome"` + Ended *time.Time `json:"ended,omitempty"` + Error string `json:"error,omitempty"` + Held int `json:"held_kernels,omitempty"` + Checked string `json:"check,omitempty"` +} + +// summary is where a run stands: its outcome is its last step that ends it (refused, finished, failed), else +// running. +func summary(r Run) RunSummary { + s := RunSummary{Node: r.Node, Run: r.Run, Kind: r.Kind, From: r.From, Round: r.Round, Outcome: "running"} + if t, ok := runTime(r.Run); ok { + s.Began = t + } + for _, st := range r.Steps { + switch st.Step { + case StepRefused, StepFinished, StepFailed: + at := st.At + s.Outcome, s.Ended = st.Step, &at + s.Error = st.Error + if st.Step == StepRefused { + s.Error = st.Why + } + case StepCheckedBefore: + s.Held = len(st.Held) + case StepCheckedAfter: + s.Checked = "failed" + if st.Check != nil && st.Check.Passed { + s.Checked = "passed" + } + } + } + return s +} + +// ListUpgrades answers pacman_upgrades from `runs`: one run whole, or the runs of every node or one, between +// two days, newest first, at most limit of them. +func ListUpgrades(runs Bucket, node, from, until, run string, limit int) (map[string]any, error) { + if run != "" { + if node == "" { + return nil, fmt.Errorf("a run is named with its node") + } + e, err := runs.Get(NodeKey(node, run)) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("no run %s of %s", run, node) + } + var r Run + if err := jsonValue(e.Value, &r); err != nil { + return nil, err + } + return map[string]any{"run": r, "summary": summary(r)}, nil + } + day := func(s string, end bool) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + t, err := time.ParseInLocation("2006-01-02", s, time.Local) + if err != nil { + return t, fmt.Errorf("%q is not a day as YYYY-MM-DD", s) + } + if end { + t = t.AddDate(0, 0, 1) + } + return t, nil + } + since, err := day(from, false) + if err != nil { + return nil, err + } + before, err := day(until, true) + if err != nil { + return nil, err + } + keys, err := runs.Keys() + if err != nil { + return nil, err + } + var listed []RunSummary + for _, k := range keys { + n, id, ok := strings.Cut(k, ".") + if !ok || node != "" && n != node { + continue + } + t, ok := runTime(id) + if !ok || t.Before(since) || !before.IsZero() && !t.Before(before) { + continue + } + e, err := runs.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var r Run + if err := jsonValue(e.Value, &r); err != nil { + return nil, fmt.Errorf("run %s: %w", k, err) + } + listed = append(listed, summary(r)) + } + sort.Slice(listed, func(i, j int) bool { + if !listed[i].Began.Equal(listed[j].Began) { + return listed[i].Began.After(listed[j].Began) + } + return listed[i].Node < listed[j].Node + }) + truncated := len(listed) > limit + if truncated { + listed = listed[:limit] + } + if listed == nil { + listed = []RunSummary{} + } + return map[string]any{"count": len(listed), "runs": listed, "truncated": truncated}, nil +} + +// jsonValue reads a state's value into its shape. +func jsonValue(raw []byte, v any) error { return json.Unmarshal(raw, v) } + +// upgradeState is a state of this module as the tools bundle reads it; a test replaces it. +var upgradeState = func(name string) Bucket { return stdio.State(name) } diff --git a/modules/pacman/module.json b/modules/pacman/module.json index 4e3d0084..e80c7836 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -71,6 +71,7 @@ "pacman_updates", "pacman_upgrade", "pacman_upgrade_health", + "pacman_upgrades", "pacman_orphans", "pacman_remove_orphans", "pacman_cache",