diff --git a/modules/kernel/README.md b/modules/kernel/README.md index 6acb6a37..266ef7f2 100644 --- a/modules/kernel/README.md +++ b/modules/kernel/README.md @@ -1,8 +1,9 @@ # kernel -The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one: -**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot -entries) is not built yet. +The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). It holds two things: +**transparent huge pages**, set per machine, and **the fallback kernel** (`linux-lts` with its headers) and +its place on the boot menu (novox/hq ADR 0508 §9 and §10). The default kernel's package, and microcode, are +not built yet. ## Why (novox/hq issue 746) @@ -35,16 +36,34 @@ under the new mode. A failed rotation is said in the unit's journal and does not rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing` or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit. +## The fallback kernel (novox/hq ADR 0508, issue 1841) + +Every machine keeps a second, older kernel to start from, so a kernel upgrade that does not boot is not the +end of the machine. The module declares the packages `linux-lts` and `linux-lts-headers` (so every DKMS module is +built for the fallback kernel too) after its boot-entries service, so the entry is in place before the kernel is. + +| | what | notes | +|---|---|---| +| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly | +| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched | +| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names, with `GRUB_TOP_LEVEL` naming the kernel the machine booted (`BOOT_IMAGE`; else the non-fallback kernel), since GRUB lists kernels in reverse version order and would otherwise make the fallback kernel the default; `/etc/default/grub` still overrides it when it sets one | +| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) | +| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks | +| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network | +| file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed | +| packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook | + ## Tools | tool | | answers | |---|---|---| +| `kernel_boot` | r | the loader (`systemd-boot`, `grub` or `none`), the menu's entries, the kernels in `/boot`, the entry booted (systemd-boot) or the kernel image booted (GRUB), whether the fallback kernel is on the menu with its kernel in `/boot`, and the room free on `/boot` in KiB | | `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) | ## Tests `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the -manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel diff --git a/modules/kernel/cmd/kernel-tools/boot.go b/modules/kernel/cmd/kernel-tools/boot.go new file mode 100644 index 00000000..10a895a1 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/boot.go @@ -0,0 +1,89 @@ +package main + +// kernel_boot: which loader booted the machine, its entries, the kernels in /boot, the entry it booted and +// whether the fallback kernel is on the menu (novox/hq ADR 0508 §9, issue 1841). Reads only. + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "syscall" + "unicode/utf16" +) + +const loaderSelected = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + +// BootState is what kernel_boot answers. +type BootState struct { + Loader string `json:"loader"` + Entries []string `json:"entries"` + Kernels []string `json:"kernels"` + Booted string `json:"booted"` + FallbackPresent bool `json:"fallback_present"` + BootFreeKiB int64 `json:"boot_free_kib"` +} + +var grubKernel = regexp.MustCompile(`(?m)^\s*linux\s+(?:/boot)?/(vmlinuz-\S+)`) +var grubTitle = regexp.MustCompile(`(?m)^\s*menuentry\s+'([^']*)'`) +var bootImage = regexp.MustCompile(`BOOT_IMAGE=(?:/boot)?/?(\S+)`) + +// Boot reads the machine, the loader the way the boot-entries script does: the loader's EFI variable means +// systemd-boot, a grub.cfg without it GRUB. +func (m *Machine) Boot() (*BootState, error) { + b := &BootState{Loader: "none", Entries: []string{}, Kernels: []string{}} + files, _ := filepath.Glob(m.path("boot/vmlinuz-*")) + for _, f := range files { + b.Kernels = append(b.Kernels, filepath.Base(f)) + } + sort.Strings(b.Kernels) + hasKernel := func(name string) bool { + for _, k := range b.Kernels { + if k == name { + return true + } + } + return false + } + var s syscall.Statfs_t + if err := syscall.Statfs(m.path("boot"), &s); err == nil { + b.BootFreeKiB = int64(s.Bavail) * int64(s.Bsize) / 1024 + } + if raw, err := os.ReadFile(m.path(loaderSelected)); err == nil { + b.Loader = "systemd-boot" + if len(raw) > 4 { + var u []uint16 + for i := 4; i+1 < len(raw); i += 2 { + u = append(u, uint16(raw[i])|uint16(raw[i+1])<<8) + } + b.Booted = strings.TrimRight(string(utf16.Decode(u)), "\x00") + } + files, _ := filepath.Glob(m.path("boot/loader/entries/*.conf")) + for _, f := range files { + b.Entries = append(b.Entries, filepath.Base(f)) + } + sort.Strings(b.Entries) + if raw, err := os.ReadFile(m.path("boot/loader/entries/mesh-linux-lts.conf")); err == nil { + b.FallbackPresent = strings.Contains(string(raw), "/vmlinuz-linux-lts") && hasKernel("vmlinuz-linux-lts") + } + return b, nil + } + if raw, err := os.ReadFile(m.path("boot/grub/grub.cfg")); err == nil { + b.Loader = "grub" + for _, t := range grubTitle.FindAllStringSubmatch(string(raw), -1) { + b.Entries = append(b.Entries, t[1]) + } + for _, k := range grubKernel.FindAllStringSubmatch(string(raw), -1) { + if k[1] == "vmlinuz-linux-lts" && hasKernel("vmlinuz-linux-lts") { + b.FallbackPresent = true + } + } + if cmd, err := m.read("proc/cmdline"); err == nil { + if i := bootImage.FindStringSubmatch(cmd); i != nil { + b.Booted = i[1] + } + } + } + return b, nil +} diff --git a/modules/kernel/cmd/kernel-tools/boot_test.go b/modules/kernel/cmd/kernel-tools/boot_test.go new file mode 100644 index 00000000..9c8d964f --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/boot_test.go @@ -0,0 +1,90 @@ +package main + +// kernel_boot over stand-in roots: a systemd-boot machine with and without the fallback entry, a GRUB machine +// with and without the fallback kernel in its menu, and a machine with neither (novox/hq ADR 0508 §9, issue 1841). + +import ( + "os" + "path/filepath" + "testing" +) + +const selectedVar = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + +func put(t *testing.T, root, p, content string) { + t.Helper() + full := filepath.Join(root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + write(t, full, content, 0o644) +} + +func bootOf(t *testing.T, root string) *BootState { + t.Helper() + got, err := (&Machine{Root: root}).Boot() + if err != nil { + t.Fatal(err) + } + return got +} + +func TestKernelBootReadsASystemdBootMachineWithItsFallbackEntry(t *testing.T) { + root := t.TempDir() + put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + put(t, root, "boot/loader/entries/arch.conf", defaultEntry) + put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\n") + put(t, root, "boot/vmlinuz-linux", "k") + put(t, root, "boot/vmlinuz-linux-lts", "k") + got := bootOf(t, root) + if got.Loader != "systemd-boot" || got.Booted != "arch.conf" || !got.FallbackPresent { + t.Fatalf("%+v", got) + } + if len(got.Entries) != 2 || got.Entries[0] != "arch.conf" || got.Entries[1] != "mesh-linux-lts.conf" { + t.Fatalf("entries: %v", got.Entries) + } + if len(got.Kernels) != 2 || got.Kernels[0] != "vmlinuz-linux" || got.Kernels[1] != "vmlinuz-linux-lts" { + t.Fatalf("kernels: %v", got.Kernels) + } + if got.BootFreeKiB <= 0 { + t.Fatalf("free: %d", got.BootFreeKiB) + } +} + +func TestKernelBootSaysASystemdBootMachineHasNoFallbackYet(t *testing.T) { + root := t.TempDir() + put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + put(t, root, "boot/loader/entries/arch.conf", defaultEntry) + put(t, root, "boot/vmlinuz-linux", "k") + // The entry file alone is not the fallback: the kernel it names must be in /boot. + put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "linux /vmlinuz-linux-lts\n") + if got := bootOf(t, root); got.FallbackPresent { + t.Fatalf("a fallback entry without its kernel: %+v", got) + } +} + +func TestKernelBootReadsAGrubMachine(t *testing.T) { + root := t.TempDir() + put(t, root, "boot/grub/grub.cfg", grubMenu) + put(t, root, "boot/vmlinuz-linux", "k") + put(t, root, "proc/cmdline", bootedCmdline) + got := bootOf(t, root) + if got.Loader != "grub" || got.FallbackPresent || got.Booted != "vmlinuz-linux" { + t.Fatalf("%+v", got) + } + if len(got.Entries) != 2 || got.Entries[0] != "Arch Linux" { + t.Fatalf("entries: %v", got.Entries) + } + put(t, root, "boot/vmlinuz-linux-lts", "k") + put(t, root, "boot/grub/grub.cfg", grubMenu+"menuentry 'Arch Linux LTS' {\n\tlinux\t/boot/vmlinuz-linux-lts root=x rw\n}\n") + if got := bootOf(t, root); !got.FallbackPresent { + t.Fatalf("%+v", got) + } +} + +func TestKernelBootSaysWhenThereIsNoLoader(t *testing.T) { + got := bootOf(t, t.TempDir()) + if got.Loader != "none" || got.FallbackPresent || len(got.Entries) != 0 { + t.Fatalf("%+v", got) + } +} diff --git a/modules/kernel/cmd/kernel-tools/bootentries_test.go b/modules/kernel/cmd/kernel-tools/bootentries_test.go new file mode 100644 index 00000000..19044bc3 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/bootentries_test.go @@ -0,0 +1,284 @@ +package main + +// The boot-entries script as the mesh writes it, run with sh over a stand-in root (novox/hq ADR 0508 §9 and +// §10, issue 1841). The fixtures are the shapes read from machines: a systemd-boot entries directory, a GRUB +// menu, a hook that copies every kernel image to the default kernel's name, and the command line a machine +// booted with. + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const bootedCmdline = "initrd=\\initramfs-linux.img BOOT_IMAGE=/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3 initrd=\\intel-ucode.img\n" + +const defaultEntry = "title Arch Linux\nlinux /vmlinuz-linux\ninitrd /intel-ucode.img\ninitrd /initramfs-linux.img\noptions root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet\n" + +const loaderConf = "default arch.conf\ntimeout 3\nconsole-mode max\n" + +// grubMenu is the part of a generated grub.cfg that names kernels, for a machine holding one kernel. +const grubMenu = `menuentry 'Arch Linux' --class arch { + set root='hd0,gpt2' + linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet + initrd /boot/initramfs-linux.img +} +menuentry 'Arch Linux, with fallback initramfs' --class arch { + linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet + initrd /boot/initramfs-linux-fallback.img +} +` + +// kernelCopyingHook copies every installed kernel image to the default kernel's name on the boot partition. +const kernelCopyingHook = `[Trigger] +Operation = Install +Operation = Upgrade +Type = Path +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Copying the kernel to the default name +When = PostTransaction +Exec = /usr/bin/sh -c '/usr/bin/cp /usr/lib/modules/$(uname -r)/vmlinuz /boot/vmlinuz-linux' +` + +// otherHook is a hook of the distribution's shape that has nothing to do with kernel images. +const otherHook = `[Trigger] +Operation = Upgrade +Type = Package +Target = pacman-mirrorlist + +[Action] +Description = Updating the mirror list +When = PostTransaction +Exec = /usr/bin/cp /etc/pacman.d/mirrorlist.pacnew /etc/pacman.d/mirrorlist +` + +type bootBed struct { + script, root, state, bin, mkconfig string + need string +} + +func newBootBed(t *testing.T) bootBed { + t.Helper() + dir := t.TempDir() + b := bootBed{script: filepath.Join(dir, "boot-entries"), root: filepath.Join(dir, "root"), state: filepath.Join(dir, "state"), + bin: filepath.Join(dir, "bin"), mkconfig: filepath.Join(dir, "mkconfig")} + content := manifest(t).resource(t, "boot-entries").text(t) + content = strings.ReplaceAll(content, "${dir:state}", b.state) + if strings.Contains(content, "${") { + t.Fatalf("a substitution the test does not make is left in the script") + } + for _, d := range []string{b.root + "/boot", b.root + "/proc", b.root + "/etc/pacman.d/hooks", b.state, b.bin} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write(t, b.script, content, 0o755) + write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644) + // the stand-in grub-mkconfig records the kernel it was told to put first, then its arguments. + write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"top=$GRUB_TOP_LEVEL $@\" >>"+b.mkconfig+"\n", 0o755) + return b +} + +func (b bootBed) put(t *testing.T, p, content string) { + t.Helper() + full := filepath.Join(b.root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + write(t, full, content, 0o644) +} + +func (b bootBed) systemdBoot(t *testing.T) { + b.put(t, "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f", "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00") + b.put(t, "boot/loader/loader.conf", loaderConf) + b.put(t, "boot/loader/entries/arch.conf", defaultEntry) + b.put(t, "boot/vmlinuz-linux", "k") +} + +func (b bootBed) run(t *testing.T) (string, error) { + t.Helper() + args := []string{b.script, "run", b.root, b.state} + if b.need != "" { + args = append(args, b.need) + } + cmd := exec.Command("sh", args...) + cmd.Env = append(os.Environ(), "PATH="+b.bin+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (b bootBed) read(t *testing.T, p string) string { + t.Helper() + raw, _ := os.ReadFile(filepath.Join(b.root, p)) + return string(raw) +} + +func TestASystemdBootMachineGetsTheFallbackEntryFromItsBootedCommandLine(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.put(t, "boot/intel-ucode.img", "u") + out, err := b.run(t) + if err != nil || !strings.Contains(out, "wrote") { + t.Fatalf("%v %s", err, out) + } + want := "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\ninitrd /intel-ucode.img\ninitrd /initramfs-linux-lts.img\n" + + "options root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3\n" + if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); got != want { + t.Fatalf("entry:\n%s\nwant:\n%s", got, want) + } + // The default entry and the loader's own file are as they were. + if b.read(t, "boot/loader/entries/arch.conf") != defaultEntry || b.read(t, "boot/loader/loader.conf") != loaderConf { + t.Fatalf("the default entry or loader.conf was touched") + } + if out, err := b.run(t); err != nil || !strings.Contains(out, "as it was") { + t.Fatalf("again: %v %s", err, out) + } +} + +func TestWithoutMicrocodeThereIsNoMicrocodeLine(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + if out, err := b.run(t); err != nil { + t.Fatalf("%v %s", err, out) + } + if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); strings.Contains(got, "ucode") || !strings.Contains(got, "initrd /initramfs-linux-lts.img") { + t.Fatalf("entry: %s", got) + } +} + +func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) { + b := newBootBed(t) + b.put(t, "boot/grub/grub.cfg", grubMenu) + b.put(t, "boot/vmlinuz-linux", "k") + if out, err := b.run(t); err != nil || !strings.Contains(out, "left as it is") { + t.Fatalf("same: %v %s", err, out) + } + if _, err := os.Stat(b.mkconfig); err == nil { + t.Fatalf("the menu was generated again with the same kernels") + } + b.put(t, "boot/vmlinuz-linux-lts", "k") + if out, err := b.run(t); err != nil || !strings.Contains(out, "generated again") { + t.Fatalf("changed: %v %s", err, out) + } + raw, _ := os.ReadFile(b.mkconfig) + if string(raw) != "top=/boot/vmlinuz-linux -o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" { + t.Fatalf("grub-mkconfig was asked %q", raw) + } +} + +// ADR 0508 §9: GRUB's menu lists kernels in reverse version order, so linux-lts would come first and become the +// default entry (GRUB_DEFAULT=0); the kernel the machine booted is put first, and the fallback kernel never. +func TestGrubKeepsTheBootedKernelFirstAndNeverTheFallback(t *testing.T) { + for name, c := range map[string]struct{ cmdline, want string }{ + "booted from the default kernel": {"BOOT_IMAGE=/vmlinuz-linux root=UUID=x rw\n", "top=/boot/vmlinuz-linux"}, + "booted from a grub device path": {"BOOT_IMAGE=(hd0,msdos1)/vmlinuz-linux-zen root=UUID=x rw\n", "top=/boot/vmlinuz-linux-zen"}, + "booted from the fallback kernel": {"BOOT_IMAGE=/vmlinuz-linux-lts root=UUID=x rw\n", "top=/boot/vmlinuz-linux"}, + "no image on the command line": {"root=UUID=x rw\n", "top=/boot/vmlinuz-linux"}, + "an image no longer in /boot": {"BOOT_IMAGE=/vmlinuz-linux-gone root=UUID=x rw\n", "top=/boot/vmlinuz-linux"}, + } { + b := newBootBed(t) + b.put(t, "boot/grub/grub.cfg", grubMenu) + b.put(t, "proc/cmdline", c.cmdline) + for _, k := range []string{"vmlinuz-linux", "vmlinuz-linux-lts", "vmlinuz-linux-zen"} { + b.put(t, "boot/"+k, "k") + } + if out, err := b.run(t); err != nil { + t.Fatalf("%s: %v %s", name, err, out) + } + raw, _ := os.ReadFile(b.mkconfig) + if !strings.HasPrefix(string(raw), c.want+" -o ") { + t.Errorf("%s: grub-mkconfig was asked %q, want %s first", name, raw, c.want) + } + } +} + +func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) { + b := newBootBed(t) + out, err := b.run(t) + if err == nil || !strings.Contains(out, "no boot loader found") { + t.Fatalf("%v %s", err, out) + } +} + +func TestABootPartitionWithoutRoomFailsBeforeAnythingIsWritten(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.need = "999999999999" + out, err := b.run(t) + if err == nil || !strings.Contains(out, "free room there before the kernel is installed") { + t.Fatalf("%v %s", err, out) + } + if b.read(t, "boot/loader/entries/mesh-linux-lts.conf") != "" { + t.Fatalf("an entry was written") + } + // Once the fallback kernel is in place the room is not asked for again. + b.put(t, "boot/vmlinuz-linux-lts", "k") + if out, err := b.run(t); err != nil { + t.Fatalf("%v %s", err, out) + } +} + +func TestAHookThatCopiesKernelsToOneFixedNameIsMovedIntoTheStateAndNamed(t *testing.T) { + b := newBootBed(t) + b.systemdBoot(t) + b.put(t, "etc/pacman.d/hooks/60-copy-kernel.hook", kernelCopyingHook) + b.put(t, "etc/pacman.d/hooks/70-mirrorlist.hook", otherHook) + b.put(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", "[Trigger]\nTarget = usr/lib/modules/*/vmlinuz\n[Action]\nExec = /bin/cp x /boot/y\n") + out, err := b.run(t) + if err != nil || !strings.Contains(out, "taken over: the pacman hook 60-copy-kernel.hook") { + t.Fatalf("%v %s", err, out) + } + if b.read(t, "etc/pacman.d/hooks/60-copy-kernel.hook") != "" { + t.Fatalf("the copying hook is still in the hook directory") + } + if raw, _ := os.ReadFile(filepath.Join(b.state, "taken-over", "60-copy-kernel.hook")); string(raw) != kernelCopyingHook { + t.Fatalf("moved hook: %q", raw) + } + if b.read(t, "etc/pacman.d/hooks/70-mirrorlist.hook") != otherHook || b.read(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook") == "" { + t.Fatalf("a hook that copies no kernel was moved") + } +} + +func TestTheBootEntriesUnitAndHookAreTheModules(t *testing.T) { + m := manifest(t) + unit := m.resource(t, "boot-unit").text(t) + for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/boot-entries run", "ProtectSystem=strict", + "ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks", "NoNewPrivileges=yes", "WantedBy=multi-user.target"} { + if !strings.Contains(unit, line+"\n") { + t.Errorf("the unit lacks %q", line) + } + } + hook := m.resource(t, "boot-hook") + text := hook.text(t) + if hook["path"] != "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook" || !strings.Contains(text, "Target = usr/lib/modules/*/vmlinuz\n") || + !strings.Contains(text, "When = PostTransaction\n") || !strings.Contains(text, "Exec = ${dir:state}/boot-entries run\n") { + t.Fatalf("the hook: %v\n%s", hook["path"], text) + } + apply := m.resource(t, "boot-apply") + if apply["unit"] != "mesh-kernel-boot-entries.service" || apply["state"] != "running" || apply["boot"] != "enabled" { + t.Fatalf("apply: %v", apply) + } +} + +// The fallback kernel and its headers are declared after the boot-entries service, so the entry is placed +// before the kernel is installed and every DKMS module is built for it (ADR 0508 §9). +func TestTheFallbackKernelAndItsHeadersAreDeclaredAfterTheBootEntriesUnit(t *testing.T) { + m := manifest(t) + at := map[string]int{} + for i, r := range m.Resources { + at[r["id"].(string)] = i + } + for id, pkg := range map[string]string{"linux-lts": "linux-lts", "linux-lts-headers": "linux-lts-headers"} { + r := m.resource(t, id) + if r["type"] != "package" || r["package"] != pkg || r["absent"] == true { + t.Errorf("%s: %v", id, r) + } + if at[id] < at["boot-apply"] || at[id] < at["boot-hook"] { + t.Errorf("%s is declared before the boot-entries unit and hook", id) + } + } +} diff --git a/modules/kernel/cmd/kernel-tools/main.go b/modules/kernel/cmd/kernel-tools/main.go index ec15c916..93cdece8 100644 --- a/modules/kernel/cmd/kernel-tools/main.go +++ b/modules/kernel/cmd/kernel-tools/main.go @@ -46,5 +46,15 @@ func tools(m *Machine) []stdio.Tool { return m.HugePages(n) }, }, + { + Name: "kernel_boot", + Description: "How this machine boots and whether the fallback kernel is on its menu (novox/hq ADR 0508 §9): the loader that booted it " + + "(systemd-boot, read from the EFI variable it sets, grub, or none), the menu's entries, the kernels in /boot, the entry " + + "the machine booted (systemd-boot) or the kernel image it booted (grub), whether the fallback kernel linux-lts is on " + + "the menu with its kernel in /boot, and the room free on /boot in KiB (the fallback kernel needs about 72 MB). " + + "Replaces ls /boot, bootctl list, reading grub.cfg and df /boot. (r)", + Input: schema(map[string]any{}), + Run: func(args map[string]any) (any, error) { return m.Boot() }, + }, } } diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index a3594f35..aad91fc6 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { for _, r := range m.Resources { ids = append(ids, r["id"].(string)) } - if strings.Join(ids, " ") != "state huge-pages unit apply" { + if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" { t.Fatalf("order: %v", ids) } script := m.resource(t, "huge-pages") diff --git a/modules/kernel/files/90-mesh-kernel-boot-entries.hook b/modules/kernel/files/90-mesh-kernel-boot-entries.hook new file mode 100644 index 00000000..d671caaf --- /dev/null +++ b/modules/kernel/files/90-mesh-kernel-boot-entries.hook @@ -0,0 +1,13 @@ +# Module kernel, novox/hq ADR 0508 §9, issue 1841. After a kernel image is installed, upgraded or removed, the +# fallback kernel's boot entry (or the GRUB menu) is placed again. Written by the mesh; change the module. +[Trigger] +Operation = Install +Operation = Upgrade +Operation = Remove +Type = Path +Target = usr/lib/modules/*/vmlinuz + +[Action] +Description = Placing the fallback kernel on the boot menu +When = PostTransaction +Exec = ${dir:state}/boot-entries run diff --git a/modules/kernel/files/boot-entries.sh b/modules/kernel/files/boot-entries.sh new file mode 100644 index 00000000..ed174906 --- /dev/null +++ b/modules/kernel/files/boot-entries.sh @@ -0,0 +1,133 @@ +#!/bin/sh +# The fallback kernel's place on this machine's boot menu (module kernel, novox/hq ADR 0508 §9 and §10, +# issue 1841). Written by the mesh and run as root by mesh-kernel-boot-entries.service at boot and when the +# module is applied, and by the pacman hook 90-mesh-kernel-boot-entries.hook after a kernel is installed, +# upgraded or removed. Replaced on every push; change the module, never this file. +# +# boot-entries run [ [ []]] +# +# The three are arguments only so the module's tests can run this against a stand-in tree. +# +# It reads from the machine which loader booted it, never from a setting: +# systemd-boot the EFI variable LoaderEntrySelected is present: the entry mesh-linux-lts.conf is written +# from the command line the machine booted with. +# GRUB /boot/grub/grub.cfg exists and that variable does not: the menu is generated again, only +# when the set of kernels in /boot differs from the menu's. +# neither it fails loudly. The default entry is never touched. +# And a pacman hook of someone else's that copies kernel images to one fixed name in /boot is moved into this +# module's state first (ADR 0508 §10): it would put the fallback kernel in the default kernel's place. +set -eu + +root=/ +state='${dir:state}' +[ $# -ge 2 ] && root=$(printf '%s' "$2" | sed 's|/$||') +[ $# -ge 3 ] && state=$3 +boot=$root/boot +hooks=$root/etc/pacman.d/hooks +efivar=$root/sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f +ours=90-mesh-kernel-boot-entries.hook +# The fallback kernel and its initramfs are about 72 MB in /boot (issue 1836): less room fails the unit +# before a package writes half of them. +need_kib=98304 +[ $# -ge 4 ] && need_kib=$4 + +[ $# -ge 1 ] && [ "$1" = run ] || { + echo "usage: boot-entries run [ [ []]]" >&2 + exit 2 +} + +# A hook that is not ours, triggered by kernel images, whose action copies to one fixed name in /boot. +copies_kernels() { + grep -Eqi '^[[:space:]]*Target[[:space:]]*=.*(vmlinuz|usr/lib/modules)' "$1" && + grep -Eq '^[[:space:]]*Exec[[:space:]]*=.*(cp|install|ln|rsync)[[:space:]].*/boot/[A-Za-z0-9._+-]+([^A-Za-z0-9._+/%$*-]|$)' "$1" +} + +if [ -d "$hooks" ]; then + for hook in "$hooks"/*.hook; do + [ -f "$hook" ] || continue + [ "$(basename "$hook")" = "$ours" ] && continue + copies_kernels "$hook" || continue + mkdir -p "$state/taken-over" + mv -f "$hook" "$state/taken-over/$(basename "$hook")" + echo "taken over: the pacman hook $(basename "$hook") copied kernel images to one fixed name in /boot; moved to $state/taken-over/ so each kernel keeps its own package's name" + done +fi + +if [ -e "$efivar" ]; then + loader=systemd-boot +elif [ -f "$boot/grub/grub.cfg" ]; then + loader=grub +else + echo "no boot loader found: the machine was not booted by systemd-boot (no LoaderEntrySelected variable) and there is no $boot/grub/grub.cfg; the fallback kernel has no menu to be placed on" >&2 + exit 1 +fi + +# Room first, only while the fallback kernel is not in /boot yet. +if [ ! -e "$boot/vmlinuz-linux-lts" ]; then + free=$(df -Pk "$boot" | awk 'NR==2 {print $4}') + [ -n "$free" ] || free=0 + if [ "$free" -lt "$need_kib" ]; then + echo "$boot has $free KiB free and the fallback kernel needs $need_kib KiB: free room there before the kernel is installed" >&2 + exit 1 + fi +fi + +case "$loader" in +systemd-boot) + entries=$boot/loader/entries + cmdline=$(sed -e 's/BOOT_IMAGE=[^ ]*//g' -e 's/initrd=[^ ]*//g' "$root/proc/cmdline" | tr -s ' \n' ' ' | sed -e 's/^ //' -e 's/ $//') + [ -n "$cmdline" ] || { + echo "$root/proc/cmdline holds no command line to boot the fallback kernel with" >&2 + exit 1 + } + want=$( + echo "title Arch Linux (linux-lts, fallback)" + echo "linux /vmlinuz-linux-lts" + for ucode in intel-ucode.img amd-ucode.img; do + [ -e "$boot/$ucode" ] && echo "initrd /$ucode" + done + echo "initrd /initramfs-linux-lts.img" + echo "options $cmdline" + ) + mkdir -p "$entries" + if [ "$(cat "$entries/mesh-linux-lts.conf" 2>/dev/null || true)" = "$want" ]; then + echo "systemd-boot: mesh-linux-lts.conf, as it was" + else + printf '%s\n' "$want" >"$entries/mesh-linux-lts.conf" + echo "systemd-boot: wrote $entries/mesh-linux-lts.conf from the command line the machine booted with; the default entry is as it was" + fi + ;; +grub) + cfg=$boot/grub/grub.cfg + menu=$(sed -n 's|^[[:space:]]*linux[[:space:]]*\(/boot\)\{0,1\}/\(vmlinuz-[^[:space:]]*\).*|\2|p' "$cfg" | sort -u) + have=$(cd "$boot" && ls -d vmlinuz-* 2>/dev/null | sort -u || true) + if [ "$menu" = "$have" ]; then + echo "grub: the menu lists the kernels in /boot, left as it is" + else + # GRUB lists kernels in reverse version order, so the fallback kernel would come first and become the + # default entry: the kernel the machine booted is put first (GRUB_TOP_LEVEL; /etc/default/grub still + # overrides it when it sets one), the fallback kernel never. + top=$(tr ' ' '\n' <"$root/proc/cmdline" | sed -n 's|^BOOT_IMAGE=\(.*\)$|\1|p' | head -n 1) + top=$(printf '%s' "$top" | sed 's|.*/||') + case "$top" in + vmlinuz-linux-lts | "") top= ;; + esac + [ -n "$top" ] && [ -e "$boot/$top" ] || top= + if [ -z "$top" ]; then + if [ -e "$boot/vmlinuz-linux" ]; then + top=vmlinuz-linux + else + top=$(printf '%s\n' $have | grep -v '^vmlinuz-linux-lts$' | head -n 1 || true) + fi + fi + if [ -n "$top" ]; then + GRUB_TOP_LEVEL=/boot/$top grub-mkconfig -o "$cfg" + else + grub-mkconfig -o "$cfg" + fi + first=$top + [ -n "$first" ] || first="the menu's own order" + echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again with $first first; the default entry is as it was" + fi + ;; +esac diff --git a/modules/kernel/files/mesh-kernel-boot-entries.service b/modules/kernel/files/mesh-kernel-boot-entries.service new file mode 100644 index 00000000..56cfa319 --- /dev/null +++ b/modules/kernel/files/mesh-kernel-boot-entries.service @@ -0,0 +1,22 @@ +[Unit] +Description=The fallback kernel's place on this machine's boot menu +# Module kernel, novox/hq ADR 0508 §9, issue 1841. Starting this unit reads which loader booted the machine and +# places the fallback kernel's boot entry (systemd-boot) or generates the menu again (GRUB); it fails, and says +# why, when there is no loader or the boot partition has no room. The kernel packages are declared after this +# unit, so the entry is in place before they are installed. +After=local-fs.target +Wants=local-fs.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=${dir:state}/boot-entries run +# It writes the boot partition, the pacman hook directory and its own state, and nothing else. +ProtectSystem=strict +ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks +ProtectHome=yes +PrivateNetwork=yes +NoNewPrivileges=yes + +[Install] +WantedBy=multi-user.target diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 67d32156..2a63563e 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -13,7 +13,8 @@ } }, "tools": [ - "kernel_huge_pages" + "kernel_huge_pages", + "kernel_boot" ], "resources": [ { @@ -53,6 +54,52 @@ "health": { "kind": "unit" } + }, + { + "id": "boot-entries", + "type": "file", + "path": "${dir:state}/boot-entries", + "mode": "0755", + "trusted": true, + "template": "files/boot-entries.sh" + }, + { + "id": "boot-unit", + "type": "file", + "path": "/etc/systemd/system/mesh-kernel-boot-entries.service", + "mode": "0644", + "template": "files/mesh-kernel-boot-entries.service" + }, + { + "id": "boot-hook", + "type": "file", + "path": "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", + "mode": "0644", + "template": "files/90-mesh-kernel-boot-entries.hook" + }, + { + "id": "boot-apply", + "type": "service", + "unit": "mesh-kernel-boot-entries.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "boot-unit", + "boot-entries" + ], + "health": { + "kind": "unit" + } + }, + { + "id": "linux-lts", + "type": "package", + "package": "linux-lts" + }, + { + "id": "linux-lts-headers", + "type": "package", + "package": "linux-lts-headers" } ], "build": { diff --git a/modules/pacman/README.md b/modules/pacman/README.md index 4ba4969a..fd400323 100644 --- a/modules/pacman/README.md +++ b/modules/pacman/README.md @@ -10,10 +10,10 @@ that seat (ADR 0207). (ADR 0207). - **`/etc/pacman.conf`, whole.** A block cannot be added to `[options]` by appending: anything added at the end of the file lands in the last repository's section. So the module owns the file: - - The repositories are the union of what the four machines had enabled on 2026-10-04: `core`, - `extra` and `multilib`. All four had all three. + - The repositories are the union of what the mesh's machines had enabled on 2026-10-04: `core`, + `extra` and `multilib`. All had all three. - The options are the distribution's defaults, plus `Color`, `ParallelDownloads = 5`, - `VerbosePkgLists`, and `DownloadUser = alpm` (pacman 7; the `alpm` user exists on all four). + `VerbosePkgLists`, and `DownloadUser = alpm` (pacman 7; the `alpm` user exists on every machine). - The manifest test runs `pacman-conf` on the rendered file and checks the repository list and the options as pacman reads them. It skips that check where `pacman-conf` is absent. - The host keeps the machine's previous file once, the first time it writes over it (ADR 0102). @@ -23,23 +23,44 @@ that seat (ADR 0207). list stays reflector's to write, not the mesh's. - **Cache cleaning.** The `pacman-contrib` package and `paccache.timer` running and enabled. Each week it keeps the last three versions of each package. -- **The weekly upgrade (ADR 0477 clause 5, issue 1574).** `mesh-pacman-weekly-upgrade.timer` and its - oneshot service, placed whole in `/etc/systemd/system`. The timer fires every Saturday at 04:00 in the - node's local time, the same moment on every node (no random delay), and is `Persistent=true`: a node that - was off or asleep then upgrades at its next start. The service runs `/usr/bin/pacman -Syu --noconfirm`, the - same arguments `pacman_upgrade` starts (`UpgradeArgs` in the code; a manifest test holds the two together). - No other moment upgrades but a hand `pacman_upgrade`, and the mesh restarts no machine for an upgrade: a new - kernel waits for the next restart. - - **The outcome is the module's health.** The timer's resource asks `pacman_upgrade_health`: unhealthy, - naming the exit and the last `error:` line of the journal, when the last weekly upgrade failed; healthy - while it runs, after it passed, and before it ever ran. The controller raises the module's condition for - the node from it. A failed service stays failed until - the next run, so the condition stands until an upgrade passes; a hand `pacman_upgrade` does not clear - it. A restart of the machine clears the failed state and with it the condition, and the next Saturday - runs again. A run that goes on longer than two hours is unhealthy too, naming when it started and the - last line of its journal: it is hung and holds pacman's database lock. The service has no start - timeout on purpose, because killing pacman in the middle of a transaction is worse; the hung run is - said, not killed. +- **The package view on the bus (ADR 0507).** A process of its own, the keeper (`pacman-tools keeper`, account + `pacman-keeper`, its own bus account), keeps the node's packages in the state `installed` and what an upgrade + would change in `updates`, written only on change: at its start, whenever pacman's database lock is released + (an inotify watch), when reflector rewrites the mirror list, and once a day. Each change of the packages is + `package.changed`. A list over the bus's cap on a value is not written: the key keeps its last value and + `.not-current` says the size and since when, which the health says too. `pacman_packages`, + `pacman_updates`, `pacman_installed` and `pacman_foreign` answer every node from these states, and what no + module declares from the controller's `declared-packages`, without asking a machine. +- **Upgrades as runs, one node at a time (ADR 0508).** Every full upgrade and every rollback is a run in the + state `runs` at `.`, written at each step and emitted as it: the package record taken and read back + (`records`, kept `record-days`), the DKMS gate before the transaction (a kernel an installed DKMS module's own + `Linux-Maximum` or `BUILD_EXCLUSIVE_KERNEL` excludes is held with its headers, `upgrade.kernel-held`), the + transaction as a unit of its own, its end, and the after-upgrade check (`.pacnew`/`.pacsave` files, the news + since the previous upgrade, units newly failed, a restart needed, processes still mapping a replaced file by + unit and executable, DKMS modules not built for an installed kernel). pacman's log is read once into `runs` + as history. + - **The round.** The nodes upgrade in the order of the mesh-level key `upgrade-order` (a trusted list, + `predefined.json`; the module's config value of the same name falls back to it). On Saturday at 04:00 local + time the first node of the order opens the week's round in `rounds`; each node's keeper watches `rounds` and + `runs` and starts its own run when the node before it passed its after-upgrade check. A failed run or check + stops the round there; only the operator passes a node (`pacman_round` with `pass` and a reason asks on the + operator's channel, and the round goes on only on the operator's Approve). With no order set no round opens. + - **Acts are asks.** `pacman_upgrade` and `pacman_rollback` with `apply` write an ask in `asks` at + `.`, which that node's keeper alone takes. + - **The root helper (ADR 0510).** The keeper never runs as root. The acts that need root go through a second + process of the same build, `pacman-root` (`pacman-tools root`), running as root with no bus account, on a + unix socket in `${dir:state}/root/` that only the keeper's account may open, the peer's account checked on + every connection. It serves four verbs and refuses everything else: start a run's full upgrade as the unit + `mesh-pacman-upgrade-` (leaving out held packages), start a rollback's unit with the plan in the request + (packages from the cache or the Arch Linux Archive, the dated repository's file found by name), read which + processes map a replaced file, and read a transaction unit's journal. No sudo rule is placed. + - **The health is read from the bus.** The keeper's health check (`pacman_upgrade_health`) is unhealthy while + no `upgrade-order` is set; while the node's latest run was refused, failed or did not pass its check (until + a later run passes); while its transaction goes on past two hours (hung, said and not killed); while a + kernel is held; and while the round stands still at this node or at the node it waits for, naming it and + since when. A node that slept through a run is judged by what the bus holds, not by a unit's age (issue + 1579). The weekly timer of ADR 0477 §5 and its service are dropped from the manifest, so the node-engine + removes them. ## What it improves @@ -62,21 +83,27 @@ that seat (ADR 0207). |---|---|---| | `pacman_search` | r | `pacman -Ss`: repository, name, version, groups, installed and at which version, description | | `pacman_info` | r | `-Qi`, or `-Si` when not installed, with lists as lists | -| `pacman_installed` | r | every package with version, explicit or dependency, foreign; filters and totals | +| `pacman_installed` | r | every package with version, explicit or dependency, foreign; filters and totals, from `installed` on the bus | +| `pacman_packages` | r | every node's packages from the bus: one package everywhere, differing versions, packages on some nodes only, each node's explicit packages no module declares, foreign apart, nodes not current | | `pacman_owns` | r | which package owns a path, or `owned: false` | | `pacman_files` | r | what a package placed, bounded | -| `pacman_updates` | r | `checkupdates`: what a full upgrade would change, never setting up a partial upgrade | -| `pacman_upgrade` | a | starts `pacman -Syu --noconfirm` (sudo -n) as a transient unit that outlives the call; answers the unit and the news since the last upgrade; given the unit, how it went | -| `pacman_upgrade_health` | r | whether the weekly upgrade last passed: the timer's health check (see the weekly upgrade above) | +| `pacman_updates` | r | what a full upgrade would change, for one node or all, from `updates` on the bus | +| `pacman_upgrade` | a | asks a node's keeper for a full upgrade as a run (an ask in `asks`); answers the ask and the news since the last upgrade; given a transaction unit, how it goes | +| `pacman_upgrade_health` | r | the keeper's health check, read from the bus (see the health above) | +| `pacman_upgrades` | r | every run of every node from `runs`: narrowed to a node and days, or one run whole | +| `pacman_round` | r/a | where every node stands in the newest rounds; `pass` with a reason asks the operator to let a round go on past a node | +| `pacman_rollback` | r/a | what returning a node to a run's record would change, each package with its source; `apply` asks the node's keeper to run it | | `pacman_orphans` | r | `pacman -Qdt` | | `pacman_remove_orphans` | a | `pacman -Rs` on named orphans, or all of them, as a unit of its own; a name that is not an orphan is refused | | `pacman_cache` | r/a | size, interrupted downloads, what paccache would free keeping N; `clean: true` removes them | | `pacman_history` | r | `/var/log/pacman.log`: installs, upgrades, downgrades, reinstalls and removals since a day, and the last full upgrade | | `pacman_mirrors` | r/a | the list, its generator and age, reflector's options, timer and last run; `refresh: true` starts reflector | -| `pacman_foreign` | r | `pacman -Qm` | +| `pacman_foreign` | r | the foreign packages, from `installed` on the bus | | `pacman_news` | r | the distribution's news since the last full upgrade (or a day), over https; no network is `reachable: false` | | `pacman_config` | r | `pacman-conf`: options, repositories, and whether `/etc/pacman.conf` is the module's | A transaction never runs as the tool's own child. An upgrade takes longer than the 20 s a call may take, and a pacman killed mid-transaction leaves a half-upgraded machine and a lock. So a transaction -runs as a transient unit (`systemd-run`, named `mesh-pacman-…`), and its log is read from the journal. +runs as a transient unit (`systemd-run`, named `mesh-pacman-…`), and its log is read from the journal. A +run's transaction is started by the root helper; removing orphans and cleaning the cache still go through +`sudo -n` from the tools bundle. diff --git a/modules/pacman/cmd/pacman-tools/acts.go b/modules/pacman/cmd/pacman-tools/acts.go index 146d6d5e..ef3682ef 100644 --- a/modules/pacman/cmd/pacman-tools/acts.go +++ b/modules/pacman/cmd/pacman-tools/acts.go @@ -55,30 +55,11 @@ func (m *Machine) journal(unit string, n int) []string { return lines(out) } -// UpgradeArgs is the full system upgrade, in one place: pacman_upgrade starts pacman with these arguments as -// a transient unit, and the weekly upgrade's service (files/mesh-pacman-weekly-upgrade.service) runs -// pacman with the same ones; a test holds the two together (issue 1574, ADR 0477 clause 5). +// UpgradeArgs is the full system upgrade, in one place: the root helper starts a run's transaction with these +// arguments (ADR 0510), and the packages a run holds back are added to them as --ignore. var UpgradeArgs = []string{"-Syu", "--noconfirm"} -// Upgrade starts a full system upgrade as a transient unit and answers at once, with the -// distribution's news since the last upgrade; or, given a unit it started, answers how it went. -func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) { - if unit != "" { - return m.UpgradeStatus(unit, n) - } - news := m.News("") - started, _, err := m.transaction("upgrade", false, UpgradeArgs...) - if err != nil { - return nil, err - } - return map[string]any{ - "started": started, - "follow": "call pacman_upgrade with this unit to read how it goes", - "news": news, - }, nil -} - -var unitName = regexp.MustCompile(`^` + unitPrefix + `[a-z-]+-[0-9]+$`) +var unitName = regexp.MustCompile(`^` + unitPrefix + `[a-z-]+-[0-9a-z]+$`) // UpgradeStatus is a transaction unit's state and the tail of what it logged. func (m *Machine) UpgradeStatus(unit string, n int) (map[string]any, error) { diff --git a/modules/pacman/cmd/pacman-tools/aftercheck.go b/modules/pacman/cmd/pacman-tools/aftercheck.go new file mode 100644 index 00000000..d7a4c05b --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/aftercheck.go @@ -0,0 +1,151 @@ +package main + +// The after-upgrade check (novox/hq ADR 0508 §4), written to the run as its last step: the .pacnew and .pacsave +// files the transaction left, the distribution's news since the node's previous upgrade, the units failed now +// that were not failed in the record, and whether a restart is needed (the running kernel is no longer the one +// its package installed). It passes when the transaction succeeded and no unit newly failed (and, with the +// second part, every DKMS module is built); the rest is said and does not stop the round. The mesh restarts no +// machine (ADR 0477 §5). + +import ( + "sort" + "strings" + "time" +) + +// AfterCheck is the after-upgrade check of a run whose transaction ended, against the record taken before it +// (nil when there is none: every unit failed now is then said as newly failed). +func (u *Upgrades) AfterCheck(run Run, rec *Record) Check { + c := Check{Pacnew: []string{}, Pacsave: []string{}, News: []NewsTitle{}, Mapped: []Mapped{}, NewlyFailed: []string{}, Unbuilt: []DKMSBuild{}} + succeeded := transactionSucceeded(run) + var started time.Time + for _, s := range run.Steps { + if s.Step == StepStarted { + started = s.At + } + } + if !started.IsZero() { + if b := u.transactionBlock(run.Kind, started); b != nil { + c.Pacnew = append(c.Pacnew, b.Pacnew...) + c.Pacsave = append(c.Pacsave, b.Pacsave...) + } + } + c.News, c.NewsError = u.newsSince(u.previousUpgrade(run)) + healthy := true + if failed, err := u.Machine.FailedUnits(); err != nil { + healthy = false + c.NewlyFailed = append(c.NewlyFailed, "(the failed units could not be read: "+err.Error()+")") + } else { + before := map[string]bool{} + if rec != nil { + for _, f := range rec.FailedUnits { + before[f] = true + } + } + for _, f := range failed { + if !before[f] { + c.NewlyFailed = append(c.NewlyFailed, f) + } + } + } + c.RunningKernel, c.InstalledKernel, c.RestartNeeded = u.restartNeeded() + c.Passed = succeeded && healthy && len(c.NewlyFailed) == 0 + u.afterCheckII(&c) + return c +} + +// previousUpgrade is when the node's upgrade before this run started: its previous run with a transaction. +func (u *Upgrades) previousUpgrade(run Run) time.Time { + runs, err := u.ownRuns() + if err != nil { + return time.Time{} + } + var last time.Time + for _, r := range runs { + if r.Run.Run >= run.Run || r.Kind != RunUpgrade { + continue + } + for _, s := range r.Steps { + if s.Step == StepStarted && s.At.After(last) { + last = s.At + } + } + } + return last +} + +// newsSince is the distribution's news published after a time, titles only; or why it could not be read. +func (u *Upgrades) newsSince(since time.Time) ([]NewsTitle, string) { + body, err := fetch(NewsFeed) + if err != nil { + return []NewsTitle{}, err.Error() + } + items, err := ParseNews(body, since) + if err != nil { + return []NewsTitle{}, "the feed could not be read: " + err.Error() + } + titles := []NewsTitle{} + for _, it := range items { + titles = append(titles, NewsTitle{Title: it.Title, Link: it.Link, Published: it.Published}) + } + return titles, "" +} + +// kernelTag is a release's flavour, its letters alone: `7.2.8-arch1-2` is arch, `6.18.54-2-lts` is lts. +func kernelTag(release string) string { + return strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' { + return r + } + return -1 + }, release) +} + +// installedKernels are the releases the installed kernel packages install, sorted. +func (m *Machine) installedKernels() ([]string, error) { + out, err := m.Out("pacman", "-Qq") + if err != nil { + return nil, err + } + var releases []string + for _, name := range lines(out) { + name = strings.TrimSpace(name) + if name != "linux" && !strings.HasPrefix(name, "linux-") { + continue + } + release, kernel, err := m.kernelRelease(name) + if err != nil { + return nil, err + } + if kernel { + releases = append(releases, release) + } + } + sort.Strings(releases) + return releases, nil +} + +// restartNeeded compares the running kernel with what its package installs now: when no installed kernel is the +// running one, the one of its flavour is what a restart would boot. +func (u *Upgrades) restartNeeded() (running, installed string, needed bool) { + raw, err := u.Machine.ReadFile(OSRelease) + if err != nil { + return "(the running kernel could not be read: " + err.Error() + ")", "", false + } + running = strings.TrimSpace(string(raw)) + kernels, err := u.Machine.installedKernels() + if err != nil { + return running, "(the installed kernels could not be read: " + err.Error() + ")", false + } + for _, k := range kernels { + if k == running { + return running, k, false + } + } + for _, k := range kernels { + if kernelTag(k) == kernelTag(running) { + return running, k, true + } + } + return running, "", true +} diff --git a/modules/pacman/cmd/pacman-tools/aftercheck_test.go b/modules/pacman/cmd/pacman-tools/aftercheck_test.go new file mode 100644 index 00000000..5410f8bb --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/aftercheck_test.go @@ -0,0 +1,121 @@ +package main + +// The after-upgrade check, its first part (novox/hq ADR 0508 §4; acceptance 1841/AC5): .pacnew and .pacsave +// files from the transaction's log lines, the news since the previous upgrade, units newly failed against the +// record, and a restart needed. + +import ( + "errors" + "strings" + "testing" + "time" +) + +const checkLog = "[2026-10-04T12:00:01+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + + "[2026-10-04T12:00:02+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-04T12:00:09+0000] [ALPM] transaction started\n" + + "[2026-10-04T12:00:10+0000] [ALPM] upgraded pacman-mirrorlist (20260901-1 -> 20261001-1)\n" + + "[2026-10-04T12:00:10+0000] [ALPM] warning: /etc/pacman.d/mirrorlist installed as /etc/pacman.d/mirrorlist.pacnew\n" + + "[2026-10-04T12:00:10+0000] [ALPM] warning: /etc/lemurs/config.toml saved as /etc/lemurs/config.toml.pacsave\n" + + "[2026-10-04T12:00:10+0000] [ALPM] removed lemurs-git (0.4.0.r17.g7151336-2)\n" + + "[2026-10-04T12:00:11+0000] [ALPM] transaction completed\n" + + "[2026-10-04T12:05:00+0000] [PACMAN] Running 'pacman -S --noconfirm made-up'\n" + + "[2026-10-04T12:05:01+0000] [ALPM] transaction started\n" + + "[2026-10-04T12:05:01+0000] [ALPM] warning: /etc/made-up.conf installed as /etc/made-up.conf.pacnew\n" + + "[2026-10-04T12:05:02+0000] [ALPM] transaction completed\n" + +const newsFeed = ` +A manual interventionhttps://example.org/news/2Fri, 02 Oct 2026 10:00:00 +0000x +An older posthttps://example.org/news/1Mon, 21 Sep 2026 10:00:00 +0000y +` + +// checkMachine is a machine after an upgrade: these units failed now, this kernel running, and the linux +// package now installing linuxNow. +func checkMachine(failed, linuxNow string) *Machine { + m := machine(byLine(map[string]Ran{ + "systemctl list-units --state=failed --plain --no-legend --no-pager": {Stdout: failed}, + "pacman -Qq": {Stdout: "bash\nlinux\nlinux-firmware\nlinux-lts\n"}, + "dkms status": {Status: 127, Err: "ENOENT"}, + "pacman -Qlq -- linux": {Stdout: "/usr/lib/modules/" + linuxNow + "/vmlinuz\n"}, + "pacman -Qlq -- linux-firmware": {Stdout: "/usr/lib/firmware/\n"}, + "pacman -Qlq -- linux-lts": {Stdout: "/usr/lib/modules/6.18.54-2-lts/vmlinuz\n"}, + }, nil), 1000) + m.ReadFile = func(path string) ([]byte, error) { + switch path { + case OSRelease: + return []byte("7.2.8-arch1-2\n"), nil + case PacmanLog: + return []byte(checkLog), nil + } + return nil, errNoFile + } + return m +} + +func checkedRun(steps ...string) Run { + r := Run{Node: "node-a", Run: "20261004T120000Z", Kind: RunUpgrade, From: FromAsk} + at := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) + for _, s := range steps { + r.Steps = append(r.Steps, Step{Step: s, At: at}) + } + return r +} + +func checkUpgrades(t *testing.T, m *Machine) *Upgrades { + runs := newBucket() + previous := Run{Node: "node-a", Run: "20260926T040000Z", Kind: RunUpgrade, From: FromRound, + Steps: []Step{{Step: StepStarted, At: time.Date(2026, 9, 26, 4, 0, 0, 0, time.UTC)}}} + if _, err := runs.Create(NodeKey("node-a", previous.Run), previous); err != nil { + t.Fatal(err) + } + saved := fetch + fetch = func(string) ([]byte, error) { return []byte(newsFeed), nil } + t.Cleanup(func() { fetch = saved }) + return &Upgrades{Node: "node-a", Machine: m, Runs: runs} +} + +func TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade(t *testing.T) { + u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) + c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) + if strings.Join(c.Pacnew, " ") != "/etc/pacman.d/mirrorlist.pacnew" || strings.Join(c.Pacsave, " ") != "/etc/lemurs/config.toml.pacsave" { + t.Errorf("only this transaction's files: %v %v", c.Pacnew, c.Pacsave) + } + if len(c.News) != 1 || c.News[0].Title != "A manual intervention" || c.NewsError != "" { + t.Errorf("the news since the previous upgrade (26 September): %+v %q", c.News, c.NewsError) + } + if !c.Passed || c.RestartNeeded || c.RunningKernel != "7.2.8-arch1-2" || c.InstalledKernel != "7.2.8-arch1-2" { + t.Errorf("%+v", c) + } +} + +func TestNewsThatCannotBeReadIsSaidNotShownAsNone(t *testing.T) { + u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) + fetch = func(string) ([]byte, error) { return nil, errors.New("no route to host") } + c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) + if !strings.Contains(c.NewsError, "no route to host") || !c.Passed { + t.Errorf("%+v", c) + } +} + +func TestAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot(t *testing.T) { + u := checkUpgrades(t, checkMachine("backup.service loaded failed failed The backup\nweb.service loaded failed failed The web\n", "7.2.8-arch1-2")) + c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{FailedUnits: []string{"backup.service"}}) + if strings.Join(c.NewlyFailed, " ") != "web.service" || c.Passed { + t.Errorf("%+v", c) + } +} + +func TestARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne(t *testing.T) { + u := checkUpgrades(t, checkMachine("", "7.2.9-arch1-1")) + c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) + if !c.RestartNeeded || c.InstalledKernel != "7.2.9-arch1-1" || !c.Passed { + t.Errorf("a restart is said and does not fail the check: %+v", c) + } +} + +func TestTheCheckPassesOnlyWhenTheTransactionSucceeded(t *testing.T) { + u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) + if c := u.AfterCheck(checkedRun(StepStarted, StepFailed), &Record{}); c.Passed { + t.Errorf("%+v", c) + } +} diff --git a/modules/pacman/cmd/pacman-tools/bus_fake_test.go b/modules/pacman/cmd/pacman-tools/bus_fake_test.go new file mode 100644 index 00000000..0d90a846 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/bus_fake_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// memBucket is a state in memory, with the bus's revisions and its refusal of a lost race. +type memBucket struct { + mu sync.Mutex + values map[string][]byte + revs map[string]uint64 + rev uint64 + writes []string // "op key", in order + fail map[string]error +} + +func newBucket() *memBucket { + return &memBucket{values: map[string][]byte{}, revs: map[string]uint64{}, fail: map[string]error{}} +} + +func (b *memBucket) Get(key string) (*stdio.StateEntry, error) { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.fail["get "+key]; err != nil { + return nil, err + } + v, ok := b.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: b.revs[key]}, nil +} + +func (b *memBucket) write(op, key string, value any) (uint64, error) { + if err := b.fail[op+" "+key]; err != nil { + return 0, err + } + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + b.rev++ + b.values[key], b.revs[key] = data, b.rev + b.writes = append(b.writes, op+" "+key) + return b.rev, nil +} + +func (b *memBucket) Create(key string, value any) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if _, ok := b.values[key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return b.write("create", key, value) +} + +func (b *memBucket) Update(key string, value any, revision uint64) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.revs[key] != revision { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return b.write("update", key, value) +} + +func (b *memBucket) Put(key string, value any) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.write("put", key, value) +} + +func (b *memBucket) Delete(key string) error { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.fail["delete "+key]; err != nil { + return err + } + delete(b.values, key) + delete(b.revs, key) + b.writes = append(b.writes, "delete "+key) + return nil +} + +func (b *memBucket) Keys() ([]string, error) { + b.mu.Lock() + defer b.mu.Unlock() + keys := []string{} + for k := range b.values { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} + +// value reads a key into its shape; a missing key is the zero value and false. +func (b *memBucket) value(key string, v any) bool { + b.mu.Lock() + defer b.mu.Unlock() + raw, ok := b.values[key] + if ok { + if err := json.Unmarshal(raw, v); err != nil { + panic(err) + } + } + return ok +} + +// writesTo are the writes to keys starting with prefix. +func (b *memBucket) writesTo(prefix string) []string { + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for _, w := range b.writes { + if _, key, _ := strings.Cut(w, " "); strings.HasPrefix(key, prefix) { + out = append(out, w) + } + } + return out +} + +// sentEvent is one event a fake sink was handed. +type sentEvent struct { + Name string + Body any +} + +type eventLog struct { + mu sync.Mutex + sent []sentEvent +} + +func (e *eventLog) Emit(name string, body any, _ ...stdio.EmitOptions) error { + e.mu.Lock() + defer e.mu.Unlock() + e.sent = append(e.sent, sentEvent{name, body}) + return nil +} + +func (e *eventLog) names() []string { + e.mu.Lock() + defer e.mu.Unlock() + var out []string + for _, s := range e.sent { + out = append(out, s.Name) + } + return out +} diff --git a/modules/pacman/cmd/pacman-tools/busshapes_test.go b/modules/pacman/cmd/pacman-tools/busshapes_test.go new file mode 100644 index 00000000..afabca1b --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/busshapes_test.go @@ -0,0 +1,203 @@ +package main + +// The keeper's bus shapes carry only their named fields (novox/hq ADR 0507 §1, AC7; ADR 0508): every key's value +// and every event's body, filled whole, marshals to exactly the fields named here, and no shape holds a map or a +// free-form value through which anything else could reach the bus. + +import ( + "encoding/json" + "reflect" + "regexp" + "sort" + "strings" + "testing" + "time" +) + +// busFields is every shape's JSON fields, by Go type: the interface the keeper and the verbs share. +var busFields = map[string][]string{ + "InstalledValue": {"at", "packages"}, + "InstalledPackage": {"name", "version", "explicit", "repo"}, + "NotCurrent": {"bytes", "since"}, + "UpdatesValue": {"reachable", "packages"}, + "VersionChange": {"name", "from", "to"}, + "PackageVersion": {"name", "version"}, + "Run": {"node", "run", "kind", "from", "round", "ask", "of", "steps"}, + "Step": {"step", "at", "why", "held", "unit", "ignored", "plan", "exit", "error", "changes", "check"}, + "HeldKernel": {"package", "version", "module", "bound"}, + "Changes": {"added", "removed", "changed"}, + "Check": {"passed", "pacnew", "pacsave", "news", "news_error", "mapped", "mapped_error", "newly_failed", "restart_needed", "running_kernel", + "installed_kernel", "unbuilt", "dkms_error"}, + "NewsTitle": {"title", "link", "published"}, + "Mapped": {"unit", "container", "executable"}, + "DKMSBuild": {"module", "version", "kernel"}, + "RollbackItem": {"name", "from", "to", "source", "location"}, + "Record": {"node", "run", "at", "installed", "kernel", "failed_units", "dkms"}, + "DKMSModule": {"module", "version", "kernels"}, + "Round": {"round", "opened", "opened_by", "order"}, + "RoundNode": {"round", "node", "run", "pass"}, + "Pass": {"reason", "ask", "asked", "answer", "answered"}, + "Ask": {"id", "node", "act", "at", "of", "remove_added", "round", "pass", "reason"}, + "PackageChanged": {"node", "at", "added", "removed", "changed"}, + "UpgradeRecorded": {"node", "run", "packages", "kernel"}, + "KernelHeld": {"node", "run", "held"}, + "UpgradeStarted": {"node", "run", "from", "round", "unit", "ignored"}, + "UpgradeFinished": {"node", "run", "changes"}, + "UpgradeFailed": {"node", "run", "step", "exit", "error"}, + "UpgradeChecked": {"node", "run", "kind", "passed"}, + "RollbackStarted": {"node", "run", "of", "unit"}, + "RollbackFinished": {"node", "run", "of", "exit", "error"}, +} + +// The value of each state's keys. +var stateValues = []any{InstalledValue{}, NotCurrent{}, UpdatesValue{}, Run{}, Record{}, Round{}, RoundNode{}, Ask{}} + +var stamp = time.Date(2026, 10, 24, 4, 0, 12, 0, time.UTC) + +// fill sets every field of v, recursively, to a value that is not empty: one element in each list, every pointer +// set, so that every field marshals, omitempty or not. +func fill(v reflect.Value) { + switch v.Kind() { + case reflect.String: + v.SetString("x") + case reflect.Bool: + v.SetBool(true) + case reflect.Int, reflect.Int64: + v.SetInt(1) + case reflect.Pointer: + v.Set(reflect.New(v.Type().Elem())) + fill(v.Elem()) + case reflect.Slice: + s := reflect.MakeSlice(v.Type(), 1, 1) + fill(s.Index(0)) + v.Set(s) + case reflect.Struct: + if v.Type() == reflect.TypeOf(time.Time{}) { + v.Set(reflect.ValueOf(stamp)) + return + } + for i := 0; i < v.NumField(); i++ { + fill(v.Field(i)) + } + } +} + +// fieldsOf walks a filled value against its marshalled JSON: each struct's object holds exactly its named +// fields. It answers the struct types it met. +func fieldsOf(t *testing.T, v reflect.Value, raw json.RawMessage, met map[string]bool) { + t.Helper() + switch v.Kind() { + case reflect.Map, reflect.Interface: + t.Errorf("%s holds a free-form value: a bus shape names its fields", v.Type()) + case reflect.Pointer: + fieldsOf(t, v.Elem(), raw, met) + case reflect.Slice: + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil || len(items) != v.Len() { + t.Errorf("%s: %s is not its list", v.Type(), raw) + return + } + for i := range items { + fieldsOf(t, v.Index(i), items[i], met) + } + case reflect.Struct: + if v.Type() == reflect.TypeOf(time.Time{}) { + return + } + name := v.Type().Name() + met[name] = true + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + t.Errorf("%s: %s is not an object", name, raw) + return + } + var got []string + for k := range obj { + got = append(got, k) + } + want, named := busFields[name] + if !named { + t.Errorf("%s is on the bus and its fields are not named in busFields", name) + return + } + want = append([]string(nil), want...) + sort.Strings(got) + sort.Strings(want) + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("%s marshals to %v, its named fields are %v", name, got, want) + } + for i := 0; i < v.NumField(); i++ { + tag, _, _ := strings.Cut(v.Type().Field(i).Tag.Get("json"), ",") + if r, ok := obj[tag]; ok { + fieldsOf(t, v.Field(i), r, met) + } + } + } +} + +func TestEveryStateValueAndEventBodyCarriesOnlyItsNamedFields(t *testing.T) { + roots := append([]any(nil), stateValues...) + for _, e := range Events { + roots = append(roots, e.Body) + } + met := map[string]bool{} + for _, root := range roots { + v := reflect.New(reflect.TypeOf(root)).Elem() + fill(v) + raw, err := json.Marshal(v.Interface()) + if err != nil { + t.Fatal(err) + } + fieldsOf(t, v, raw, met) + } + for name := range busFields { + if !met[name] { + t.Errorf("busFields names %s, which no state value or event body holds", name) + } + } +} + +func TestEveryEventIsNamedOnceWithItsBody(t *testing.T) { + seen := map[string]bool{} + for _, e := range Events { + if seen[e.Name] { + t.Errorf("%s twice", e.Name) + } + seen[e.Name] = true + if e.Body == nil || reflect.TypeOf(e.Body).Kind() != reflect.Struct { + t.Errorf("%s has no body type", e.Name) + } + } + if len(Events) != 9 { + t.Errorf("package.changed and the eight upgrade and rollback events, not %d", len(Events)) + } +} + +// A key is node and id tokens joined by dots; each token is one NATS key token. +func TestKeysAreTokensJoinedByDots(t *testing.T) { + token := regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + run := RunID(time.Date(2026, 10, 24, 6, 0, 12, 0, time.FixedZone("CEST", 2*3600))) + if run != "20261024T040012Z" { + t.Errorf("RunID: %s", run) + } + round := RoundID(time.Date(2026, 10, 24, 4, 0, 0, 0, time.Local)) + if round != "2026-W43" { + t.Errorf("RoundID: %s", round) + } + if RoundID(time.Date(2027, 1, 2, 4, 0, 0, 0, time.UTC)) != "2026-W53" { + t.Errorf("the ISO week of 2 January 2027 is 2026's 53rd: %s", RoundID(time.Date(2027, 1, 2, 4, 0, 0, 0, time.UTC))) + } + for key, parts := range map[string]int{ + NodeKey("node-a", run): 2, NotCurrentKey("node-a"): 2, RoundNodeKey(round, "node-b"): 2, + } { + ts := strings.Split(key, ".") + if len(ts) != parts { + t.Errorf("%s: %d tokens", key, len(ts)) + } + for _, x := range ts { + if !token.MatchString(x) { + t.Errorf("%s: token %q", key, x) + } + } + } +} diff --git a/modules/pacman/cmd/pacman-tools/dkms.go b/modules/pacman/cmd/pacman-tools/dkms.go new file mode 100644 index 00000000..11f324a6 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/dkms.go @@ -0,0 +1,166 @@ +package main + +// The DKMS gate (novox/hq ADR 0508 §3): before the transaction, every kernel package the upgrade would change is +// compared with the kernel bound each installed DKMS module declares itself: `Linux-Maximum` in its source's +// META, or `BUILD_EXCLUSIVE_KERNEL` (a pattern of kernel releases) in its dkms.conf. A kernel past a bound is +// left out of the run with its headers, and the rest is upgraded. A kernel package is known by its files, not +// its name: one whose files hold usr/lib/modules//vmlinuz. + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +var ( + vmlinuzPath = regexp.MustCompile(`^/?usr/lib/modules/([^/]+)/vmlinuz$`) + linuxMaximum = regexp.MustCompile(`(?m)^Linux-Maximum:\s*(\S+)\s*$`) + buildExclusive = regexp.MustCompile(`(?m)^BUILD_EXCLUSIVE_KERNEL=(?:"([^"]*)"|'([^']*)'|(\S+))\s*$`) + leadingNumber = regexp.MustCompile(`^[0-9]+(\.[0-9]+)*`) +) + +// kernelRelease is the release an installed kernel package's image is for, and whether the package is a kernel. +func (m *Machine) kernelRelease(pkg string) (string, bool, error) { + out, err := m.Out("pacman", "-Qlq", "--", pkg) + if err != nil { + return "", false, fmt.Errorf("the files of %s: %w", pkg, err) + } + for _, l := range lines(out) { + if g := vmlinuzPath.FindStringSubmatch(strings.TrimSpace(l)); g != nil { + return g[1], true, nil + } + } + return "", false, nil +} + +// NewRelease is the release a kernel package's new version will install, derived from it as the installed +// release is from the installed version: `7.2.8.arch1-2` installs `7.2.8-arch1-2`, `6.18.54-2` (the LTS +// kernel) installs `6.18.54-2-lts`. +func NewRelease(installedVersion, installedRelease, newVersion string) string { + base := func(v string) string { return strings.Replace(v, ".arch", "-arch", 1) } + suffix := "" + if b := base(installedVersion); strings.HasPrefix(installedRelease, b) { + suffix = strings.TrimPrefix(installedRelease, b) + } + return base(newVersion) + suffix +} + +// pastMaximum says whether a release is past a Linux-Maximum bound, compared on as many numbers as the bound +// gives: 7.3.1 is past 7.2, 7.2.9 is not. +func pastMaximum(release, maximum string) bool { + bound := strings.Split(leadingNumber.FindString(maximum), ".") + have := strings.Split(leadingNumber.FindString(release), ".") + if bound[0] == "" || have[0] == "" { + return false + } + for i, b := range bound { + bn, _ := strconv.Atoi(b) + hn := 0 + if i < len(have) { + hn, _ = strconv.Atoi(have[i]) + } + if hn != bn { + return hn > bn + } + } + return false +} + +// dkmsBound is the bound one DKMS module declares, read from its source. +type dkmsBound struct { + Maximum string + Exclusive string +} + +func (m *Machine) dkmsBound(mod DKMSModule) dkmsBound { + dir := "/usr/src/" + mod.Module + "-" + mod.Version + "/" + var b dkmsBound + if meta, err := m.ReadFile(dir + "META"); err == nil { + if g := linuxMaximum.FindSubmatch(meta); g != nil { + b.Maximum = string(g[1]) + } + } + if conf, err := m.ReadFile(dir + "dkms.conf"); err == nil { + if g := buildExclusive.FindSubmatch(conf); g != nil { + b.Exclusive = string(g[1]) + string(g[2]) + string(g[3]) + } + } + return b +} + +// heldBy says which bound of a module holds a release, or nothing. +func heldBy(b dkmsBound, release string) (string, error) { + if b.Maximum != "" && pastMaximum(release, b.Maximum) { + return "Linux-Maximum: " + b.Maximum, nil + } + if b.Exclusive != "" { + re, err := regexp.Compile(b.Exclusive) + if err != nil { + return "", fmt.Errorf("BUILD_EXCLUSIVE_KERNEL %q is not a pattern this reads: %w", b.Exclusive, err) + } + if !re.MatchString(release) { + return "BUILD_EXCLUSIVE_KERNEL=" + b.Exclusive, nil + } + } + return "", nil +} + +// Gate is the DKMS gate for the node's next run: the kernels it holds, the packages left out of the transaction +// for them (each held kernel and its headers), or why the run cannot start. +func (u *Upgrades) Gate() (held []HeldKernel, ignore []string, why string) { + e, err := u.Updates.Get(u.Node) + if err != nil { + return nil, nil, "what an upgrade would change could not be read from the bus: " + err.Error() + } + if e == nil { + return nil, nil, "what an upgrade would change is not on the bus yet, so no kernel it would bring can be checked" + } + var updates UpdatesValue + if err := json.Unmarshal(e.Value, &updates); err != nil { + return nil, nil, "what an upgrade would change is not readable: " + err.Error() + } + if !updates.Reachable { + return nil, nil, "the node could not reach its mirrors, so what an upgrade would change is not known" + } + var modules []DKMSModule + var bounds []dkmsBound + held, ignore = []HeldKernel{}, []string{} + for _, p := range updates.Packages { + if p.Name != "linux" && !strings.HasPrefix(p.Name, "linux-") { + continue + } + release, kernel, err := u.Machine.kernelRelease(p.Name) + if err != nil { + return nil, nil, "whether " + p.Name + " is a kernel could not be read: " + err.Error() + } + if !kernel { + continue + } + if modules == nil { + if modules, err = u.Machine.DKMSModules(); err != nil { + return nil, nil, "the DKMS modules could not be read: " + err.Error() + } + for _, mod := range modules { + b := u.Machine.dkmsBound(mod) + bounds = append(bounds, b) + } + } + next := NewRelease(p.From, release, p.To) + for i, mod := range modules { + bound, err := heldBy(bounds[i], next) + if err != nil { + return nil, nil, mod.Module + " " + mod.Version + ": " + err.Error() + } + if bound == "" { + continue + } + held = append(held, HeldKernel{Package: p.Name, Version: p.To, Module: mod.Module + "/" + mod.Version, Bound: bound}) + if !contains(ignore, p.Name) { + ignore = append(ignore, p.Name, p.Name+"-headers") + } + } + } + return held, ignore, "" +} diff --git a/modules/pacman/cmd/pacman-tools/dkms_test.go b/modules/pacman/cmd/pacman-tools/dkms_test.go new file mode 100644 index 00000000..1c396138 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/dkms_test.go @@ -0,0 +1,115 @@ +package main + +// The DKMS gate (novox/hq ADR 0508 §3; acceptance 1841/AC4). The META is captured from a machine whose ZFS +// module declares Linux-Maximum 7.2; the BUILD_EXCLUSIVE_KERNEL case is a made-up module's pattern. + +import ( + "os" + "strings" + "testing" +) + +func TestTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs(t *testing.T) { + for _, c := range [][4]string{ + {"7.2.8.arch1-2", "7.2.8-arch1-2", "7.3.1.arch1-1", "7.3.1-arch1-1"}, + {"6.18.54-2", "6.18.54-2-lts", "6.18.60-1", "6.18.60-1-lts"}, + } { + if got := NewRelease(c[0], c[1], c[2]); got != c[3] { + t.Errorf("%v: %s", c, got) + } + } + for release, past := range map[string]bool{"7.3.1-arch1-1": true, "7.2.9-arch1-1": false, "7.2-arch1-1": false, "8.0.1": true, "6.18.60-1-lts": false} { + if pastMaximum(release, "7.2") != past { + t.Errorf("%s past 7.2: want %v", release, past) + } + } +} + +// gateMachine is a machine with the linux and LTS kernels and these DKMS modules' sources. +func gateMachine(t *testing.T, dkmsStatus string, sources map[string]string) *Machine { + files := func(paths ...string) Ran { return Ran{Stdout: strings.Join(paths, "\n") + "\n"} } + m := machine(byLine(map[string]Ran{ + "pacman -Qlq -- linux": files("/usr/lib/modules/7.2.8-arch1-2/", "/usr/lib/modules/7.2.8-arch1-2/vmlinuz"), + "pacman -Qlq -- linux-headers": files("/usr/lib/modules/7.2.8-arch1-2/build/Makefile"), + "pacman -Qlq -- linux-lts": files("/usr/lib/modules/6.18.54-2-lts/vmlinuz"), + "pacman -Qlq -- linux-firmware": files("/usr/lib/firmware/amdgpu/"), + "dkms status": {Stdout: dkmsStatus}, + }, nil), 1000) + m.ReadFile = func(path string) ([]byte, error) { + if s, ok := sources[path]; ok { + return []byte(s), nil + } + return nil, errNoFile + } + return m +} + +func capturedZFSSource(t *testing.T) map[string]string { + out := map[string]string{} + for _, f := range []string{"META", "dkms.conf"} { + raw, err := os.ReadFile("testdata/dkms-zfs-linux-maximum-7.2/" + f) + if err != nil { + t.Fatal(err) + } + out["/usr/src/zfs-2.4.4/"+f] = string(raw) + } + return out +} + +func gateUpgrades(t *testing.T, m *Machine, linuxTo string, reachable bool) *Upgrades { + updates := newBucket() + if _, err := updates.Put("node-a", UpdatesValue{Reachable: reachable, Packages: []VersionChange{ + {Name: "bash", From: "5.3.15-1", To: "5.3.20-1"}, + {Name: "linux", From: "7.2.8.arch1-2", To: linuxTo}, + {Name: "linux-firmware", From: "20261001-1", To: "20261015-1"}, + {Name: "linux-headers", From: "7.2.8.arch1-2", To: linuxTo}, + {Name: "linux-lts", From: "6.18.54-2", To: "6.18.60-1"}, + }}); err != nil { + t.Fatal(err) + } + return &Upgrades{Node: "node-a", Machine: m, Updates: updates} +} + +const zfsStatus = "zfs/2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs/2.4.4, 6.18.54-2-lts, x86_64: installed\n" + +func TestAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders(t *testing.T) { + u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.3.1.arch1-1", true) + held, ignore, why := u.Gate() + if why != "" { + t.Fatal(why) + } + if len(held) != 1 || held[0] != (HeldKernel{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs/2.4.4", Bound: "Linux-Maximum: 7.2"}) { + t.Errorf("held: %+v", held) + } + if strings.Join(ignore, " ") != "linux linux-headers" { + t.Errorf("left out: %v (the LTS kernel stays within the bound and is upgraded)", ignore) + } +} + +func TestAKernelWithinTheBoundIsNotHeld(t *testing.T) { + u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.2.9.arch1-1", true) + if held, ignore, why := u.Gate(); why != "" || len(held) != 0 || len(ignore) != 0 { + t.Errorf("%+v %v %q", held, ignore, why) + } +} + +func TestABuildExclusivePatternHoldsTheKernelsItDoesNotMatch(t *testing.T) { + src := map[string]string{"/usr/src/made-up-driver-1.0/dkms.conf": "PACKAGE_NAME=\"made-up-driver\"\nBUILD_EXCLUSIVE_KERNEL=\"^(6\\.|7\\.[0-2]\\.)\"\n"} + u := gateUpgrades(t, gateMachine(t, "made-up-driver/1.0, 7.2.8-arch1-2, x86_64: installed\n", src), "7.3.1.arch1-1", true) + held, ignore, why := u.Gate() + if why != "" || len(held) != 1 || held[0].Package != "linux" || held[0].Bound != `BUILD_EXCLUSIVE_KERNEL=^(6\.|7\.[0-2]\.)` || + strings.Join(ignore, " ") != "linux linux-headers" { + t.Errorf("%+v %v %q", held, ignore, why) + } +} + +func TestNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown(t *testing.T) { + u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.3.1.arch1-1", false) + if _, _, why := u.Gate(); !strings.Contains(why, "could not reach its mirrors") { + t.Errorf("unreachable: %q", why) + } + u.Updates = newBucket() + if _, _, why := u.Gate(); !strings.Contains(why, "not on the bus") { + t.Errorf("no updates key: %q", why) + } +} diff --git a/modules/pacman/cmd/pacman-tools/installed_health.go b/modules/pacman/cmd/pacman-tools/installed_health.go new file mode 100644 index 00000000..bf47c9a8 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/installed_health.go @@ -0,0 +1,32 @@ +package main + +// A node whose packages are over the bus's cap is unhealthy, naming the size (novox/hq ADR 0507 §2, acceptance +// 1840/AC3): `installed.` keeps its last value, which is no longer the node's, and the module's health says +// so until the keeper writes it again and deletes the mark. + +import ( + "encoding/json" + "fmt" +) + +// InstalledHealth is the health of this node's `installed` key: unhealthy while `.not-current` is there, +// with its size and since when; ok false when there is nothing to say (the key is current, or the state cannot +// be read here, which the keeper's own log says). +func (m *Machine) InstalledHealth() (Health, bool) { + if m.States == nil || m.Node == "" { + return Health{}, false + } + e, err := m.States(StateInstalled).Get(NotCurrentKey(m.Node)) + if err != nil { + return Health{Why: "the node's packages on the bus could not be read: " + err.Error()}, true + } + if e == nil { + return Health{}, false + } + var nc NotCurrent + if err := json.Unmarshal(e.Value, &nc); err != nil { + return Health{Why: fmt.Sprintf("installed.%s is marked not current, and its mark is not readable: %v", m.Node, err)}, true + } + return Health{Why: fmt.Sprintf("the node's packages come to %d bytes, over the bus's cap of %d on a value, since %s: "+ + "installed.%s keeps an older list", nc.Bytes, ValueCap, nc.Since.Format("2006-01-02 15:04 MST"), m.Node)}, true +} diff --git a/modules/pacman/cmd/pacman-tools/installed_health_test.go b/modules/pacman/cmd/pacman-tools/installed_health_test.go new file mode 100644 index 00000000..5b3f559f --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/installed_health_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// fakeReader reads a fakeStore as a verb does. +type fakeReader struct{ *fakeStore } + +func (r fakeReader) History(key string) ([]stdio.StateRevision, error) { + e, _ := r.Get(key) + if e == nil { + return []stdio.StateRevision{}, nil + } + return []stdio.StateRevision{{Key: key, Value: e.Value, Revision: e.Revision, Op: "put", At: r.at[key]}}, nil +} + +// readerOf is a machine's States over fake states. +func readerOf(states *fakeStates) func(string) StateReader { + return func(name string) StateReader { return fakeReader{states.store(name)} } +} + +// 1840/AC3: the health is unhealthy naming the size while the node is marked not current, and says nothing of +// it once the mark is gone. +func TestTheHealthNamesTheSizeWhileTheNodeIsNotCurrent(t *testing.T) { + states := &fakeStates{} + m := machine(nil, 1000) + m.Node, m.States = "node-a", readerOf(states) + if _, said := m.InstalledHealth(); said { + t.Fatal("a current node's health said something") + } + states.store(StateInstalled).Put("node-a.not-current", NotCurrent{Bytes: 300000, Since: t0}) + states.store(StateInstalled).Put("node-b.not-current", NotCurrent{Bytes: 1, Since: t0}) + h, said := m.InstalledHealth() + if !said || h.Healthy || !strings.Contains(h.Why, "300000 bytes") || !strings.Contains(h.Why, "2026-10-24 04:00") { + t.Fatalf("%+v", h) + } + states.store(StateInstalled).Delete("node-a.not-current") + if _, said := m.InstalledHealth(); said { + t.Fatal("the mark is gone and the health still says it") + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper.go b/modules/pacman/cmd/pacman-tools/keeper.go new file mode 100644 index 00000000..7c6eb981 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper.go @@ -0,0 +1,639 @@ +package main + +// The keeper (novox/hq ADR 0507 §1-§4; ADR 0508): the module's process on every node, under an account of its +// own with a bus account of its own (ADR 0436 §1). It keeps the node's packages in the state `installed` and what +// an upgrade would change in `updates`, and says each change of the packages as `package.changed`. It reads the +// machine at its start and whenever the machine may have changed, never on a timer but the one daily read of +// `updates` ADR 0507 §4 allows: +// +// - pacman's database lock released (an inotify watch on its directory: the lock file deleted) ends a +// transaction, whoever ran it: `installed` and then `updates` are read again; +// - the mirror list rewritten (reflector writes /etc/pacman.d/mirrorlist): `updates` is read again; +// - a day after `updates` was last read: it is read again, since the repositories change what an upgrade +// would do without any transaction on the node. +// +// A key is written only when what it holds changed (ADR 0289 §1). A value over the bus's cap is not written: +// the key keeps its last value and `.not-current` says the size and since when, written once and deleted +// once the node's packages are current again (ADR 0507 §2). +// +// Everything the keeper touches is injected (its states, its events, the machine, the clock, the watch), so the +// loop is tested without a machine or a bus. One goroutine runs it all: the watch's signals, the clock, and every +// change a hook watches are handled one at a time, in the order they came. +// +// # Hooks (the upgrade parts of ADR 0508 join here) +// +// Hooks are where further work joins the loop, each run on the loop's goroutine: +// +// - Start: once, after the first reads of `installed` and `updates`, before anything else; +// - AfterTransaction: after every released lock, once `installed` and `updates` were read again, with what the +// transaction changed (nil when nothing); +// - Changes: each a state and a key pattern (`*` one name, `**` the rest) watched for the life of the keeper; +// its Handle sees the current values first (Current set), then every put and delete in order; +// - Wakes: each a function the keeper calls at its start and again at the time it last answered; it answers when +// it next wants to be called (a zero time: not again). It is how a hook keeps a time (a round's opening, a +// stall) without polling. +// +// A hook's error is logged and the keeper goes on: a fault in one part never stops the view. keeperHooks builds +// the hooks the keeper runs with, once connected; it is the one place a part is added. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// ValueCap is the bus's cap on one value of a state (ADR 0201 §14): the mesh's bound, not the module's. +const ValueCap = 256 << 10 + +// Where pacman keeps its database and its lock, and where the mirror list is. +const ( + PacmanDBDir = "/var/lib/pacman" + LockName = "db.lck" + MirrorDir = "/etc/pacman.d" + MirrorName = "mirrorlist" +) + +// UpdatesEvery is how long after the last read of `updates` it is read again with nothing else asking (ADR 0507 +// §4: once a day). +const UpdatesEvery = 24 * time.Hour + +// Store is one state as the keeper uses it: a process state on the bus (process.State), or a fake in a test. +type Store interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) + Put(key string, value any) (uint64, error) + Delete(key string) error + Keys() ([]string, error) + Watch(key string, handle func(stdio.StateChange) error) error +} + +// Clock is the time as the keeper sees it. +type Clock interface { + Now() time.Time + // After fires once, d from now. + After(d time.Duration) <-chan time.Time +} + +type realClock struct{} + +func (realClock) Now() time.Time { return time.Now() } +func (realClock) After(d time.Duration) <-chan time.Time { return time.After(d) } + +// Signal is what the machine's watch saw. +type Signal int + +const ( + // LockReleased: pacman's database lock was deleted, a transaction (or a database sync) ended. + LockReleased Signal = iota + 1 + // MirrorsRewritten: the mirror list was written or replaced. + MirrorsRewritten +) + +func (s Signal) String() string { + switch s { + case LockReleased: + return "the lock released" + case MirrorsRewritten: + return "the mirror list rewritten" + } + return fmt.Sprintf("signal %d", int(s)) +} + +// Watcher is the machine's watch: its signals, until it is closed or fails (the channel closes). +type Watcher interface { + Signals() <-chan Signal + Close() error +} + +// Transaction is a released lock as AfterTransaction sees it: when, and what it changed in `installed` +// (nil when it changed nothing, or the value could not be written). +type Transaction struct { + At time.Time + Changed *PackageChanged +} + +// StateHook is a watch on one of the module's states for the life of the keeper. +type StateHook struct { + State string + Key string + Handle func(k *Keeper, c stdio.StateChange) error +} + +// Wake is called at the keeper's start and then at the time it last answered; it answers when next. +type Wake func(k *Keeper, now time.Time) time.Time + +// Hooks are the parts that join the keeper's loop (see the file's comment). +type Hooks struct { + Start []func(k *Keeper) error + AfterTransaction []func(k *Keeper, t Transaction) error + Changes []StateHook + Wakes []Wake +} + +// keeperHooks is the hooks the keeper runs with, built once the keeper is connected: the one place a part of +// the keeper is added. Today: the upgrades of ADR 0508 (upgrades.go, record.go, transaction.go, rounds.go), for +// this node. +// +// - Start: the package manager's log read into `runs` once (ImportLog), records past record-days pruned +// (PruneRecords), and every unfinished run carried on (Follow), a run a keeper that went away left included; +// - AfterTransaction: every unfinished run carried on (Follow): a released lock may be a run's transaction ending; +// - Changes: `asks` at `.*`, each change handed to TakeAsk; `rounds` and `runs`, every change, to the +// round's step (RoundStep): a node starts the moment the node before it passed; +// - Wakes: PruneRecords once a day; the round's step at the time it answered (the Saturday's opening, a pass +// whose answer is read again, a refused run tried again). +func keeperHooks(k *Keeper) Hooks { + root := k.Root + if root == nil { + root = newRootActs() + } + u := &Upgrades{Node: k.Node, Config: k.Config, Machine: k.Machine, Root: root, Events: k.Events, + Installed: k.State(StateInstalled), Updates: k.State(StateUpdates), Runs: k.State(StateRuns), + Records: k.State(StateRecords), Rounds: k.State(StateRounds), Asks: k.State(StateAsks), AskRecord: k.AskRecord, + Check: k.Check} + k.upgrades = u + const roundWake = 1 // the round's step is the second wake + roundStep := func(k *Keeper, now time.Time) time.Time { + next, err := u.RoundStep() + if err != nil { + k.logf("[pacman] keeper: the round: %v", err) + } + return next + } + // onRoundChange takes the round's step on a change of `rounds` or `runs`, and keeps the time it answers. The + // current values handed at the watch's start are not each a step: the wake at the start takes one. + onRoundChange := func(k *Keeper, c stdio.StateChange) error { + if c.Current { + return nil + } + k.wakeAt(roundWake, roundStep(k, k.Clock.Now())) + return nil + } + return Hooks{ + Start: []func(*Keeper) error{ + func(k *Keeper) error { + n, err := u.ImportLog() + if n > 0 { + k.logf("[pacman] keeper: %d past upgrades read from the package manager's log into runs", n) + } + return err + }, + func(*Keeper) error { _, err := u.PruneRecords(); return err }, + func(*Keeper) error { return u.Follow() }, + }, + AfterTransaction: []func(*Keeper, Transaction) error{ + func(*Keeper, Transaction) error { return u.Follow() }, + }, + Changes: []StateHook{ + {State: StateAsks, Key: k.Node + ".*", Handle: func(_ *Keeper, c stdio.StateChange) error { + return u.TakeAsk(c) + }}, + {State: StateRounds, Key: "**", Handle: onRoundChange}, + {State: StateRuns, Key: "**", Handle: onRoundChange}, + }, + Wakes: []Wake{ + func(k *Keeper, now time.Time) time.Time { + if !now.Equal(k.started) { + if _, err := u.PruneRecords(); err != nil { + k.logf("[pacman] keeper: pruning records: %v", err) + } + } + return now.Add(UpdatesEvery) + }, + roundStep, + }, + } +} + +// newRootActs is the client of the module's root helper (ADR 0510), which carries out the acts that need root, +// at the socket the manifest names to the keeper; with none named, every act is refused, named. +var newRootActs = func() RootActs { + if sock := os.Getenv(EnvRootSocket); sock != "" { + return rootClient{Socket: sock} + } + return noRoot{} +} + +type noRoot struct{} + +const noRootSaid = "the keeper was started without " + EnvRootSocket + ", the root helper's socket" + +func (noRoot) Transaction(kind, run string, ignore []string) error { + return errors.New(noRootSaid + ": no transaction can start") +} + +func (noRoot) Journal(unit string, n int) ([]string, error) { + return nil, errors.New(noRootSaid + ": no journal can be read") +} + +func (noRoot) Rollback(run string, plan []RollbackItem) error { + return errors.New(noRootSaid + ": no rollback can start") +} + +func (noRoot) Maps() ([]Mapped, error) { + return nil, errors.New(noRootSaid + ": no process's maps can be read") +} + +// Keeper is the module's process on one node. +type Keeper struct { + Node string + Config KeeperConfig + Machine *Machine + State func(name string) Store + // Events emits the keeper's events; package.changed carries the id `installed..`, the bus's + // dedup id, so the same change said twice is dropped. + Events EventSink + Clock Clock + Watcher Watcher + // Locked says pacman's lock is held now: a released lock followed at once by another transaction is read + // at that one's end. + Locked func() bool + // DBChanged is when the local package database last changed, the time of a value read at the start. + DBChanged func() time.Time + Logf func(format string, a ...any) + // Hooks are the parts joining the loop; MakeHooks, when set, builds them once the keeper is connected + // (its State and Events set), replacing Hooks. + Hooks Hooks + MakeHooks func(k *Keeper) Hooks + // AskRecord reads the operator channel's record of one of the module's asks (process.Conn.Ask). + AskRecord func(id string) (*stdio.StateEntry, error) + // Root carries out the acts that need root; nil is the root helper's client (newRootActs). Check is the + // after-upgrade check; nil is the machine's (AfterCheck). A test sets both. + Root RootActs + Check func(run Run, record *Record) Check + + upgrades *Upgrades + + started time.Time + + nextUpdates time.Time + wakes []time.Time +} + +func (k *Keeper) logf(format string, a ...any) { + if k.Logf != nil { + k.Logf(format, a...) + } +} + +type change struct { + hook StateHook + c stdio.StateChange +} + +// Run reads the machine into the bus and then follows it until ctx ends or the watch fails. +func (k *Keeper) Run(ctx context.Context) error { + if k.MakeHooks != nil { + k.Hooks = k.MakeHooks(k) + } + k.started = k.Clock.Now() + at := k.started + if k.DBChanged != nil { + if t := k.DBChanged(); !t.IsZero() { + at = t + } + } + k.refreshInstalled(at) + k.refreshUpdates() + for _, h := range k.Hooks.Start { + if err := h(k); err != nil { + k.logf("[pacman] keeper: a start hook: %v", err) + } + } + changes := make(chan change, 256) + for _, h := range k.Hooks.Changes { + h := h + err := k.State(h.State).Watch(h.Key, func(c stdio.StateChange) error { + if c.Current { + // the current values are handed over inside Watch, on this goroutine. + k.handle(h, c) + return nil + } + select { + case changes <- change{h, c}: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + if err != nil { + return fmt.Errorf("watching %s %q: %w", h.State, h.Key, err) + } + } + now := k.started + k.wakes = make([]time.Time, len(k.Hooks.Wakes)) + for i, w := range k.Hooks.Wakes { + k.wakes[i] = w(k, now) + } + signals := k.Watcher.Signals() + for { + now = k.Clock.Now() + next := k.nextUpdates + for _, w := range k.wakes { + if !w.IsZero() && w.Before(next) { + next = w + } + } + timer := k.Clock.After(next.Sub(now)) + select { + case <-ctx.Done(): + return ctx.Err() + case s, open := <-signals: + if !open { + return errors.New("the machine's watch ended") + } + k.signal(s) + case c := <-changes: + k.handle(c.hook, c.c) + case <-timer: + k.due() + } + } +} + +// wakeAt sets when wake i is called next, from a change a hook handled; before the wakes are first called +// (the current values handed at a watch's start) there is nothing to set. +func (k *Keeper) wakeAt(i int, at time.Time) { + if i < len(k.wakes) { + k.wakes[i] = at + } +} + +func (k *Keeper) handle(h StateHook, c stdio.StateChange) { + if err := h.Handle(k, c); err != nil { + k.logf("[pacman] keeper: handling %s %s: %v", h.State, c.Key, err) + } +} + +// signal acts on what the machine's watch saw. +func (k *Keeper) signal(s Signal) { + switch s { + case LockReleased: + if k.Locked != nil && k.Locked() { + return + } + at := k.Clock.Now() + changed := k.refreshInstalled(at) + k.refreshUpdates() + for _, h := range k.Hooks.AfterTransaction { + if err := h(k, Transaction{At: at, Changed: changed}); err != nil { + k.logf("[pacman] keeper: after a transaction: %v", err) + } + } + case MirrorsRewritten: + k.refreshUpdates() + } +} + +// due runs what the clock has reached: the daily read of `updates`, and every wake whose time came. +func (k *Keeper) due() { + now := k.Clock.Now() + if !now.Before(k.nextUpdates) { + k.refreshUpdates() + } + for i, w := range k.wakes { + if !w.IsZero() && !now.Before(w) { + k.wakes[i] = k.Hooks.Wakes[i](k, now) + } + } +} + +// refreshInstalled reads the node's packages and writes `installed.` when they differ from the key, +// emitting `package.changed` with what changed. It answers that change, nil when nothing was written. +func (k *Keeper) refreshInstalled(at time.Time) *PackageChanged { + pkgs, skipped, err := k.Machine.ReadInstalled() + if err != nil { + k.logf("[pacman] keeper: reading the installed packages: %v", err) + return nil + } + if skipped > 0 { + k.logf("[pacman] keeper: %d lines of pacman's list were not a package and were left out", skipped) + } + st := k.State(StateInstalled) + var before InstalledValue + entry, err := st.Get(k.Node) + if err != nil { + k.logf("[pacman] keeper: reading installed.%s: %v", k.Node, err) + return nil + } + if entry != nil { + if err := json.Unmarshal(entry.Value, &before); err != nil { + k.logf("[pacman] keeper: installed.%s holds no package list, so it is written whole: %v", k.Node, err) + entry = nil + } + } + if entry != nil && samePackages(before.Packages, pkgs) { + k.current() + return nil + } + value := InstalledValue{At: at.UTC(), Packages: pkgs} + data, err := json.Marshal(value) + if err != nil { + k.logf("[pacman] keeper: %v", err) + return nil + } + if len(data) > ValueCap { + k.notCurrent(len(data), at) + return nil + } + rev, err := st.Put(k.Node, value) + if err != nil { + k.logf("[pacman] keeper: writing installed.%s: %v", k.Node, err) + return nil + } + k.current() + c := DiffInstalled(before.Packages, pkgs) + if c.Empty() { + return nil + } + ev := &PackageChanged{Node: k.Node, At: at.UTC(), Added: c.Added, Removed: c.Removed, Changed: c.Changed} + id := fmt.Sprintf("%s.%s.%d", StateInstalled, k.Node, rev) + if err := k.Events.Emit(EventPackageChanged, *ev, stdio.EmitOptions{MessageID: id}); err != nil { + k.logf("[pacman] keeper: emitting %s for installed.%s at revision %d: %v", EventPackageChanged, k.Node, rev, err) + } + return ev +} + +func samePackages(a, b []InstalledPackage) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// notCurrent marks the node's packages as not current, once: the first size and moment stay until it is current +// again. +func (k *Keeper) notCurrent(size int, at time.Time) { + st := k.State(StateInstalled) + key := NotCurrentKey(k.Node) + k.logf("[pacman] keeper: the node's packages come to %d bytes, over the bus's cap of %d: installed.%s keeps its last value", size, ValueCap, k.Node) + if e, err := st.Get(key); err != nil || e != nil { + if err != nil { + k.logf("[pacman] keeper: reading %s: %v", key, err) + } + return + } + if _, err := st.Put(key, NotCurrent{Bytes: size, Since: at.UTC()}); err != nil { + k.logf("[pacman] keeper: writing %s: %v", key, err) + } +} + +// current deletes the not-current mark, when there is one: the key holds the node's packages again. +func (k *Keeper) current() { + st := k.State(StateInstalled) + key := NotCurrentKey(k.Node) + e, err := st.Get(key) + if err != nil { + k.logf("[pacman] keeper: reading %s: %v", key, err) + return + } + if e == nil { + return + } + if err := st.Delete(key); err != nil { + k.logf("[pacman] keeper: deleting %s: %v", key, err) + } +} + +// refreshUpdates reads what a full upgrade would change and writes `updates.` when the list or reachable +// changed. A failed read keeps the last list and says reachable false, never an empty list in its place. +func (k *Keeper) refreshUpdates() { + k.nextUpdates = k.Clock.Now().Add(UpdatesEvery) + list, err := k.Machine.UpdateList() + st := k.State(StateUpdates) + entry, gerr := st.Get(k.Node) + if gerr != nil { + k.logf("[pacman] keeper: reading updates.%s: %v", k.Node, gerr) + return + } + var before UpdatesValue + if entry != nil { + if uerr := json.Unmarshal(entry.Value, &before); uerr != nil { + entry = nil + } + } + next := UpdatesValue{Reachable: true, Packages: list} + if err != nil { + k.logf("[pacman] keeper: reading what an upgrade would change: %v", err) + next = UpdatesValue{Reachable: false, Packages: before.Packages} + } + if next.Packages == nil { + next.Packages = []VersionChange{} + } + if entry != nil && sameUpdates(before, next) { + return + } + if _, err := st.Put(k.Node, next); err != nil { + k.logf("[pacman] keeper: writing updates.%s: %v", k.Node, err) + } +} + +func sameUpdates(a, b UpdatesValue) bool { + if a.Reachable != b.Reachable || len(a.Packages) != len(b.Packages) { + return false + } + for i := range a.Packages { + if a.Packages[i] != b.Packages[i] { + return false + } + } + return true +} + +// The keeper as a process. + +// ReadKeeperConfig reads the keeper's config file, which the mesh writes from the module's config values. +func ReadKeeperConfig(path string) (KeeperConfig, error) { + var c KeeperConfig + if path == "" { + return c, fmt.Errorf("the keeper was started without %s, its config file", EnvConfig) + } + data, err := os.ReadFile(path) + if err != nil { + return c, fmt.Errorf("reading the keeper's config: %w", err) + } + if err := json.Unmarshal(data, &c); err != nil { + return c, fmt.Errorf("the keeper's config at %s is not its JSON: %w", path, err) + } + return c, nil +} + +// keeper is `pacman-tools keeper`: it reads its bus account and its config, watches the machine, connects and +// runs until stopped. It refuses to start naming what is missing, and never logs a password. +func keeper() int { + logf := func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) } + brokerFile := os.Getenv(EnvBroker) + if brokerFile == "" { + logf("[pacman] keeper was started without %s, its bus account's file", EnvBroker) + return 2 + } + account, err := process.ReadAccount(brokerFile) + if err != nil { + logf("[pacman] keeper: %v", err) + return 1 + } + cfg, err := ReadKeeperConfig(os.Getenv(EnvConfig)) + if err != nil { + logf("[pacman] keeper: %v", err) + return 1 + } + // the watch starts before the first read, so a transaction between the two is not missed. + w, err := WatchMachine(PacmanDBDir, LockName, MirrorDir, MirrorName) + if err != nil { + logf("[pacman] keeper: watching the machine: %v", err) + return 1 + } + defer w.Close() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + k := &Keeper{ + Node: account.Node, Config: cfg, Machine: ThisMachine(), Clock: realClock{}, Watcher: w, Logf: logf, + Locked: func() bool { _, err := os.Stat(filepath.Join(PacmanDBDir, LockName)); return err == nil }, + DBChanged: func() time.Time { return modified(filepath.Join(PacmanDBDir, "local")) }, + MakeHooks: keeperHooks, + } + logf("[pacman] keeper of %s is starting as %s", account.Node, account.User) + if err := runKeeper(ctx, account, k); err != nil && ctx.Err() == nil { + logf("[pacman] keeper stopped: %v", err) + return 1 + } + return 0 +} + +func modified(path string) time.Time { + if fi, err := os.Stat(path); err == nil { + return fi.ModTime() + } + return time.Time{} +} + +// runKeeper connects as the keeper's own account and runs k on that connection until ctx ends. Apart from keeper +// so a test can drive it over a bus of its own. +func runKeeper(ctx context.Context, account process.Account, k *Keeper) error { + conn, err := process.Connect(account) + if err != nil { + return fmt.Errorf("the bus cannot be reached: %w", err) + } + defer conn.Close() + if k.Node == "" { + k.Node = account.Node + } + k.State = func(name string) Store { return conn.State(name) } + k.Events = conn + k.AskRecord = func(id string) (*stdio.StateEntry, error) { return conn.Ask(MessengerAsksBucket, id) } + return k.Run(ctx) +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_bus_test.go b/modules/pacman/cmd/pacman-tools/keeper_bus_test.go new file mode 100644 index 00000000..f337c7e3 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_bus_test.go @@ -0,0 +1,97 @@ +package main + +// What the keeper puts on the bus is only its named fields (novox/hq ADR 0507 §1; acceptance 1840/AC7): pacman's +// output holding command lines, environment variables and messages leaves none of them in a value or an event, +// and no command the keeper runs names a package, a foreign one least of all. (busshapes_test.go holds every +// shape to its named fields.) + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPacmanOutputHoldingCommandLinesAndEnvironmentPutsNeitherOnTheBus(t *testing.T) { + dir := filepath.Join("testdata", "pacman-output-with-command-lines-and-environment") + read := func(name string) string { + b, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + db := &pacmanDB{sl: read("pacman-Sl"), checkup: Ran{Stdout: read("checkupdates")}} + db.set(read("pacman-Q"), read("pacman-Qeq")) + k, states, ev, clock, w := newTestKeeper(db) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- k.Run(ctx) }() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if e, _ := states.store(StateUpdates).Get("node-a"); e != nil { + break + } + } + clock.Advance(time.Minute) + w.c <- LockReleased + w.c <- MirrorsRewritten + time.Sleep(20 * time.Millisecond) + cancel() + <-done + + var onBus []string + for _, name := range []string{StateInstalled, StateUpdates} { + s := states.store(name) + keys, _ := s.Keys() + for _, key := range keys { + e, _ := s.Get(key) + onBus = append(onBus, name+"."+key+" "+string(e.Value)) + } + } + events := ev.Take() + if len(events) == 0 { + t.Fatal("the first write said nothing") + } + for _, e := range events { + body, _ := json.Marshal(e.body) + onBus = append(onBus, e.name+" "+string(body)) + } + if len(onBus) < 3 { + t.Fatalf("on the bus: %v", onBus) + } + whole := strings.Join(onBus, "\n") + for _, never := range []string{"=", "sudo", "--noconfirm", "/", "rm", "warning", "hunter2", "Synchronizing", "HOME", "TOKEN", "SECRET"} { + if strings.Contains(whole, never) { + t.Errorf("%q reached the bus:\n%s", never, whole) + } + } + var inst InstalledValue + e, _ := states.store(StateInstalled).Get("node-a") + json.Unmarshal(e.Value, &inst) + if !samePackages(inst.Packages, []InstalledPackage{{"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"}}) { + t.Errorf("the packages are read, the rest left out: %+v", inst.Packages) + } + var up UpdatesValue + e, _ = states.store(StateUpdates).Get("node-a") + json.Unmarshal(e.Value, &up) + if len(up.Packages) != 1 || up.Packages[0] != (VersionChange{"glibc", "2.42-1", "2.42-2"}) { + t.Errorf("updates: %+v", up) + } + + // no command the keeper ran names a package; the foreign one least of all. + db.mu.Lock() + defer db.mu.Unlock() + for _, c := range db.commands { + switch c { + case "pacman -Q", "pacman -Qeq", "pacman -Sl", "checkupdates": + default: + t.Errorf("the keeper ran %q", c) + } + if strings.Contains(c, "yay") { + t.Errorf("a command named the foreign package: %q", c) + } + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go b/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go new file mode 100644 index 00000000..a2f8c13f --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go @@ -0,0 +1,172 @@ +package main + +// The upgrade parts joined to the keeper's loop (keeperHooks): the log read into runs and old records pruned at +// the start, an ask for this node taken from `asks`, and the records pruned again a day later. + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func TestTheKeeperRunsTheUpgradePartsAtItsStartOnItsAsksAndDaily(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, _, clock, _ := newTestKeeper(db) + log := capturedLog(t) + k.Machine.ReadFile = func(path string) ([]byte, error) { + if path == PacmanLog { + return []byte(log), nil + } + return nil, errNoFile + } + k.MakeHooks = keeperHooks + k.Logf = t.Logf + k.Config = KeeperConfig{RecordDays: 180} + records := states.store(StateRecords) + records.Put(NodeKey("node-a", "20250101T040000Z"), Record{Node: "node-a", Run: "20250101T040000Z"}) + records.Put(NodeKey("node-b", "20250101T040000Z"), Record{Node: "node-b", Run: "20250101T040000Z"}) + done := Run{Node: "node-a", Run: "20261003T040000Z", Kind: RunUpgrade, From: FromAsk, Ask: "ask-1", + Steps: []Step{{Step: StepRefused, At: t0, Why: "a test"}}} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ended := make(chan error, 1) + go func() { ended <- k.Run(ctx) }() + until := func(what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) + } + until("the log read into runs", func() bool { + keys, _ := states.store(StateRuns).Keys() + return len(keys) > 1 + }) + until("the old record pruned at the start", func() bool { + e, _ := records.Get(NodeKey("node-a", "20250101T040000Z")) + return e == nil + }) + if e, _ := records.Get(NodeKey("node-b", "20250101T040000Z")); e == nil { + t.Error("another node's record was pruned") + } + + // a run an ask started, ended; its ask left behind by a keeper that went away. + states.store(StateRuns).Put(NodeKey("node-a", done.Run), done) + asks := states.store(StateAsks) + asks.Put(NodeKey("node-a", "ask-1"), Ask{ID: "ask-1", Node: "node-a", Act: ActUpgrade, At: t0}) + until("the ask a run names deleted", func() bool { + e, _ := asks.Get(NodeKey("node-a", "ask-1")) + return e == nil + }) + + records.Put(NodeKey("node-a", "20250102T040000Z"), Record{Node: "node-a", Run: "20250102T040000Z"}) + time.Sleep(20 * time.Millisecond) + if e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")); e == nil { + t.Fatal("pruned before a day passed") + } + clock.Advance(UpdatesEvery) + until("the old record pruned a day later", func() bool { + e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")) + return e == nil + }) + cancel() + if err := <-ended; err == nil || !strings.Contains(err.Error(), "canceled") { + t.Errorf("Run: %v", err) + } +} + +// roundKeeper is a keeper of node in a round of the made-up nodes order, its machine's time the keeper's clock. +func roundKeeper(t *testing.T, node string, order []string) (*Keeper, *fakeStates, *fakeClock) { + t.Helper() + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, _, clock, _ := newTestKeeper(db) + k.Node = node + k.Machine.Now = clock.Now + k.MakeHooks = keeperHooks + k.Logf = t.Logf + k.Config = KeeperConfig{RecordDays: 180, RoundStartMinutes: 120, UpgradeOrder: order} + k.AskRecord = func(string) (*stdio.StateEntry, error) { return nil, nil } + return k, states, clock +} + +func until(t *testing.T, what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func roundNodeOf(s *fakeStore, round, node string) RoundNode { + var rn RoundNode + if e, _ := s.Get(RoundNodeKey(round, node)); e != nil { + _ = json.Unmarshal(e.Value, &rn) + } + return rn +} + +// 1841/AC6 through the keeper's loop: the first of the order opens the week's round at its start on Saturday +// 04:00 and starts its run in it; the round's parts get the operator channel's reader and the rounds state. +func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) { + k, states, _ := roundKeeper(t, "node-a", []string{"node-a", "node-b"}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = k.Run(ctx) }() + rounds := states.store(StateRounds) + round := RoundID(t0) + until(t, "the round opened", func() bool { e, _ := rounds.Get(round); return e != nil }) + until(t, "node-a's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-a").Run != "" }) + if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" { + t.Errorf("node-b started in node-a's keeper: %+v", rn) + } +} + +// 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on +// the change of `runs` that says so, with no wait for a timer. +func TestTheNextNodesKeeperStartsWhenTheRunBeforeItPassed(t *testing.T) { + k, states, clock := roundKeeper(t, "node-b", []string{"node-a", "node-b"}) + round := RoundID(t0) + rounds, runs := states.store(StateRounds), states.store(StateRuns) + runA := RunID(t0) + rounds.Put(round, Round{Round: round, Opened: t0, OpenedBy: "node-a", Order: []string{"node-a", "node-b"}}) + rounds.Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: runA}) + a := Run{Node: "node-a", Run: runA, Kind: RunUpgrade, From: FromRound, Round: round, + Steps: []Step{{Step: StepRecorded, At: t0}, {Step: StepStarted, At: t0}}} + runs.Put(NodeKey("node-a", runA), a) + clock.Advance(10 * time.Minute) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = k.Run(ctx) }() + time.Sleep(30 * time.Millisecond) + if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" { + t.Fatalf("node-b started before node-a passed: %+v", rn) + } + a.Steps = append(a.Steps, Step{Step: StepFinished, At: clock.Now()}, Step{Step: StepCheckedAfter, At: clock.Now(), Check: &Check{Passed: true}}) + runs.Put(NodeKey("node-a", runA), a) + until(t, "node-b's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-b").Run != "" }) +} + +// The pass of a node is answered from the operator channel's record: the keeper hands its reader on. +func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) { + k, states, _ := roundKeeper(t, "node-a", nil) + k.State = states.State + h := keeperHooks(k) + if k.upgrades == nil || k.upgrades.AskRecord == nil || k.upgrades.Rounds == nil { + t.Fatalf("the round's parts lack the channel's reader or the rounds state: %+v", k.upgrades) + } + if len(h.Wakes) < 2 { + t.Fatalf("no wake for the round: %d wakes", len(h.Wakes)) + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_integration_test.go b/modules/pacman/cmd/pacman-tools/keeper_integration_test.go new file mode 100644 index 00000000..4cb3c2f2 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_integration_test.go @@ -0,0 +1,216 @@ +//go:build integration + +package main + +// The keeper whole over a real bus (novox/hq ADR 0507 §2, §3, §6; acceptance 1840/AC10; the shape of brother's +// device_integration_test.go): a JetStream server of the test's own, the module's state buckets and its event +// stream, and a fake machine. At its start and after a released lock the node's key and package.changed are on +// the bus, and pacman_packages on another instance reads them from there. Compiled by the merge gate's +// `go vet -tags integration`; run by the merge check, never by hand. + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func newTestBus(t *testing.T) string { + t.Helper() + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), + NoLog: true, NoSigs: true} + s, err := server.NewServer(opts) + if err != nil { + t.Fatalf("a bus for this test could not be made: %v", err) + } + go s.Start() + if !s.ReadyForConnections(30 * time.Second) { + s.Shutdown() + t.Fatal("a bus for this test did not come up within 30s") + } + t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) + return s.ClientURL() +} + +// kvReader reads a bucket as a tools bundle's state does (stdio.State): what another instance's verbs see. +type kvReader struct{ kv jetstream.KeyValue } + +func (r kvReader) Get(key string) (*stdio.StateEntry, error) { + e, err := r.kv.Get(context.Background(), key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: json.RawMessage(e.Value()), Revision: e.Revision()}, nil +} + +func (r kvReader) Keys() ([]string, error) { + keys, err := r.kv.Keys(context.Background()) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return []string{}, nil + } + return keys, err +} + +func (r kvReader) History(key string) ([]stdio.StateRevision, error) { + h, err := r.kv.History(context.Background(), key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return []stdio.StateRevision{}, nil + } + if err != nil { + return nil, err + } + out := []stdio.StateRevision{} + for _, e := range h { + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), Op: op, At: e.Created()}) + } + return out, nil +} + +func TestTheKeeperPutsTheNodesPackagesOnARealBusAndAnotherInstanceReadsThem(t *testing.T) { + url := newTestBus(t) + ctx := context.Background() + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + account := process.Account{URL: url, Node: "node-a", Module: "pacman", User: "node-a.pacman.keeper", Password: "pw"} + buckets := map[string]jetstream.KeyValue{} + for _, name := range States { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: account.Module + "_" + name, History: 8}) + if err != nil { + t.Fatal(err) + } + buckets[name] = kv + } + declared, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_declared-packages"}) + if err != nil { + t.Fatal(err) + } + declared.Put(ctx, "node-a", []byte(`{"node":"node-a","packages":[{"package":"zsh","modules":["shell"],"absent":false}]}`)) + stream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "pacman-events", Subjects: []string{"mesh.mod.pacman.event.>"}}) + if err != nil { + t.Fatal(err) + } + + db := &pacmanDB{sl: syncDBs, checkup: Ran{Stdout: "zsh 5.9-1 -> 5.9-2\n"}} + db.set("glibc 2.42-1\nzsh 5.9-1\n", "zsh\n") + signals := make(chan Signal) + k := &Keeper{Config: KeeperConfig{}, Machine: machine(db.runner(), 1000), Clock: realClock{}, + Watcher: fakeWatcher{signals}, Logf: t.Logf, MakeHooks: keeperHooks} + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- runKeeper(runCtx, account, k) }() + + waitKey := func(kv jetstream.KeyValue, key, holds string) string { + t.Helper() + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := kv.Get(ctx, key); err == nil && strings.Contains(string(e.Value()), holds) { + return string(e.Value()) + } + } + t.Fatalf("%s never held %q", key, holds) + return "" + } + waitKey(buckets[StateInstalled], "node-a", `"zsh"`) + waitKey(buckets[StateUpdates], "node-a", `"reachable":true`) + + // a transaction: tmux added, zsh upgraded; the lock released. + db.set("glibc 2.42-1\ntmux 3.5-1\nzsh 5.9-2\n", "tmux\nzsh\n") + db.setUpdates(Ran{Status: 2}) + signals <- LockReleased + waitKey(buckets[StateInstalled], "node-a", `"tmux"`) + waitKey(buckets[StateUpdates], "node-a", `"packages":[]`) + + var changes []PackageChanged + for deadline := time.Now().Add(15 * time.Second); len(changes) < 2 && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + changes = nil + info, err := stream.Info(ctx) + if err != nil { + t.Fatal(err) + } + for seq := uint64(1); seq <= info.State.LastSeq; seq++ { + msg, err := stream.GetMsg(ctx, seq) + if err != nil || msg.Subject != "mesh.mod.pacman.event."+EventPackageChanged { + continue + } + var pc PackageChanged + if err := json.Unmarshal(msg.Data, &pc); err != nil { + t.Fatal(err) + } + changes = append(changes, pc) + } + } + if len(changes) != 2 { + t.Fatalf("package.changed at the start and after the transaction: %+v", changes) + } + last := changes[1] + if last.Node != "node-a" || len(last.Added) != 1 || last.Added[0].Name != "tmux" || len(last.Changed) != 1 || + last.Changed[0] != (VersionChange{"zsh", "5.9-1", "5.9-2"}) || len(last.Removed) != 0 { + t.Errorf("the transaction's package.changed: %+v", last) + } + + // another instance's verbs read what the keeper wrote, asking no machine. + other := machine(func(_ context.Context, name string, args ...string) Ran { + t.Errorf("another instance asked a machine: %s %v", name, args) + return Ran{Status: 99} + }, 1000) + other.Node = "node-b" + other.States = func(name string) StateReader { + if name == DeclaredPackagesState { + return kvReader{declared} + } + return kvReader{buckets[name]} + } + r, err := other.Packages("zsh", 500) + if err != nil { + t.Fatal(err) + } + on := r["nodes"].([]PackageOnNode) + if len(on) != 1 || on[0].Node != "node-a" || on[0].Version != "5.9-2" || strings.Join(on[0].DeclaredBy, ",") != "shell" { + t.Errorf("pacman_packages zsh from another instance: %+v", on) + } + r, err = other.Packages("", 500) + if err != nil { + t.Fatal(err) + } + if v := r["nodes"].([]NodeView); len(v) != 1 || v[0].Packages != 3 || len(v[0].Undeclared) != 1 || v[0].Undeclared[0].Name != "tmux" { + t.Errorf("the view from another instance: %+v", v) + } + u, err := other.UpdatesFor("node-a") + if err != nil { + t.Fatal(err) + } + if v := u["nodes"].([]UpdatesView); len(v) != 1 || !v[0].Reachable || v[0].Count != 0 || v[0].Changed == nil { + t.Errorf("pacman_updates from another instance: %+v", v) + } + + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("runKeeper ended with %v, want its context's end", err) + } + case <-time.After(10 * time.Second): + t.Fatal("runKeeper did not stop once its context was cancelled") + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go new file mode 100644 index 00000000..e0d9db80 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go @@ -0,0 +1,213 @@ +package main + +// The manifest of the keeper (novox/hq ADR 0507 §1-§3, §5; ADR 0508 §1, §2, §5, §6, §8; acceptance 1841/AC12): +// the module's background work is a process of its own under an account the module makes, with a bus account of +// its own, the module's six states and the nine events it emits; the module's config values land in the +// keeper's config file, which restarts it; the weekly timer and its service are gone (a dropped file or service +// is removed by the node-engine); `upgrade-order` falls back to the mesh-level key of that name. + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// rawManifest is the manifest as written, every field kept. +func rawManifest(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func strs(v any) []string { + var out []string + l, _ := v.([]any) + for _, x := range l { + s, _ := x.(string) + out = append(out, s) + } + return out +} + +func TestTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { + m := manifest(t) + for _, r := range m.Resources { + id, _ := r["id"].(string) + path, _ := r["path"].(string) + unit, _ := r["unit"].(string) + if id == "weekly-upgrade" || id == "upgrade-timer" || id == "upgrade-service" || + strings.Contains(path, "weekly-upgrade") || strings.Contains(unit, "weekly-upgrade") { + t.Errorf("the weekly upgrade is replaced by the round (ADR 0508 §5), and %v still declares it", id) + } + } + for _, name := range []string{"mesh-pacman-weekly-upgrade.timer", "mesh-pacman-weekly-upgrade.service"} { + if _, err := os.Stat(filepath.Join(moduleDir, "files", name)); err == nil { + t.Errorf("files/%s is still shipped", name) + } + } +} + +func TestTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn(t *testing.T) { + m := manifest(t) + user := m.resource(t, "keeper-account") + if user["type"] != "user" || user["name"] != "pacman-keeper" || user["shell"] != "/usr/bin/nologin" { + t.Fatalf("the keeper's account: %v", user) + } + k := m.resource(t, "keeper") + if k["type"] != "process" || k["name"] != "pacman-keeper" || k["artifact"] != "tools" || + strings.Join(strs(k["run"]), " ") != "./"+binaryName+" keeper" { + t.Fatalf("the keeper: %v", k) + } + if k["user"] != "pacman-keeper" { + t.Errorf("the keeper runs as its own account, never root (ADR 0508 §8): user %v", k["user"]) + } + env, _ := k["env"].(map[string]any) + if len(env) != 3 || env[EnvBroker] != "${dir:state}/broker-keeper.secret" || env[EnvConfig] != "${dir:state}/keeper.json" || + env[EnvRootSocket] != RootSocketPath { + t.Errorf("the keeper's environment names its account file, its config file and the root helper's socket: %v", env) + } + secrets, _ := rawManifest(t)["own-secrets"].(map[string]any) + broker, _ := secrets["broker-keeper"].(map[string]any) + if broker["path"] != env[EnvBroker] || broker["owner"] != "pacman-keeper" || len(secrets) != 1 { + t.Errorf("the bus account is the keeper's own secret, owned by its account: %v", secrets) + } + state := m.resource(t, "state") + if state["type"] != "directory" || state["place"] != "." || state["unknown"] == nil { + t.Errorf("the state directory: %v", state) + } + order := map[string]int{} + for i, r := range m.Resources { + order[r["id"].(string)] = i + } + if order["keeper-account"] > order["keeper"] || order["state"] > order["keeper-config"] || order["keeper-config"] > order["keeper"] { + t.Errorf("the account, the directory and the config file come before the process: %v", order) + } +} + +func TestTheKeeperDeclaresTheModulesStatesAndEvents(t *testing.T) { + raw := rawManifest(t) + k := manifest(t).resource(t, "keeper") + bus, _ := k["bus"].(map[string]any) + var events []string + for _, e := range Events { + events = append(events, e.Name) + } + if got := strs(bus["state"]); !reflect.DeepEqual(got, States) { + t.Errorf("the keeper's states: %v, want %v", got, States) + } + if got := strs(bus["emits"]); !reflect.DeepEqual(got, events) { + t.Errorf("the keeper's events: %v, want %v", got, events) + } + for key := range bus { + if key != "state" && key != "emits" && key != "asks" { + t.Errorf("the keeper's bus declares %s: it keeps its own states, emits its events and reads its asks' answers, nothing else", key) + } + } + // A pass is answered on the operator's channel: the tools ask, the keeper reads the channel's record of its own + // asks (ADR 0508 §6, ADR 0259, ADR 0436 §1). + if bus["asks"] != true || strings.Join(strs(raw["uses"]), " ") != "operator-channel" { + t.Errorf("the keeper reads its asks' answers and the module uses the operator's channel: %v %v", bus["asks"], raw["uses"]) + } + // The tools read round-start-minutes from the keeper's config file. + for _, a := range manifest(t).Build.Artifacts { + if env, _ := a["env"].(map[string]any); a["name"] == "tools" && env[EnvConfig] != "${dir:state}/keeper.json" { + t.Errorf("the tools bundle is not named the keeper's config file: %v", a["env"]) + } + } + if got := strs(raw["state"]); !reflect.DeepEqual(got, States) { + t.Errorf("the module's states: %v, want %v", got, States) + } + if got := strs(raw["emits"]); !reflect.DeepEqual(got, events) { + t.Errorf("the module's events: %v, want %v", got, events) + } + // The verbs read which packages each node's modules declare; the process reads no other module's state + // (ADR 0507 §5, ADR 0436 §1). + if got := strs(raw["reads"]); !reflect.DeepEqual(got, []string{"mesh-controller.declared-packages"}) { + t.Errorf("reads: %v", got) + } +} + +// configValue is the module's config value of that key. +func configValue(t *testing.T, key string) map[string]any { + t.Helper() + config, _ := rawManifest(t)["config"].(map[string]any) + v, _ := config[key].(map[string]any) + if v == nil { + t.Fatalf("no config value %s", key) + } + if d, _ := v["description"].(string); len(d) < 40 { + t.Errorf("%s says what it means: %q", key, d) + } + return v +} + +func TestTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt(t *testing.T) { + order := configValue(t, "upgrade-order") + if order["type"] != "list" || order["trusted"] != true || order["predefined"] != "upgrade-order" || + !reflect.DeepEqual(order["default"], []any{}) || order["required"] != nil { + t.Errorf("upgrade-order is a trusted list falling back to the mesh-level key, empty by default: %v", order) + } + for key, def := range map[string]float64{"record-days": 180, "round-start-minutes": 120} { + v := configValue(t, key) + if v["type"] != "number" || v["default"] != def || v["minimum"] != float64(1) { + t.Errorf("%s is a number of at least 1, %v by default: %v", key, def, v) + } + } + config, _ := rawManifest(t)["config"].(map[string]any) + if len(config) != 3 { + t.Errorf("config: %v", config) + } + m := manifest(t) + f := m.resource(t, "keeper-config") + if f["type"] != "file" || f["path"] != "${dir:state}/keeper.json" || f["merge"] != "json" || f["trusted"] != true || f["mode"] != "0644" { + t.Fatalf("the keeper's config file: %v", f) + } + if got := strs(f["takes-config"]); strings.Join(got, " ") != "upgrade-order record-days round-start-minutes" { + t.Errorf("the file takes every config value: %v", got) + } + if got := strs(m.resource(t, "keeper")["restart-on"]); strings.Join(got, " ") != "keeper-config" { + t.Errorf("a changed config file restarts the keeper, which reads it at its start: %v", got) + } + // The template is the file a node starts with: the defaults, read as the keeper reads them. + dec := json.NewDecoder(bytes.NewReader([]byte(templateText(t, f)))) + dec.DisallowUnknownFields() + var c KeeperConfig + if err := dec.Decode(&c); err != nil { + t.Fatalf("the keeper cannot read its config file: %v", err) + } + if c.UpgradeOrder == nil || len(c.UpgradeOrder) != 0 || c.RecordDays != 180 || c.RoundStartMinutes != 120 { + t.Errorf("the template holds the defaults: %+v", c) + } +} + +func TestUpgradeOrderIsATrustedListOfTheMesh(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "..", "..", "predefined.json")) + if err != nil { + t.Fatal(err) + } + var keys map[string]map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + k := keys["upgrade-order"] + if k["type"] != "list" || k["trusted"] != true { + t.Fatalf("predefined upgrade-order: %v", k) + } + if d, _ := k["description"].(string); !strings.Contains(d, "order") || !strings.Contains(d, "round") { + t.Errorf("its description says the nodes' order and the round: %q", d) + } + if k["type"] != configValue(t, "upgrade-order")["type"] { + t.Error("a value falls back only to a key of its own type (ADR 0348 §4)") + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_test.go b/modules/pacman/cmd/pacman-tools/keeper_test.go new file mode 100644 index 00000000..21ddf094 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_test.go @@ -0,0 +1,630 @@ +package main + +// The keeper's start and its lock watch (novox/hq ADR 0507 §2, §3; acceptance 1840/AC1-AC3), over a fake state, +// a fake machine, a fake clock and a fake watch. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// fakeStore is one state in memory, recording every write. +type fakeStore struct { + mu sync.Mutex + name string + values map[string]json.RawMessage + revs map[string]uint64 + at map[string]time.Time + rev uint64 + writes []string // "put ", "delete " + watchers []fakeWatch + now func() time.Time +} + +type fakeWatch struct { + key string + handle func(stdio.StateChange) error +} + +func newFakeStore(name string) *fakeStore { + return &fakeStore{name: name, values: map[string]json.RawMessage{}, revs: map[string]uint64{}, at: map[string]time.Time{}} +} + +func (s *fakeStore) Get(key string) (*stdio.StateEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: v, Revision: s.revs[key]}, nil +} + +func (s *fakeStore) write(key string, value any, op string) (uint64, error) { + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + s.mu.Lock() + s.rev++ + s.values[key], s.revs[key], s.at[key] = data, s.rev, s.clock() + s.writes = append(s.writes, op+" "+key) + rev, ws := s.rev, append([]fakeWatch(nil), s.watchers...) + s.mu.Unlock() + for _, w := range ws { + if matchKey(w.key, key) { + w.handle(stdio.StateChange{State: s.name, Key: key, Op: "put", Value: data, Revision: rev}) + } + } + return rev, nil +} + +// clock is when a write lands: the store's own time, which a test sets. +func (s *fakeStore) clock() time.Time { + if s.now != nil { + return s.now() + } + return t0 +} + +func (s *fakeStore) Put(key string, value any) (uint64, error) { return s.write(key, value, "put") } + +func (s *fakeStore) Create(key string, value any) (uint64, error) { + if e, _ := s.Get(key); e != nil { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return s.write(key, value, "create") +} + +func (s *fakeStore) Update(key string, value any, revision uint64) (uint64, error) { + if e, _ := s.Get(key); e == nil || e.Revision != revision { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return s.write(key, value, "update") +} + +func (s *fakeStore) Delete(key string) error { + s.mu.Lock() + delete(s.values, key) + delete(s.revs, key) + s.rev++ + s.writes = append(s.writes, "delete "+key) + rev, ws := s.rev, append([]fakeWatch(nil), s.watchers...) + s.mu.Unlock() + for _, w := range ws { + if matchKey(w.key, key) { + w.handle(stdio.StateChange{State: s.name, Key: key, Op: "delete", Revision: rev}) + } + } + return nil +} + +func (s *fakeStore) Keys() ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + keys := []string{} + for k := range s.values { + keys = append(keys, k) + } + sortStrings(keys) + return keys, nil +} + +func (s *fakeStore) Watch(key string, handle func(stdio.StateChange) error) error { + keys, _ := s.Keys() + for _, k := range keys { + if !matchKey(key, k) { + continue + } + e, _ := s.Get(k) + if err := handle(stdio.StateChange{State: s.name, Key: k, Op: "put", Value: e.Value, Revision: e.Revision, Current: true}); err != nil { + return err + } + } + s.mu.Lock() + s.watchers = append(s.watchers, fakeWatch{key, handle}) + s.mu.Unlock() + return nil +} + +// Writes is what was written, in order, and forgets it. +func (s *fakeStore) Writes() []string { + s.mu.Lock() + defer s.mu.Unlock() + w := s.writes + s.writes = nil + return w +} + +// matchKey narrows as the bus does: `*` one name, `**` the rest, empty every key. +func matchKey(pattern, key string) bool { + if pattern == "" { + return true + } + p, k := strings.Split(pattern, "."), strings.Split(key, ".") + for i := range p { + if p[i] == "**" { + return len(k) > i + } + if i >= len(k) || (p[i] != "*" && p[i] != k[i]) { + return false + } + } + return len(p) == len(k) +} + +func sortStrings(s []string) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } +} + +// fakeStates is the module's states in memory. +type fakeStates struct { + mu sync.Mutex + states map[string]*fakeStore +} + +func (f *fakeStates) State(name string) Store { return f.store(name) } + +func (f *fakeStates) store(name string) *fakeStore { + f.mu.Lock() + defer f.mu.Unlock() + if f.states == nil { + f.states = map[string]*fakeStore{} + } + if f.states[name] == nil { + f.states[name] = newFakeStore(name) + } + return f.states[name] +} + +// emitted is the events a keeper emitted. +type emitted struct { + mu sync.Mutex + events []emittedEvent +} + +type emittedEvent struct { + name string + body any + id string +} + +func (e *emitted) Emit(name string, body any, opts ...stdio.EmitOptions) error { + e.mu.Lock() + defer e.mu.Unlock() + id := "" + for _, o := range opts { + id = o.MessageID + } + e.events = append(e.events, emittedEvent{name, body, id}) + return nil +} + +func (e *emitted) Take() []emittedEvent { + e.mu.Lock() + defer e.mu.Unlock() + out := e.events + e.events = nil + return out +} + +// fakeClock moves only when the test moves it; After fires once the clock passes it. +type fakeClock struct { + mu sync.Mutex + now time.Time + timers []fakeTimer +} + +type fakeTimer struct { + at time.Time + c chan time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) After(d time.Duration) <-chan time.Time { + c.mu.Lock() + defer c.mu.Unlock() + t := fakeTimer{c.now.Add(d), make(chan time.Time, 1)} + if d <= 0 { + t.c <- c.now + return t.c + } + c.timers = append(c.timers, t) + return t.c +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) + kept := c.timers[:0] + for _, t := range c.timers { + if !t.at.After(c.now) { + t.c <- c.now + continue + } + kept = append(kept, t) + } + c.timers = kept +} + +// fakeWatcher hands the test's signals to the keeper. +type fakeWatcher struct{ c chan Signal } + +func (w fakeWatcher) Signals() <-chan Signal { return w.c } +func (w fakeWatcher) Close() error { return nil } + +// pacmanDB is a node's package database as pacman answers it: the test changes it between reads. +type pacmanDB struct { + mu sync.Mutex + q, qe string + sl string + checkup Ran + commands []string +} + +func (d *pacmanDB) set(q, qe string) { + d.mu.Lock() + defer d.mu.Unlock() + d.q, d.qe = q, qe +} + +func (d *pacmanDB) setUpdates(r Ran) { + d.mu.Lock() + defer d.mu.Unlock() + d.checkup = r +} + +func (d *pacmanDB) runner() Runner { + return func(_ context.Context, name string, args ...string) Ran { + d.mu.Lock() + defer d.mu.Unlock() + line := strings.TrimSpace(name + " " + strings.Join(args, " ")) + d.commands = append(d.commands, line) + switch line { + case "pacman -Q": + return Ran{Stdout: d.q} + case "pacman -Qeq": + return Ran{Stdout: d.qe} + case "pacman -Sl": + return Ran{Stdout: d.sl} + case "checkupdates": + return d.checkup + } + return Ran{Status: 99, Stderr: "unexpected command: " + line} + } +} + +var syncDBs = "core glibc 2.42-1 [installed]\ncore linux 6.17.1-1 [installed]\nextra zsh 5.9-2 [installed: 5.9-1]\n" + + "extra zsh-completions 0.35-1\nextra tmux 3.5-1\nextra-testing zsh 5.9-3\n" + +var t0 = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + +func newTestKeeper(db *pacmanDB) (*Keeper, *fakeStates, *emitted, *fakeClock, fakeWatcher) { + states, ev, clock := &fakeStates{}, &emitted{}, &fakeClock{now: t0} + w := fakeWatcher{make(chan Signal)} + m := machine(db.runner(), 1000) + k := &Keeper{Node: "node-a", Machine: m, State: states.State, Events: ev, Clock: clock, Watcher: w, + Logf: func(string, ...any) {}} + return k, states, ev, clock, w +} + +func installedOf(t *testing.T, s *fakeStore, key string) InstalledValue { + t.Helper() + e, _ := s.Get(key) + if e == nil { + t.Fatalf("no %s", key) + } + var v InstalledValue + if err := json.Unmarshal(e.Value, &v); err != nil { + t.Fatal(err) + } + return v +} + +// 1840/AC1: at the start every package is written with its version, its reason and its repository or foreign, +// and written again only when it differs from the key. +func TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\nlinux 6.17.1-1\nyay 12.0-1\nzsh 5.9-1\n", "linux\nyay\nzsh\n") + k, states, ev, _, _ := newTestKeeper(db) + k.DBChanged = func() time.Time { return t0.Add(-time.Hour) } + k.refreshInstalled(k.DBChanged()) + inst := states.store(StateInstalled) + if w := inst.Writes(); strings.Join(w, ",") != "put node-a" { + t.Fatalf("writes: %v", w) + } + v := installedOf(t, inst, "node-a") + want := []InstalledPackage{ + {"glibc", "2.42-1", false, "core"}, {"linux", "6.17.1-1", true, "core"}, + {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"}, + } + if !samePackages(v.Packages, want) || !v.At.Equal(t0.Add(-time.Hour)) { + t.Fatalf("installed.node-a: %+v", v) + } + if e := ev.Take(); len(e) != 1 || len(e[0].body.(PackageChanged).Added) != 4 { + t.Fatalf("the first write adds every package: %+v", e) + } + // a second start over the same database writes nothing and says nothing, whatever the time. + k.refreshInstalled(t0) + if w := inst.Writes(); len(w) != 0 { + t.Errorf("the same packages were written again: %v", w) + } + if e := ev.Take(); len(e) != 0 { + t.Errorf("the same packages were said again: %+v", e) + } +} + +// 1840/AC2: a transaction that adds, removes and changes packages is one package.changed naming the node and +// exactly those; one that changes nothing writes and emits nothing; a lock held again waits for its release. +func TestAReleasedLockSaysExactlyWhatTheTransactionChangedOnce(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\nlinux 6.17.1-1\nzsh 5.9-1\n", "linux\nzsh\n") + k, states, ev, clock, _ := newTestKeeper(db) + k.refreshInstalled(t0) + ev.Take() + states.store(StateInstalled).Writes() + + held := false + k.Locked = func() bool { return held } + db.set("glibc 2.42-1\nlinux 6.17.2-1\ntmux 3.5-1\n", "linux\ntmux\n") + held = true + k.signal(LockReleased) + if w := states.store(StateInstalled).Writes(); len(w) != 0 { + t.Fatalf("read while the lock was held again: %v", w) + } + held = false + clock.Advance(time.Minute) + k.signal(LockReleased) + if w := states.store(StateInstalled).Writes(); strings.Join(w, ",") != "put node-a" { + t.Fatalf("writes: %v", w) + } + e := ev.Take() + if len(e) != 1 || e[0].name != EventPackageChanged { + t.Fatalf("events: %+v", e) + } + got := e[0].body.(PackageChanged) + if got.Node != "node-a" || !got.At.Equal(t0.Add(time.Minute)) || + fmt.Sprint(got.Added) != "[{tmux 3.5-1}]" || fmt.Sprint(got.Removed) != "[{zsh 5.9-1}]" || + fmt.Sprint(got.Changed) != "[{linux 6.17.1-1 6.17.2-1}]" { + t.Fatalf("package.changed: %+v", got) + } + if !strings.HasPrefix(e[0].id, "installed.node-a.") { + t.Errorf("the event's id is the key's revision, so a second emit is dropped: %q", e[0].id) + } + // a transaction that changed nothing (a database sync alone). + k.signal(LockReleased) + if w := states.store(StateInstalled).Writes(); len(w) != 0 { + t.Errorf("nothing changed and the key was written: %v", w) + } + if e := ev.Take(); len(e) != 0 { + t.Errorf("nothing changed and it was said: %+v", e) + } + // a change of reason alone rewrites the key and is no package.changed. + db.set("glibc 2.42-1\nlinux 6.17.2-1\ntmux 3.5-1\n", "glibc\nlinux\ntmux\n") + k.signal(LockReleased) + if w := states.store(StateInstalled).Writes(); len(w) != 1 { + t.Errorf("a changed reason is a changed key: %v", w) + } + if e := ev.Take(); len(e) != 0 { + t.Errorf("a changed reason is no change of a version: %+v", e) + } +} + +// bigDB is a package database whose value is over the bus's cap. +func bigDB(n int) string { + var b strings.Builder + for i := 0; i < n; i++ { + fmt.Fprintf(&b, "package-with-a-long-name-%05d 1.0.0-1\n", i) + } + return b.String() +} + +// 1840/AC3: a value over the bus's cap is not written: the key keeps its last value and `.not-current` +// holds the size and since when, written once; the next write that fits deletes it. +func TestAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, ev, clock, _ := newTestKeeper(db) + k.refreshInstalled(t0) + inst := states.store(StateInstalled) + inst.Writes() + ev.Take() + + db.set(bigDB(5000), "") + clock.Advance(time.Minute) + k.signal(LockReleased) + if w := inst.Writes(); strings.Join(w, ",") != "put node-a.not-current" { + t.Fatalf("writes: %v", w) + } + if v := installedOf(t, inst, "node-a"); len(v.Packages) != 1 { + t.Fatalf("the last value is kept: %d packages", len(v.Packages)) + } + if e := ev.Take(); len(e) != 0 { + t.Errorf("nothing was written and it was said: %+v", e) + } + e, _ := inst.Get("node-a.not-current") + var nc NotCurrent + json.Unmarshal(e.Value, &nc) + if nc.Bytes <= ValueCap || !nc.Since.Equal(t0.Add(time.Minute)) { + t.Fatalf("not-current: %+v", nc) + } + // still over: the mark is written once. + db.set(bigDB(5001), "") + clock.Advance(time.Minute) + k.signal(LockReleased) + if w := inst.Writes(); len(w) != 0 { + t.Errorf("the mark was written again: %v", w) + } + // back under the cap: written, and the mark deleted. + db.set("glibc 2.42-1\nzsh 5.9-1\n", "") + k.signal(LockReleased) + if w := inst.Writes(); strings.Join(w, ",") != "put node-a,delete node-a.not-current" { + t.Errorf("writes: %v", w) + } +} + +// The mark goes too when the node's packages come back to what the key holds: the key is current again. +func TestTheMarkGoesWhenThePackagesComeBackToTheKey(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, _, _, _ := newTestKeeper(db) + k.refreshInstalled(t0) + db.set(bigDB(5000), "") + k.signal(LockReleased) + inst := states.store(StateInstalled) + inst.Writes() + db.set("glibc 2.42-1\n", "") + k.signal(LockReleased) + if w := inst.Writes(); strings.Join(w, ",") != "delete node-a.not-current" { + t.Errorf("writes: %v", w) + } +} + +// The loop: a released lock is read, a hook runs after it with what it changed, a watched state's change reaches +// its hook, and a wake comes at the time it asked. +func TestTheLoopRunsTheHooksOnItsOneGoroutine(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, _, clock, w := newTestKeeper(db) + var mu sync.Mutex + var seen []string + say := func(s string) { mu.Lock(); seen = append(seen, s); mu.Unlock() } + k.Hooks = Hooks{ + Start: []func(*Keeper) error{func(*Keeper) error { say("start"); return nil }}, + AfterTransaction: []func(*Keeper, Transaction) error{func(_ *Keeper, tr Transaction) error { + say(fmt.Sprintf("transaction changed=%v", tr.Changed != nil)) + return nil + }}, + Changes: []StateHook{{State: StateAsks, Key: "node-a.*", Handle: func(_ *Keeper, c stdio.StateChange) error { + say(fmt.Sprintf("%s %s current=%v", c.Op, c.Key, c.Current)) + return nil + }}}, + Wakes: []Wake{func(_ *Keeper, now time.Time) time.Time { + if now.Equal(t0) { + return t0.Add(time.Hour) + } + say("wake " + now.Sub(t0).String()) + return time.Time{} + }}, + } + states.store(StateAsks).Put("node-a.ask-1", Ask{ID: "ask-1"}) + states.store(StateAsks).Put("node-b.ask-2", Ask{ID: "ask-2"}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- k.Run(ctx) }() + waitFor(t, &mu, &seen, "start") + db.set("glibc 2.42-1\nzsh 5.9-1\n", "") + w.c <- LockReleased + waitFor(t, &mu, &seen, "transaction changed=true") + states.store(StateAsks).Put("node-a.ask-3", Ask{ID: "ask-3"}) + waitFor(t, &mu, &seen, "put node-a.ask-3 current=false") + clock.Advance(time.Hour) + waitFor(t, &mu, &seen, "wake 1h0m0s") + cancel() + if err := <-done; err != context.Canceled { + t.Errorf("Run ended with %v", err) + } + mu.Lock() + defer mu.Unlock() + if strings.Join(seen, "|") != "put node-a.ask-1 current=true|start|transaction changed=true|put node-a.ask-3 current=false|wake 1h0m0s" && + strings.Join(seen, "|") != "start|put node-a.ask-1 current=true|transaction changed=true|put node-a.ask-3 current=false|wake 1h0m0s" { + t.Errorf("seen: %v", seen) + } +} + +func waitFor(t *testing.T, mu *sync.Mutex, seen *[]string, want string) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + mu.Lock() + for _, s := range *seen { + if s == want { + mu.Unlock() + return + } + } + mu.Unlock() + } + t.Fatalf("never saw %q", want) +} + +// The watch ending ends the keeper, loudly. +func TestTheKeeperStopsWhenTheWatchEnds(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + k, _, _, _, w := newTestKeeper(db) + close(w.c) + if err := k.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "watch ended") { + t.Errorf("Run: %v", err) + } +} + +// The machine's watch: the lock deleted and the mirror list written or moved in are each a signal; anything else +// in those directories is none. +func TestTheWatchSeesTheLockReleasedAndTheMirrorListRewritten(t *testing.T) { + lockDir, mirrorDir := t.TempDir(), t.TempDir() + w, err := WatchMachine(lockDir, LockName, mirrorDir, MirrorName) + if err != nil { + t.Fatal(err) + } + next := func() Signal { + select { + case s := <-w.Signals(): + return s + case <-time.After(200 * time.Millisecond): + return 0 + } + } + lock := filepath.Join(lockDir, LockName) + os.WriteFile(lock, nil, 0o644) + os.WriteFile(filepath.Join(lockDir, "other"), nil, 0o644) + os.Remove(filepath.Join(lockDir, "other")) + if s := next(); s != 0 { + t.Fatalf("a lock taken, and another file, are no signal: %v", s) + } + os.Remove(lock) + if s := next(); s != LockReleased { + t.Fatalf("the lock deleted: %v", s) + } + os.WriteFile(filepath.Join(mirrorDir, MirrorName), []byte("Server = x\n"), 0o644) + if s := next(); s != MirrorsRewritten { + t.Fatalf("the list written: %v", s) + } + tmp := filepath.Join(mirrorDir, "mirrorlist.new") + os.WriteFile(tmp, []byte("Server = y\n"), 0o644) + if s := next(); s != 0 { + t.Fatalf("another file written: %v", s) + } + os.Rename(tmp, filepath.Join(mirrorDir, MirrorName)) + if s := next(); s != MirrorsRewritten { + t.Fatalf("the list moved in: %v", s) + } + w.Close() + for deadline := time.After(2 * time.Second); ; { + select { + case _, open := <-w.Signals(): + if !open { + return + } + case <-deadline: + t.Fatal("closing the watch did not end its signals") + } + } +} diff --git a/modules/pacman/cmd/pacman-tools/machine.go b/modules/pacman/cmd/pacman-tools/machine.go index 691a19d9..adc85428 100644 --- a/modules/pacman/cmd/pacman-tools/machine.go +++ b/modules/pacman/cmd/pacman-tools/machine.go @@ -22,6 +22,8 @@ import ( "os/exec" "strings" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // Ran is what one command did: its output, its exit status, and why it never ran to an answer. @@ -105,6 +107,17 @@ type Machine struct { ReadFile func(path string) ([]byte, error) Now func() time.Time Sleep func(time.Duration) + // Node is the node this bundle runs on (the runtime's MESH_NODE), and States reads the module's states and + // those it reads (the manifest's `reads`), for the verbs that answer from the bus (novox/hq ADR 0507 §6). + Node string + States func(name string) StateReader +} + +// StateReader is a state as a verb reads it: stdio.State, or a fake in a test. +type StateReader interface { + Get(key string) (*stdio.StateEntry, error) + Keys() ([]string, error) + History(key string) ([]stdio.StateRevision, error) } // ThisMachine is the machine the runtime launched this bundle on. @@ -117,7 +130,8 @@ func ThisMachine() *Machine { if account == "" { account = user } - return &Machine{Run: ExecRunner, UID: os.Getuid(), User: user, Account: account, ReadFile: os.ReadFile, Now: time.Now, Sleep: time.Sleep} + return &Machine{Run: ExecRunner, UID: os.Getuid(), User: user, Account: account, ReadFile: os.ReadFile, Now: time.Now, Sleep: time.Sleep, + Node: os.Getenv("MESH_NODE"), States: func(name string) StateReader { return stdio.State(name) }} } // Out runs a command that only reads, and fails with what went wrong named. diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index c4d7136a..1df5256b 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "os" + "regexp" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -20,6 +21,19 @@ const binaryName = "pacman-tools" func bg() context.Context { return context.Background() } func main() { + // `pacman-tools keeper` is the module's process (keeper.go), `root` its root helper (root.go); without a + // word it serves the tools. + if len(os.Args) > 1 { + switch os.Args[1] { + case "keeper": + os.Exit(keeper()) + case "root": + // the root helper (root.go, ADR 0510), and the command of a rollback's unit it starts + os.Exit(rootHelper()) + case "root-rollback": + os.Exit(rootRollback(os.Args[2:])) + } + } // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): pacman. if err := stdio.Serve("", tools(ThisMachine())); err != nil { fmt.Fprintln(os.Stderr, err) @@ -33,8 +47,22 @@ var ( return map[string]any{"type": "integer", "description": fmt.Sprintf("at most this many (default %d, at most %d)", def, most)} } sinceArg = map[string]any{"type": "string", "description": "a day, YYYY-MM-DD"} + nodeArg = map[string]any{"type": "string", "description": "a node's name"} + nodeName = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) ) +// nodeOf is the node argument, empty when absent; a node's name is one key token. +func nodeOf(args map[string]any) (string, error) { + n, err := text(args, "node", false) + if err != nil { + return "", err + } + if n != "" && !nodeName.MatchString(n) { + return "", fmt.Errorf("node %q is not a node's name", n) + } + return n, nil +} + func tools(m *Machine) []stdio.Tool { return []stdio.Tool{ { @@ -66,15 +94,22 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_installed", - Description: "Installed packages with version, why each is installed (explicit or dependency) and whether it is foreign (in no repository); narrowed by name, reason or foreign; with totals.", + Name: "pacman_installed", + Description: "A node's installed packages, from the bus (ADR 0507 §2): version, why each is installed (explicit or dependency), its repository, " + + "or foreign (in no repository); narrowed by name, reason or foreign; with totals, the time of the transaction they describe and whether they are current. " + + "This node's by default. (r)", Input: schema(map[string]any{ + "node": nodeArg, "match": map[string]any{"type": "string", "description": "only names holding this"}, "reason": map[string]any{"type": "string", "enum": []string{"explicit", "dependency"}}, "foreign": map[string]any{"type": "boolean", "description": "only foreign packages"}, "limit": limitArg(5000, 20000), }), Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } match, err := text(args, "match", false) if err != nil { return nil, err @@ -94,7 +129,29 @@ func tools(m *Machine) []stdio.Tool { if err != nil { return nil, err } - return m.Installed(match, reason, foreign, n) + return m.InstalledFrom(node, match, reason, foreign, n) + }, + }, + { + Name: "pacman_packages", + Description: "Every node's packages at once, from the bus (ADR 0507 §6): with package, that package on every node (version, reason, repository " + + "or foreign, the modules declaring it); without, per node its counts, the packages installed explicitly that no module assigned to it declares, " + + "and its foreign packages apart, then the packages whose versions differ between nodes and the packages on some nodes only. " + + "A node whose packages are over the bus's cap is marked not current. No machine is asked. (r)", + Input: schema(map[string]any{ + "package": map[string]any{"type": "string", "description": "one package, on every node"}, + "limit": limitArg(500, 20000), + }), + Run: func(args map[string]any) (any, error) { + p, err := text(args, "package", false) + if err != nil { + return nil, err + } + n, err := whole(args, "limit", 500, 1, 20000) + if err != nil { + return nil, err + } + return m.Packages(p, n) }, }, { @@ -126,18 +183,29 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_updates", - Description: "What a full upgrade would change, each package from and to (checkupdates: the repositories are asked into a copy of their databases, so asking never sets up a partial upgrade). Needs the network.", - Input: schema(map[string]any{}), - Run: func(map[string]any) (any, error) { return m.Updates() }, + Name: "pacman_updates", + Description: "What a full upgrade would change, from the bus (ADR 0507 §4), on one node or on every node: each package from and to, whether the " + + "node reached its mirrors (false: the list is the last it could confirm), and when the node's list last changed. The keeper reads it with " + + "checkupdates at its start, after every transaction, after every mirror refresh and once a day. (r)", + Input: schema(map[string]any{"node": nodeArg}), + Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } + return m.UpdatesFor(node) + }, }, { Name: "pacman_upgrade", - Description: "Start a full system upgrade (pacman -Syu --noconfirm, through sudo -n) as a transient unit of its own that outlives the call, " + - "answering at once with the unit and the distribution's news since the last upgrade — read the news first. Given that unit, " + - "answer whether it is running, finished and succeeded, with the tail of its log.", + Description: "Ask for a full system upgrade of a node (this one unless named): the ask is written to the bus, where that node's keeper " + + "takes it and runs it as a run: the package record, the kernel check, the transaction as a unit of its own, and the after-upgrade " + + "check (ADR 0508 §8). Answers at once with the ask and the distribution's news since the last upgrade — read the news first; " + + "pacman_upgrades with the node shows the run that names the ask. Given a transaction unit, answers whether it is running, " + + "finished and succeeded, with the tail of its log.", Input: schema(map[string]any{ - "unit": map[string]any{"type": "string", "description": "a unit this tool started, to read how it goes (optional)"}, + "node": map[string]any{"type": "string", "description": "the node to upgrade (default: this one)"}, + "unit": map[string]any{"type": "string", "description": "a transaction unit, to read how it goes (optional)"}, "lines": limitArg(60, 400), }), Run: func(args map[string]any) (any, error) { @@ -149,15 +217,127 @@ func tools(m *Machine) []stdio.Tool { if err != nil { return nil, err } - return m.Upgrade(unit, n) + if unit != "" { + return m.UpgradeStatus(unit, n) + } + node, err := text(args, "node", false) + if err != nil { + return nil, err + } + if node == "" { + node = thisNode() + } + a, err := AskUpgrade(upgradeState(StateAsks), node, m.Now()) + if err != nil { + return nil, err + } + return map[string]any{"asked": a, "follow": "pacman_upgrades with node " + node + " lists the run that names this ask", "news": m.News("")}, nil }, }, { Name: "pacman_upgrade_health", - Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " + - "with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)", + Description: "This node's upgrades, read from the bus, as the module's health check (ADR 0508 §11): unhealthy while no upgrade-order " + + "is set, while the node's latest run is refused, failed or did not pass its after-upgrade check, hung past two hours, or held a " + + "kernel, and while the week's round stands still at this node or at the node it waits for, naming it and since when. (r)", Input: schema(map[string]any{}), - Run: func(map[string]any) (any, error) { return m.UpgradeHealth(), nil }, + Run: func(map[string]any) (any, error) { + return m.UpgradeHealth(), nil + }, + }, + { + Name: "pacman_upgrades", + Description: "Every full upgrade and rollback of every node, from the bus (ADR 0508 §1): each run's node, kind, what started it, " + + "its round, when it began and ended, its outcome (refused, finished, failed or running) and whether its after-upgrade check passed; " + + "narrowed to a node and to days, newest first. With a node and a run, that run whole, every step with its time.", + Input: schema(map[string]any{ + "node": map[string]any{"type": "string", "description": "only this node's runs"}, + "from": sinceArg, + "until": map[string]any{"type": "string", "description": "a day, YYYY-MM-DD, the last one listed"}, + "run": map[string]any{"type": "string", "description": "one run, by its id, with node"}, + "limit": limitArg(50, 1000), + }), + Run: func(args map[string]any) (any, error) { + var v [4]string + for i, k := range []string{"node", "from", "until", "run"} { + s, err := text(args, k, false) + if err != nil { + return nil, err + } + v[i] = s + } + n, err := whole(args, "limit", 50, 1, 1000) + if err != nil { + return nil, err + } + return ListUpgrades(upgradeState(StateRuns), v[0], v[1], v[2], v[3], n) + }, + }, + { + Name: "pacman_rollback", + Description: "Return a node to the package record a run took (ADR 0508 §7). Without apply it says what that would change: each package " + + "from its version now to the record's and where that version comes from (the node's package cache, the Arch Linux Archive's copy " + + "of that exact version, or the archive's repository on the record's date), the packages installed since (removed only with " + + "remove_added), and no foreign package. With apply it asks the node's keeper to carry it out as a run of its own, followed by " + + "the after-upgrade check; pacman_upgrades follows it. This node's by default.", + Input: schema(map[string]any{ + "node": nodeArg, + "run": map[string]any{"type": "string", "description": "the run whose record to return to (pacman_upgrades lists them)"}, + "remove_added": map[string]any{"type": "boolean", "description": "also remove the packages installed since the record"}, + "apply": map[string]any{"type": "boolean", "description": "carry it out"}, + }, "run"), + Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } + run, err := text(args, "run", true) + if err != nil { + return nil, err + } + if !nodeName.MatchString(run) { + return nil, fmt.Errorf("run %q is not a run's id", run) + } + remove, err := flag(args, "remove_added") + if err != nil { + return nil, err + } + apply, err := flag(args, "apply") + if err != nil { + return nil, err + } + return m.Rollback(node, run, remove, apply) + }, + }, + { + Name: "pacman_round", + Description: "Where the week's upgrade rounds stand (ADR 0508 §5, §6): each round's order and opening, and every node's state in it: " + + "waiting (for the node named), turn, stalled (its turn came longer than round-start-minutes ago), running, passed, failed " + + "(the round stops there) or refused, with since when. With pass, a node and a reason, asks the operator on their channel to let " + + "the round go on past that node; it goes on only on the operator's Approve.", + Input: schema(map[string]any{ + "round": map[string]any{"type": "string", "description": "one round, as 2026-W43 (default: the newest)"}, + "pass": map[string]any{"type": "string", "description": "a node to ask the operator to pass"}, + "reason": map[string]any{"type": "string", "description": "why the node should be passed: required with pass, and shown to the operator"}, + "limit": limitArg(4, 52), + }), + Run: func(args map[string]any) (any, error) { + var v [3]string + for i, k := range []string{"round", "pass", "reason"} { + s, err := text(args, k, false) + if err != nil { + return nil, err + } + v[i] = s + } + if v[1] != "" { + return AskPass(upgradeState(StateRounds), upgradeState(StateRuns), upgradeState(StateAsks), v[0], v[1], v[2], m.Now()) + } + n, err := whole(args, "limit", 4, 1, 52) + if err != nil { + return nil, err + } + return ListRounds(upgradeState(StateRounds), upgradeState(StateRuns), v[0], m.Now(), roundStartMinutes(), n) + }, }, { Name: "pacman_orphans", @@ -258,10 +438,17 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_foreign", - Description: "Installed packages that no repository this machine syncs carries (pacman -Qm): built from the AUR or by hand, which the mesh cannot install.", - Input: schema(map[string]any{}), - Run: func(map[string]any) (any, error) { return m.Foreign() }, + Name: "pacman_foreign", + Description: "A node's packages that no repository it syncs carries, from the bus: built from the AUR or by hand, which the mesh neither " + + "upgrades nor removes (ADR 0507 §7). This node's by default. (r)", + Input: schema(map[string]any{"node": nodeArg}), + Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } + return m.ForeignFrom(node) + }, }, { Name: "pacman_news", diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index da7fb4ad..b55f6908 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -101,53 +101,3 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { t.Fatalf("a package's file and timer come after the package: %v", order) } } - -// The weekly upgrade (issue 1574, ADR 0477 clause 5): a timer on Saturday 04:00 local time, persistent so a -// node that was off upgrades at its next start, the same moment on every node (no random delay), starting a -// oneshot service that runs the very upgrade pacman_upgrade runs. -func TestTheWeeklyUpgradeIsATimerOnSaturdayFourPersistentRunningTheToolsUpgrade(t *testing.T) { - m := manifest(t) - timer := m.resource(t, "upgrade-timer") - service := m.resource(t, "upgrade-service") - if timer["path"] != "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer" || service["path"] != "/etc/systemd/system/"+WeeklyUnit { - t.Fatalf("paths: %v %v", timer["path"], service["path"]) - } - tt, st := templateText(t, timer), templateText(t, service) - for _, want := range []string{"\nOnCalendar=Sat *-*-* 04:00:00\n", "\nPersistent=true\n", "\nWantedBy=timers.target\n"} { - if !strings.Contains(tt, want) { - t.Errorf("the timer lacks %q:\n%s", strings.TrimSpace(want), tt) - } - } - for _, not := range []string{"RandomizedDelaySec", "OnBootSec", "OnUnitActiveSec", "UTC"} { - if strings.Contains(tt, not) { - t.Errorf("the timer says %s: the same local moment on every node, and no other moment", not) - } - } - if strings.Count(tt, "OnCalendar=") != 1 { - t.Errorf("one moment only:\n%s", tt) - } - if !strings.Contains(st, "\nType=oneshot\n") { - t.Errorf("the service is a oneshot:\n%s", st) - } - if want := "\nExecStart=/usr/bin/pacman " + strings.Join(UpgradeArgs, " ") + "\n"; !strings.Contains(st, want) { - t.Errorf("the service does not run the upgrade pacman_upgrade runs (%q):\n%s", want, st) - } - if strings.Contains(st, "reboot") || strings.Contains(tt, "reboot") || strings.Contains(st, "systemctl") { - t.Error("the mesh restarts no machine for an upgrade") - } - r := m.resource(t, "weekly-upgrade") - if r["type"] != "service" || r["unit"] != "mesh-pacman-weekly-upgrade.timer" || r["state"] != "running" || r["boot"] != "enabled" { - t.Fatalf("the timer runs and is enabled: %v", r) - } - h, _ := r["health"].(map[string]any) - if h["kind"] != "tool" || h["tool"] != "pacman_upgrade_health" { - t.Errorf("the upgrade's outcome is the resource's health: %v", r["health"]) - } - order := map[string]int{} - for i, x := range m.Resources { - order[x["id"].(string)] = i - } - if order["upgrade-service"] > order["weekly-upgrade"] || order["upgrade-timer"] > order["weekly-upgrade"] || order["package"] > order["upgrade-service"] { - t.Errorf("the units are placed before the timer is started: %v", order) - } -} diff --git a/modules/pacman/cmd/pacman-tools/mapped.go b/modules/pacman/cmd/pacman-tools/mapped.go new file mode 100644 index 00000000..64e18af3 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/mapped.go @@ -0,0 +1,173 @@ +package main + +// The after-upgrade check, its second part (novox/hq ADR 0508 §4): the processes still mapping a file the upgrade +// replaced, named by their unit or container and their executable's path, never their command line or +// environment; and whether every DKMS module is built for every installed kernel. Reading another process's +// mappings needs root, so the root helper reads them (ADR 0510's `maps`), with MappedDeleted below. + +import ( + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" +) + +// ProcFS is the process table as MappedDeleted reads it: the machine's /proc, or a stand-in in a test. +type ProcFS interface { + ReadDir(dir string) ([]string, error) + ReadFile(name string) ([]byte, error) + Readlink(name string) (string, error) +} + +// osProc is the machine's own /proc. +type osProc struct{} + +func (osProc) ReadDir(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + return names, err +} +func (osProc) ReadFile(name string) ([]byte, error) { return os.ReadFile(name) } +func (osProc) Readlink(name string) (string, error) { return os.Readlink(name) } + +// replacedPrefixes are where a package's files are: a mapping deleted elsewhere (a temporary file, shared +// memory) is no file an upgrade replaced. +var replacedPrefixes = []string{"/usr/", "/opt/"} + +var containerScope = regexp.MustCompile(`(?:docker|libpod|crio|cri-containerd)-([0-9a-f]{12})[0-9a-f]*\.scope$`) + +// cgroupOwner names a process's unit or container from its cgroup line `0::/system.slice/x.service`. +func cgroupOwner(cgroup string) (unit, container string) { + for _, l := range strings.Split(cgroup, "\n") { + _, p, ok := strings.Cut(l, "::") + if !ok { + continue + } + parts := strings.Split(strings.Trim(p, "/"), "/") + for i := len(parts) - 1; i >= 0; i-- { + if g := containerScope.FindStringSubmatch(parts[i]); g != nil { + return "", g[1] + } + if i > 0 && parts[i-1] == "docker" && len(parts[i]) >= 12 { + return "", parts[i][:12] + } + } + for i := len(parts) - 1; i >= 0; i-- { + if strings.HasSuffix(parts[i], ".service") || strings.HasSuffix(parts[i], ".scope") { + return parts[i], "" + } + } + } + return "", "" +} + +// MappedDeleted reads the process table under procRoot for processes mapping a deleted package file, each named +// once by its unit or container and its executable, sorted. +func MappedDeleted(fs ProcFS, procRoot string) ([]Mapped, error) { + names, err := fs.ReadDir(procRoot) + if err != nil { + return nil, err + } + seen := map[Mapped]bool{} + out := []Mapped{} + for _, name := range names { + if _, err := strconv.Atoi(name); err != nil { + continue + } + dir := path.Join(procRoot, name) + maps, err := fs.ReadFile(dir + "/maps") + if err != nil { + continue // the process ended meanwhile + } + replaced := false + for _, l := range strings.Split(string(maps), "\n") { + if !strings.HasSuffix(l, " (deleted)") { + continue + } + f := strings.Fields(strings.TrimSuffix(l, " (deleted)")) + if len(f) < 6 { + continue + } + for _, p := range replacedPrefixes { + if strings.HasPrefix(f[5], p) { + replaced = true + } + } + } + if !replaced { + continue + } + exe, _ := fs.Readlink(dir + "/exe") + cgroup, _ := fs.ReadFile(dir + "/cgroup") + unit, container := cgroupOwner(string(cgroup)) + m := Mapped{Unit: unit, Container: container, Executable: strings.TrimSuffix(exe, " (deleted)")} + if !seen[m] { + seen[m] = true + out = append(out, m) + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Unit+a.Container != b.Unit+b.Container { + return a.Unit+a.Container < b.Unit+b.Container + } + return a.Executable < b.Executable + }) + return out, nil +} + +// unbuiltDKMS is every DKMS module built for some kernel and not for an installed one its own bound allows. +func (m *Machine) unbuiltDKMS() ([]DKMSBuild, error) { + modules, err := m.DKMSModules() + if err != nil { + return nil, err + } + kernels, err := m.installedKernels() + if err != nil { + return nil, err + } + out := []DKMSBuild{} + for _, mod := range modules { + if len(mod.Kernels) == 0 { + continue // added and never built: not a module this machine uses + } + bound := m.dkmsBound(mod) + for _, k := range kernels { + if contains(mod.Kernels, k) { + continue + } + if held, _ := heldBy(bound, k); held != "" { + continue + } + out = append(out, DKMSBuild{Module: mod.Module, Version: mod.Version, Kernel: k}) + } + } + return out, nil +} + +// afterCheckII fills the check's second part and judges it: a DKMS module not built for a kernel fails it; +// a process still mapping a replaced file is said and does not. +func (u *Upgrades) afterCheckII(c *Check) { + if u.Root == nil { + c.MappedError = "no root helper to read the processes' mapped files" + } else if mapped, err := u.Root.Maps(); err != nil { + c.MappedError = err.Error() + } else { + c.Mapped = mapped + } + unbuilt, err := u.Machine.unbuiltDKMS() + if err != nil { + c.DKMSError = err.Error() + c.Passed = false + return + } + c.Unbuilt = unbuilt + if len(unbuilt) > 0 { + c.Passed = false + } +} diff --git a/modules/pacman/cmd/pacman-tools/mapped_test.go b/modules/pacman/cmd/pacman-tools/mapped_test.go new file mode 100644 index 00000000..dc5f5574 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/mapped_test.go @@ -0,0 +1,131 @@ +package main + +// The after-upgrade check, its second part (novox/hq ADR 0508 §4; acceptance 1841/AC5): processes still mapping a +// replaced file, by unit or container and executable, never a command line or environment; DKMS built for every +// installed kernel. + +import ( + "errors" + "strings" + "testing" +) + +// standInProc is a process table in memory; it records every file read, so nothing else is read. +type standInProc struct { + files map[string]string + links map[string]string + read []string +} + +func (p *standInProc) ReadDir(dir string) ([]string, error) { + seen := map[string]bool{} + var names []string + for f := range p.files { + if rest, ok := strings.CutPrefix(f, dir+"/"); ok { + n, _, _ := strings.Cut(rest, "/") + if !seen[n] { + seen[n] = true + names = append(names, n) + } + } + } + return names, nil +} + +func (p *standInProc) ReadFile(name string) ([]byte, error) { + p.read = append(p.read, name) + if s, ok := p.files[name]; ok { + return []byte(s), nil + } + return nil, errNoFile +} + +func (p *standInProc) Readlink(name string) (string, error) { + if s, ok := p.links[name]; ok { + return s, nil + } + return "", errNoFile +} + +const libcDeleted = "7f2a1c000000-7f2a1c028000 r--p 00000000 103:02 5513 /usr/lib/libc.so.6 (deleted)\n" + +func TestAProcessMappingAReplacedFileIsNamedByItsUnitOrContainerAndExecutableOnly(t *testing.T) { + p := &standInProc{files: map[string]string{ + "/proc/1/maps": libcDeleted, + "/proc/1/cgroup": "0::/init.scope\n", + "/proc/200/maps": "5600-5700 r-xp 00000000 103:02 77 /usr/bin/sshd\n" + strings.Replace(libcDeleted, "libc.so.6", "libcrypto.so.3", 1), + "/proc/200/cgroup": "0::/system.slice/sshd.service\n", + "/proc/201/maps": libcDeleted, + "/proc/201/cgroup": "0::/system.slice/sshd.service\n", + "/proc/300/maps": libcDeleted, + "/proc/300/cgroup": "0::/system.slice/docker-0123456789abcdef0123456789abcdef.scope\n", + "/proc/400/maps": "7f00-7f10 rw-s 00000000 00:1a 9 /dev/shm/made-up (deleted)\n", + "/proc/400/cgroup": "0::/system.slice/made-up.service\n", + "/proc/500/maps": "5600-5700 r-xp 00000000 103:02 77 /usr/bin/made-up\n", + "/proc/500/cmdline": "made-up --password=secret", + "/proc/500/environ": "TOKEN=secret", + "/proc/self/maps": libcDeleted, + }, links: map[string]string{ + "/proc/1/exe": "/usr/lib/systemd/systemd", "/proc/200/exe": "/usr/bin/sshd (deleted)", "/proc/201/exe": "/usr/bin/sshd (deleted)", + "/proc/300/exe": "/usr/bin/made-up-server", + }} + got, err := MappedDeleted(p, "/proc") + if err != nil { + t.Fatal(err) + } + var said []string + for _, m := range got { + said = append(said, m.Unit+"|"+m.Container+"|"+m.Executable) + } + if strings.Join(said, " ") != "|0123456789ab|/usr/bin/made-up-server init.scope||/usr/lib/systemd/systemd sshd.service||/usr/bin/sshd" { + t.Errorf("%v", said) + } + for _, r := range p.read { + if strings.HasSuffix(r, "/cmdline") || strings.HasSuffix(r, "/environ") { + t.Errorf("read %s: a process is named by its unit and executable, never its command line or environment", r) + } + } +} + +func TestADKMSModuleNotBuiltForAnInstalledKernelFailsTheCheck(t *testing.T) { + m := checkMachine("", "7.2.9-arch1-1") + table := map[string]Ran{ + "systemctl list-units --state=failed --plain --no-legend --no-pager": {}, + "pacman -Qq": {Stdout: "linux\nlinux-lts\n"}, + "pacman -Qlq -- linux": {Stdout: "/usr/lib/modules/7.2.9-arch1-1/vmlinuz\n"}, + "pacman -Qlq -- linux-lts": {Stdout: "/usr/lib/modules/6.18.54-2-lts/vmlinuz\n"}, + "dkms status": {Stdout: "zfs/2.4.4, 6.18.54-2-lts, x86_64: installed\nzfs/2.4.4, 7.2.8-arch1-2, x86_64: installed\n" + + "made-up-driver/1.0, 6.18.54-2-lts, x86_64: installed\nxpad/0.4: added\n"}, + } + m.Run = byLine(table, nil) + read := m.ReadFile + m.ReadFile = func(path string) ([]byte, error) { + if path == "/usr/src/made-up-driver-1.0/dkms.conf" { + return []byte("BUILD_EXCLUSIVE_KERNEL=\"^6\\.\"\n"), nil + } + return read(path) + } + u := checkUpgrades(t, m) + u.Root = &fakeRoot{mapped: []Mapped{{Unit: "sshd.service", Executable: "/usr/bin/sshd"}}} + c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) + if len(c.Unbuilt) != 1 || c.Unbuilt[0] != (DKMSBuild{Module: "zfs", Version: "2.4.4", Kernel: "7.2.9-arch1-1"}) || c.Passed { + t.Errorf("zfs not built for the new kernel fails the check (made-up-driver excludes it, xpad was never built): %+v", c) + } + if len(c.Mapped) != 1 || c.Mapped[0].Unit != "sshd.service" { + t.Errorf("mapped: %+v", c.Mapped) + } + + table["dkms status"] = Ran{Stdout: "zfs/2.4.4, 6.18.54-2-lts, x86_64: installed\nzfs/2.4.4, 7.2.9-arch1-1, x86_64: installed\n"} + if c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}); !c.Passed || len(c.Mapped) != 1 { + t.Errorf("built for every kernel passes; a mapped file is said and does not fail it: %+v", c) + } + + u.Root = &fakeRoot{mapsErr: errors.New("the root helper's socket is not there")} + if c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}); !strings.Contains(c.MappedError, "socket") { + t.Errorf("unread mappings are said: %+v", c) + } + table["dkms status"] = Ran{Status: 1, Stderr: "dkms: permission denied\n"} + if c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}); c.Passed || c.DKMSError == "" { + t.Errorf("unread DKMS builds do not pass: %+v", c) + } +} diff --git a/modules/pacman/cmd/pacman-tools/packages.go b/modules/pacman/cmd/pacman-tools/packages.go new file mode 100644 index 00000000..4c52b9e7 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/packages.go @@ -0,0 +1,144 @@ +package main + +// What the keeper reads from the machine into the bus (novox/hq ADR 0507 §2, §4, §7): the local package +// database (`pacman -Q`, `pacman -Qeq`) with each package's repository from the sync databases (`pacman -Sl`), +// and what a full upgrade would change (`checkupdates`). Every line is read against the shape of a package name +// and a version, and a line that is not one is left out, so nothing but names, versions and repositories ever +// reaches a value on the bus (ADR 0507 §1): never a command line, an environment or a message pacman printed. +// None of these commands names a package, so none names a foreign one either. + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +var ( + // packageName is what pacman allows in a package's name (makepkg's lint: alphanumerics and @._+-). + packageName = regexp.MustCompile(`^[a-z0-9@._+][a-z0-9@._+-]*$`) + // packageVersion is an epoch, a version and a release: alphanumerics and ._+:~-. + packageVersion = regexp.MustCompile(`^[A-Za-z0-9._+~][A-Za-z0-9._+:~-]*$`) + // repoName is a repository's section name in pacman.conf. + repoName = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9._-]*$`) +) + +// nameVersion reads a ` ` line, refusing anything else. +func nameVersion(l string) (string, string, bool) { + f := strings.Fields(l) + if len(f) != 2 || !packageName.MatchString(f[0]) || !packageVersion.MatchString(f[1]) { + return "", "", false + } + return f[0], f[1], true +} + +// ReadInstalled is the node's installed packages, sorted by name, each with its version, whether it was installed +// explicitly, and its repository or RepoForeign; and how many lines of pacman's output were not a package and +// were left out. +func (m *Machine) ReadInstalled() ([]InstalledPackage, int, error) { + all, err := m.Out("pacman", "-Q") + if err != nil { + return nil, 0, err + } + explicit, err := m.query("-Qeq") + if err != nil { + return nil, 0, err + } + synced, err := m.Out("pacman", "-Sl") + if err != nil { + return nil, 0, err + } + skipped := 0 + ex := map[string]bool{} + for _, l := range lines(explicit) { + if n := strings.TrimSpace(l); packageName.MatchString(n) { + ex[n] = true + } + } + // A package in two repositories is the first's: `pacman -Sl` lists them in pacman.conf's order. + repo := map[string]string{} + for _, l := range lines(synced) { + f := strings.Fields(l) + if len(f) < 3 || !repoName.MatchString(f[0]) || !packageName.MatchString(f[1]) { + continue + } + if _, seen := repo[f[1]]; !seen { + repo[f[1]] = f[0] + } + } + pkgs := []InstalledPackage{} + for _, l := range lines(all) { + name, version, ok := nameVersion(l) + if !ok { + skipped++ + continue + } + p := InstalledPackage{Name: name, Version: version, Explicit: ex[name], Repo: repo[name]} + if p.Repo == "" { + p.Repo = RepoForeign + } + pkgs = append(pkgs, p) + } + sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].Name < pkgs[j].Name }) + return pkgs, skipped, nil +} + +var updateLine = regexp.MustCompile(`^(\S+) (\S+) -> (\S+)$`) + +// UpdateList is what a full upgrade would change, sorted by name, from checkupdates: a copy of the sync +// databases refreshed apart from the machine's own, so asking never makes a partial upgrade possible. Its +// status 2 is nothing to do; any other failure is an error, never an empty list. +func (m *Machine) UpdateList() ([]VersionChange, error) { + r := m.Run(bg(), "checkupdates") + switch { + case r.Err == "ENOENT": + return nil, fmt.Errorf("checkupdates is not installed: it comes with pacman-contrib, which this module declares") + case r.Err == "" && r.Status == 2: + return []VersionChange{}, nil + case r.Err != "" || r.Status != 0: + return nil, failure("checkupdates", "checkupdates", r) + } + ups := []VersionChange{} + for _, l := range lines(r.Stdout) { + u := updateLine.FindStringSubmatch(strings.TrimSpace(l)) + if u == nil || !packageName.MatchString(u[1]) || !packageVersion.MatchString(u[2]) || !packageVersion.MatchString(u[3]) { + continue + } + ups = append(ups, VersionChange{Name: u[1], From: u[2], To: u[3]}) + } + sort.Slice(ups, func(i, j int) bool { return ups[i].Name < ups[j].Name }) + return ups, nil +} + +// DiffInstalled is what changed from one package list to the next, each list sorted by name: a package added, +// removed, or at another version. A change of reason or repository alone is no change of a version. +func DiffInstalled(before, after []InstalledPackage) Changes { + c := Changes{Added: []PackageVersion{}, Removed: []PackageVersion{}, Changed: []VersionChange{}} + was := map[string]string{} + for _, p := range before { + was[p.Name] = p.Version + } + is := map[string]bool{} + for _, p := range after { + is[p.Name] = true + v, ok := was[p.Name] + switch { + case !ok: + c.Added = append(c.Added, PackageVersion{p.Name, p.Version}) + case v != p.Version: + c.Changed = append(c.Changed, VersionChange{p.Name, v, p.Version}) + } + } + for _, p := range before { + if !is[p.Name] { + c.Removed = append(c.Removed, PackageVersion{p.Name, p.Version}) + } + } + sort.Slice(c.Added, func(i, j int) bool { return c.Added[i].Name < c.Added[j].Name }) + sort.Slice(c.Removed, func(i, j int) bool { return c.Removed[i].Name < c.Removed[j].Name }) + sort.Slice(c.Changed, func(i, j int) bool { return c.Changed[i].Name < c.Changed[j].Name }) + return c +} + +// Empty says a change changed nothing. +func (c Changes) Empty() bool { return len(c.Added)+len(c.Removed)+len(c.Changed) == 0 } diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index 4ca7f5bb..0c7f1d43 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -1,7 +1,6 @@ package main import ( - "errors" "strings" "testing" "time" @@ -78,31 +77,20 @@ func TestInfoReadsListsAsListsAndFallsBackToTheRepositories(t *testing.T) { } } -func TestInstalledSaysWhyAndWhatIsForeign(t *testing.T) { +func TestReadInstalledSaysWhyAndWhichRepositoryOrForeign(t *testing.T) { m := machine(byLine(map[string]Ran{ - "pacman -Q": {Stdout: "glibc 2.42-1\nyay 12.0-1\nzsh 5.9-1\n"}, + "pacman -Q": {Stdout: "zsh 5.9-1\nglibc 2.42-1\nyay 12.0-1\n"}, "pacman -Qeq": {Stdout: "yay\nzsh\n"}, - "pacman -Qmq": {Stdout: "yay\n"}, + "pacman -Sl": {Stdout: "core glibc 2.42-1 [installed]\nextra zsh 5.9-1 [installed]\nextra-testing zsh 5.9-2\n"}, }, nil), 1000) - r, err := m.Installed("", "", false, 10) - if err != nil { - t.Fatal(err) + pk, skipped, err := m.ReadInstalled() + if err != nil || skipped != 0 { + t.Fatal(err, skipped) } - pk := r["packages"].([]Package) - if pk[0].Reason != "dependency" || pk[1].Reason != "explicit" || !pk[1].Foreign || pk[2].Foreign { + want := []InstalledPackage{{"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"}} + if !samePackages(pk, want) { t.Fatalf("%+v", pk) } - if tot := r["totals"].(map[string]int); tot["explicit"] != 2 || tot["dependency"] != 1 || tot["foreign"] != 1 { - t.Fatalf("%v", tot) - } - r, _ = m.Installed("", "", true, 10) - if r["count"] != 1 { - t.Fatalf("foreign only: %v", r) - } - r, _ = m.Installed("", "explicit", false, 1) - if r["count"] != 2 || r["truncated"] != true { - t.Fatalf("bounded: %v", r) - } } func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { @@ -125,18 +113,18 @@ func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { } } -func TestUpdatesReadsCheckupdatesAndItsNothingToDo(t *testing.T) { - m := machine(byLine(map[string]Ran{"checkupdates": {Stdout: "linux 6.1-1 -> 6.2-1\nzsh 5.9-1 -> 5.9-2\n"}}, nil), 1000) - r, err := m.Updates() - if err != nil || r["count"] != 2 || r["updates"].([]Update)[0] != (Update{"linux", "6.1-1", "6.2-1"}) { +func TestUpdateListReadsCheckupdatesAndItsNothingToDo(t *testing.T) { + m := machine(byLine(map[string]Ran{"checkupdates": {Stdout: "zsh 5.9-1 -> 5.9-2\nlinux 6.1-1 -> 6.2-1\n"}}, nil), 1000) + r, err := m.UpdateList() + if err != nil || len(r) != 2 || r[0] != (VersionChange{"linux", "6.1-1", "6.2-1"}) { t.Fatalf("%v %v", r, err) } m = machine(byLine(map[string]Ran{"checkupdates": {Status: 2}}, nil), 1000) - if r, err := m.Updates(); err != nil || r["count"] != 0 { + if r, err := m.UpdateList(); err != nil || r == nil || len(r) != 0 { t.Fatalf("%v %v", r, err) } m = machine(byLine(map[string]Ran{"checkupdates": {Status: 1, Stderr: "==> ERROR: Cannot fetch updates\n"}}, nil), 1000) - if _, err := m.Updates(); err == nil || !strings.Contains(err.Error(), "Cannot fetch updates") { + if _, err := m.UpdateList(); err == nil || !strings.Contains(err.Error(), "Cannot fetch updates") { t.Fatalf("%v", err) } } @@ -146,60 +134,17 @@ func lockless(m *Machine) *Machine { return m } -func TestAnUpgradeRunsAsAUnitOfItsOwnThroughSudoAndBringsTheNews(t *testing.T) { - fetch = func(string) ([]byte, error) { return nil, errors.New("no network") } - var calls []call - m := lockless(machine(fake(func(c call) Ran { return Ran{} }, &calls), 1000)) - r, err := m.Upgrade("", 60) - if err != nil { - t.Fatal(err) - } - unit := r["started"].(string) - if unit != "mesh-pacman-upgrade-1791115200" { - t.Fatalf("unit: %s", unit) - } - last := calls[len(calls)-1] - want := "sudo -n systemd-run --unit=" + unit - if !strings.HasPrefix(last.String(), want) || !strings.HasSuffix(last.String(), "--quiet pacman -Syu --noconfirm") || strings.Contains(last.String(), "--wait") { - t.Fatalf("started as: %s", last) - } - news := r["news"].(map[string]any) - if news["reachable"] != false || !strings.Contains(news["error"].(string), "no network") { - t.Fatalf("no network is an answer: %v", news) - } -} - -func TestAnUpgradeIsRefusedWhileTheDatabaseIsLocked(t *testing.T) { - fetch = func(string) ([]byte, error) { return nil, errors.New("offline") } - var calls []call - m := machine(fake(func(c call) Ran { return Ran{} }, &calls), 1000) - m.ReadFile = func(p string) ([]byte, error) { - if p == DBLock { - return []byte{}, nil - } - return nil, errNoFile - } - if _, err := m.Upgrade("", 60); err == nil || !strings.Contains(err.Error(), "another pacman holds") { - t.Fatalf("%v", err) - } - for _, c := range calls { - if c.name == "sudo" { - t.Fatal("started while locked") - } - } -} - func TestAnUpgradesUnitIsReadBack(t *testing.T) { unit := "mesh-pacman-upgrade-1791115200" m := machine(byLine(map[string]Ran{ "systemctl show " + unit + " --no-pager --property=LoadState --property=ActiveState --property=SubState --property=Result --property=ExecMainStatus": {Stdout: "LoadState=loaded\nActiveState=failed\nSubState=failed\nResult=exit-code\nExecMainStatus=1\n"}, "sudo -n journalctl --no-pager -o cat -n 60 -u " + unit: {Stdout: "error: failed to commit transaction (conflicting files)\n"}, }, nil), 1000) - r, err := m.Upgrade(unit, 60) + r, err := m.UpgradeStatus(unit, 60) if err != nil || r["finished"] != true || r["succeeded"] != false || r["exit_status"] != "1" || len(r["log"].([]string)) != 1 { t.Fatalf("%v %v", r, err) } - if _, err := m.Upgrade("sshd.service", 60); err == nil { + if _, err := m.UpgradeStatus("sshd.service", 60); err == nil { t.Fatal("a unit the tools did not start was read") } } @@ -374,57 +319,3 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { t.Fatalf("%+v", c) } } - -// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5). -func upgradeHealthMachine(props, journal string) *Machine { - return machine(byLine(map[string]Ran{ - "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props}, - "sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal}, - "sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit: {Stdout: journal}, - }, nil), 1000) -} - -func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) { - h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n", - ":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") { - t.Fatalf("%+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "status 1") { - t.Fatalf("a failure without a journal is still unhealthy: %+v", h) - } -} - -func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { - for name, props := range map[string]string{ - "passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n", - "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", - "never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", - } { - if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy { - t.Errorf("%s: %+v", name, h) - } - } - if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy { - t.Errorf("a unit that cannot be read is not healthy: %+v", h) - } -} - -// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574). -func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) { - // the test machine's clock is 2026-10-04 12:00 UTC - h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n", - "downloading linux-6.99...\n").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") { - t.Fatalf("started 3 h ago: %+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth() - if !h.Healthy { - t.Fatalf("started 10 min ago: %+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth() - if h.Healthy { - t.Fatalf("a start that cannot be read is not healthy: %+v", h) - } -} diff --git a/modules/pacman/cmd/pacman-tools/query.go b/modules/pacman/cmd/pacman-tools/query.go index 881263ae..ae8c6626 100644 --- a/modules/pacman/cmd/pacman-tools/query.go +++ b/modules/pacman/cmd/pacman-tools/query.go @@ -8,7 +8,6 @@ import ( "fmt" "path" "regexp" - "sort" "strings" ) @@ -177,10 +176,11 @@ func (m *Machine) Info(name string) (map[string]any, error) { // Package is an installed package and why it is installed. type Package struct { - Name string `json:"name"` - Version string `json:"version"` - Reason string `json:"reason"` - Foreign bool `json:"foreign"` + Name string `json:"name"` + Version string `json:"version"` + Reason string `json:"reason"` + Repository string `json:"repository,omitempty"` + Foreign bool `json:"foreign"` } func nameVersions(out string) [][2]string { @@ -193,51 +193,6 @@ func nameVersions(out string) [][2]string { return nv } -func nameSet(out string) map[string]bool { - s := map[string]bool{} - for _, l := range lines(out) { - s[strings.TrimSpace(l)] = true - } - return s -} - -// Installed is every installed package with its version, whether it was installed explicitly or as -// a dependency, and whether it is foreign (in no repository this machine syncs). -func (m *Machine) Installed(match, reason string, foreignOnly bool, limit int) (map[string]any, error) { - all, err := m.query("-Q") - if err != nil { - return nil, err - } - explicit, err := m.query("-Qeq") - if err != nil { - return nil, err - } - foreign, err := m.query("-Qmq") - if err != nil { - return nil, err - } - ex, fo := nameSet(explicit), nameSet(foreign) - pkgs := []Package{} - counts := map[string]int{"explicit": 0, "dependency": 0, "foreign": 0} - for _, nv := range nameVersions(all) { - p := Package{Name: nv[0], Version: nv[1], Reason: "dependency", Foreign: fo[nv[0]]} - if ex[p.Name] { - p.Reason = "explicit" - } - counts[p.Reason]++ - if p.Foreign { - counts["foreign"]++ - } - if match != "" && !strings.Contains(p.Name, match) || reason != "" && p.Reason != reason || foreignOnly && !p.Foreign { - continue - } - pkgs = append(pkgs, p) - } - out := bound("packages", pkgs, limit) - out["totals"] = counts - return out, nil -} - var ownedBy = regexp.MustCompile(`^(.*) is owned by (\S+) (\S+)$`) // Owns is which package owns a path. @@ -292,51 +247,6 @@ func (m *Machine) Orphans() (map[string]any, error) { return map[string]any{"count": len(pkgs), "orphans": pkgs}, nil } -// Foreign is every installed package no repository this machine syncs carries: built from the AUR -// or by hand, which the host's `package` shape cannot install (research 027, question 1). -func (m *Machine) Foreign() (map[string]any, error) { - out, err := m.query("-Qm") - if err != nil { - return nil, err - } - pkgs := []map[string]string{} - for _, nv := range nameVersions(out) { - pkgs = append(pkgs, map[string]string{"name": nv[0], "version": nv[1]}) - } - sort.Slice(pkgs, func(i, j int) bool { return pkgs[i]["name"] < pkgs[j]["name"] }) - return map[string]any{"count": len(pkgs), "packages": pkgs}, nil -} - -// Update is one package an upgrade would change. -type Update struct { - Name string `json:"name"` - From string `json:"from"` - To string `json:"to"` -} - -var updateLine = regexp.MustCompile(`^(\S+) (\S+) -> (\S+)`) - -// Updates is what a full upgrade would change, from checkupdates: a copy of the sync databases -// refreshed apart from the machine's own, so asking never makes a partial upgrade possible. -func (m *Machine) Updates() (map[string]any, error) { - r := m.Run(bg(), "checkupdates") - switch { - case r.Err == "ENOENT": - return nil, fmt.Errorf("checkupdates is not installed: it comes with pacman-contrib, which this module declares") - case r.Err == "" && r.Status == 2: - return map[string]any{"count": 0, "updates": []Update{}}, nil - case r.Err != "" || r.Status != 0: - return nil, failure("checkupdates", "checkupdates", r) - } - ups := []Update{} - for _, l := range lines(r.Stdout) { - if u := updateLine.FindStringSubmatch(strings.TrimSpace(l)); u != nil { - ups = append(ups, Update{u[1], u[2], u[3]}) - } - } - return map[string]any{"count": len(ups), "updates": ups}, nil -} - // Config is the configuration pacman runs with, as pacman-conf resolves it. type Config struct { Options map[string][]string `json:"options"` diff --git a/modules/pacman/cmd/pacman-tools/record.go b/modules/pacman/cmd/pacman-tools/record.go new file mode 100644 index 00000000..25570861 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record.go @@ -0,0 +1,183 @@ +package main + +// The package record taken before every upgrade (novox/hq ADR 0508 §2): a copy of the node's `installed` key, +// the running kernel, the units failed at that moment and the installed DKMS modules, written to `records` at +// `.` and read back. A run does not start, and says why, when the node's packages are marked not +// current or the record cannot be written and read back equal. Records older than `record-days` are deleted. + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// OSRelease is the running kernel's release. +const OSRelease = "/proc/sys/kernel/osrelease" + +// defaultRecordDays is record-days' default in the manifest, for a config file that holds none. +const defaultRecordDays = 180 + +// FailedUnits are the units of the service manager failed now, by name. +func (m *Machine) FailedUnits() ([]string, error) { + out, err := m.Out("systemctl", "list-units", "--state=failed", "--plain", "--no-legend", "--no-pager") + if err != nil { + return nil, err + } + units := []string{} + for _, l := range lines(out) { + if f := strings.Fields(l); len(f) > 0 { + units = append(units, f[0]) + } + } + sort.Strings(units) + return units, nil +} + +// ParseDKMSStatus reads `dkms status`: each module at its version and the kernels it is installed for. A module +// only added (not built for any kernel) is listed with none. Both forms dkms has written are read: +// `name/version, kernel, arch: state` and the older `name, version, kernel, arch: state`. +func ParseDKMSStatus(out string) []DKMSModule { + byModule := map[string]*DKMSModule{} + var order []string + for _, l := range lines(out) { + head, state, ok := strings.Cut(l, ": ") + if !ok { + continue + } + parts := strings.Split(head, ", ") + var name, version, kernel string + if n, v, slash := strings.Cut(parts[0], "/"); slash { + name, version = n, v + if len(parts) > 1 { + kernel = parts[1] + } + } else if len(parts) >= 2 { + name, version = parts[0], parts[1] + if len(parts) > 2 { + kernel = parts[2] + } + } else { + continue + } + key := name + "/" + version + m := byModule[key] + if m == nil { + m = &DKMSModule{Module: name, Version: version, Kernels: []string{}} + byModule[key] = m + order = append(order, key) + } + if kernel != "" && strings.HasPrefix(strings.TrimSpace(state), "installed") { + m.Kernels = append(m.Kernels, kernel) + } + } + sort.Strings(order) + modules := []DKMSModule{} + for _, k := range order { + sort.Strings(byModule[k].Kernels) + modules = append(modules, *byModule[k]) + } + return modules +} + +// DKMSModules are the DKMS modules installed on the machine; none when dkms is not installed. +func (m *Machine) DKMSModules() ([]DKMSModule, error) { + r := m.Run(bg(), "dkms", "status") + if r.Err == "ENOENT" { + return []DKMSModule{}, nil + } + if r.Status != 0 || r.Err != "" { + return nil, failure("dkms", "dkms", r) + } + return ParseDKMSStatus(r.Stdout), nil +} + +// TakeRecord takes the node's record for a run, writes it and reads it back. It answers the record, or why +// it could not be taken: a run that has a why does not start. +func (u *Upgrades) TakeRecord(run string) (*Record, string) { + nc, err := u.Installed.Get(NotCurrentKey(u.Node)) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if nc != nil { + var mark NotCurrent + _ = json.Unmarshal(nc.Value, &mark) + return nil, fmt.Sprintf("the node's packages on the bus are not current since %s (%d bytes, over the bus's cap on a value): a record of them would not be what is installed", + mark.Since.Format(time.RFC3339), mark.Bytes) + } + e, err := u.Installed.Get(u.Node) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if e == nil { + return nil, "the node's packages are not on the bus yet: there is nothing to record" + } + var installed InstalledValue + if err := json.Unmarshal(e.Value, &installed); err != nil { + return nil, "the node's packages on the bus are not readable: " + err.Error() + } + kernel, err := u.Machine.ReadFile(OSRelease) + if err != nil { + return nil, "the running kernel could not be read: " + err.Error() + } + failed, err := u.Machine.FailedUnits() + if err != nil { + return nil, "the failed units could not be read: " + err.Error() + } + dkms, err := u.Machine.DKMSModules() + if err != nil { + return nil, "the DKMS modules could not be read: " + err.Error() + } + rec := Record{Node: u.Node, Run: run, At: u.Machine.Now().UTC(), Installed: installed, + Kernel: strings.TrimSpace(string(kernel)), FailedUnits: failed, DKMS: dkms} + key := NodeKey(u.Node, run) + if _, err := u.Records.Put(key, rec); err != nil { + return nil, "the record could not be written: " + err.Error() + } + back, err := u.Records.Get(key) + if err != nil { + return nil, "the record could not be read back: " + err.Error() + } + if back == nil { + return nil, "the record was written and is not there when read back" + } + var read Record + want, _ := json.Marshal(rec) + if err := json.Unmarshal(back.Value, &read); err != nil { + return nil, "the record read back is not readable: " + err.Error() + } + if got, _ := json.Marshal(read); !bytes.Equal(got, want) { + return nil, "the record read back differs from the one written" + } + return &rec, "" +} + +// PruneRecords deletes the node's records older than record-days, judged by the time of their run, and +// answers how many it deleted. +func (u *Upgrades) PruneRecords() (int, error) { + days := u.Config.RecordDays + if days < 1 { + days = defaultRecordDays + } + cutoff := u.Machine.Now().AddDate(0, 0, -days) + keys, err := u.Records.Keys() + if err != nil { + return 0, fmt.Errorf("listing records: %w", err) + } + n := 0 + for _, k := range keys { + node, id, ok := strings.Cut(k, ".") + if !ok || node != u.Node { + continue + } + if t, ok := runTime(id); ok && t.Before(cutoff) { + if err := u.Records.Delete(k); err != nil { + return n, fmt.Errorf("deleting record %s: %w", k, err) + } + n++ + } + } + return n, nil +} diff --git a/modules/pacman/cmd/pacman-tools/record_test.go b/modules/pacman/cmd/pacman-tools/record_test.go new file mode 100644 index 00000000..ac1dca95 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record_test.go @@ -0,0 +1,152 @@ +package main + +// The record before every upgrade (novox/hq ADR 0508 §2; acceptance 1841/AC3). + +import ( + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func capturedDKMSStatus(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("testdata/dkms-status-installed-and-added/status") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// recordMachine is a machine running one kernel, with one unit failed and the captured DKMS modules. +func recordMachine(t *testing.T) *Machine { + m := machine(byLine(map[string]Ran{ + "systemctl list-units --state=failed --plain --no-legend --no-pager": {Stdout: "backup.service loaded failed failed The backup\n"}, + "dkms status": {Stdout: capturedDKMSStatus(t)}, + }, nil), 1000) + m.ReadFile = func(path string) ([]byte, error) { + if path == OSRelease { + return []byte("7.2.8-arch1-2\n"), nil + } + return nil, errNoFile + } + return m +} + +var nodeAPackages = InstalledValue{At: time.Date(2026, 10, 3, 9, 0, 0, 0, time.UTC), Packages: []InstalledPackage{ + {Name: "linux", Version: "7.2.8.arch1-2", Explicit: true, Repo: "core"}, + {Name: "zfs-dkms", Version: "2.4.4-1", Explicit: true, Repo: RepoForeign}, +}} + +func recordUpgrades(t *testing.T) (*Upgrades, *memBucket, *memBucket) { + installed, records := newBucket(), newBucket() + if _, err := installed.Put("node-a", nodeAPackages); err != nil { + t.Fatal(err) + } + return &Upgrades{Node: "node-a", Machine: recordMachine(t), Installed: installed, Records: records}, installed, records +} + +func TestDKMSStatusInBothFormsIsEachModuleWithTheKernelsItIsBuiltFor(t *testing.T) { + mods := ParseDKMSStatus(capturedDKMSStatus(t) + "zfs, 2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs, 2.4.4, 6.18.54-2-lts, x86_64: installed\n") + var got []string + for _, m := range mods { + got = append(got, m.Module+"/"+m.Version+"="+strings.Join(m.Kernels, "+")) + } + want := "openrazer-driver/3.12.4=7.2.8-arch1-2 vboxhost/7.2.20_OSE=7.2.8-arch1-2 xone/0.5.8=7.2.8-arch1-2 xpad/0.4= zfs/2.4.4=6.18.54-2-lts+7.2.8-arch1-2" + if strings.Join(got, " ") != want { + t.Errorf("%v", got) + } +} + +func TestARecordIsTheNodesPackagesKernelFailedUnitsAndDKMSWrittenAndReadBack(t *testing.T) { + u, _, records := recordUpgrades(t) + rec, why := u.TakeRecord("20261004T120000Z") + if why != "" || rec == nil { + t.Fatalf("refused: %s", why) + } + var kept Record + if !records.value("node-a.20261004T120000Z", &kept) { + t.Fatal("no record at .") + } + a, _ := json.Marshal(kept.Installed) + b, _ := json.Marshal(nodeAPackages) + if string(a) != string(b) { + t.Errorf("the record copies installed.: %s", a) + } + if kept.Kernel != "7.2.8-arch1-2" || strings.Join(kept.FailedUnits, " ") != "backup.service" || len(kept.DKMS) != 4 || + kept.Node != "node-a" || kept.Run != "20261004T120000Z" { + t.Errorf("record: %+v", kept) + } + if w := strings.Join(records.writes, " "); w != "put node-a.20261004T120000Z" { + t.Errorf("writes: %s", w) + } +} + +func TestNoRecordAndNoRunWhenThePackagesAreNotCurrentOrTheRecordIsNotReadBack(t *testing.T) { + u, installed, records := recordUpgrades(t) + if _, err := installed.Put(NotCurrentKey("node-a"), NotCurrent{Bytes: 300000, Since: time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)}); err != nil { + t.Fatal(err) + } + if rec, why := u.TakeRecord("r1"); rec != nil || !strings.Contains(why, "not current") || !strings.Contains(why, "300000") { + t.Errorf("not current: %v %q", rec, why) + } + if len(records.writes) != 0 { + t.Errorf("a record of packages not current was written: %v", records.writes) + } + + u, installed, _ = recordUpgrades(t) + _ = installed.Delete("node-a") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "not on the bus") { + t.Errorf("no installed key: %q", why) + } + + u, _, records = recordUpgrades(t) + records.fail["put node-a.r1"] = errors.New("value over the bucket's limit") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "could not be written") || !strings.Contains(why, "over the bucket's limit") { + t.Errorf("not written: %q", why) + } + + u, _, _ = recordUpgrades(t) + u.Records = &mangling{newBucket()} + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "differs") { + t.Errorf("read back unequal: %q", why) + } +} + +// mangling is a state that reads back other than what was written. +type mangling struct{ *memBucket } + +func (m *mangling) Get(key string) (*stdio.StateEntry, error) { + e, err := m.memBucket.Get(key) + if e != nil { + e.Value = []byte(strings.Replace(string(e.Value), "7.2.8-arch1-2", "7.2.7-arch1-1", 1)) + } + return e, err +} + +func TestRecordsOlderThanRecordDaysAreDeletedAndYoungerOnesKept(t *testing.T) { + u, _, records := recordUpgrades(t) + u.Config.RecordDays = 180 + now := u.Machine.Now() // 2026-10-04 + for _, k := range []string{ + NodeKey("node-a", RunID(now.AddDate(0, 0, -181))), + NodeKey("node-a", RunID(now.AddDate(0, 0, -179))), + NodeKey("node-b", RunID(now.AddDate(0, 0, -400))), + } { + if _, err := records.Put(k, Record{}); err != nil { + t.Fatal(err) + } + } + n, err := u.PruneRecords() + if err != nil || n != 1 { + t.Fatalf("%d deleted, %v", n, err) + } + keys, _ := records.Keys() + if strings.Join(keys, " ") != NodeKey("node-a", RunID(now.AddDate(0, 0, -179)))+" "+NodeKey("node-b", RunID(now.AddDate(0, 0, -400))) { + t.Errorf("kept: %v (the younger record, and another node's, which its own keeper prunes)", keys) + } +} diff --git a/modules/pacman/cmd/pacman-tools/rollback.go b/modules/pacman/cmd/pacman-tools/rollback.go new file mode 100644 index 00000000..690e0bb5 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/rollback.go @@ -0,0 +1,300 @@ +package main + +// A rollback (novox/hq ADR 0508 §7; acceptance 1841/AC8): a node is returned to the record a run took, and says +// first what that would change. The plan compares the record's packages with the node's `installed` now: each +// package whose version differs, or that is gone since, is brought back to the record's version, from the node's +// package cache when it holds that file, else from the Arch Linux Archive's copy of that exact version, else from +// the archive's repository as it stood on the record's date. The packages installed since are listed, and are +// removed only when asked. A foreign package, in the record or now, is never in the plan (ADR 0507 §7). +// +// The archive keeps every version its repositories published, so a version is taken from it by its file's name, +// which needs the package's architecture: the architecture of any of the package's files in the cache, else of +// its installed version (the local database). A package whose architecture is not known on this machine is +// taken from the dated repository, where pacman finds it by name. Nothing is fetched to make the plan; a file +// the archive does not hold fails the transaction, which the run then says. +// +// With apply, pacman_rollback writes an ask; the node's keeper takes it (TakeAsk), makes the plan again on the +// node itself (its cache is the one that counts), and runs it as a run of kind rollback: one transaction started +// by the root helper with the plan in the request (RootActs.Rollback), `rollback.started` and +// `rollback.finished`, then the after-upgrade check against the record (Follow, checkAfter). + +import ( + "encoding/json" + "errors" + "fmt" + "path" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// ArchiveURL is the Arch Linux Archive. +const ArchiveURL = "https://archive.archlinux.org" + +// SourceRemoved marks a plan item that removes a package installed since the record (to "" and no location): +// only when the rollback was asked to remove what the upgrade added. +const SourceRemoved = "removed" + +// RollbackPlan is what a rollback to a run's record would change. +type RollbackPlan struct { + Node string `json:"node"` + Of string `json:"of"` + // RecordAt is when the record was taken; the dated repository is that day's. + RecordAt time.Time `json:"record_at"` + // Items brings each package back, or removes one installed since when RemoveAdded. + Items []RollbackItem `json:"items"` + // Added is the packages installed since the record, not foreign. + Added []PackageVersion `json:"added"` + RemoveAdded bool `json:"remove_added"` + // Foreign is how many foreign packages the plan leaves untouched. + Foreign int `json:"foreign_untouched"` + // CacheRead says whether the node's package cache was read: only on the node itself. + CacheRead bool `json:"cache_read"` +} + +// cacheFile is `--.pkg.tar.`; a name may hold dashes, a version and an arch none. +var cacheFile = regexp.MustCompile(`^(.+)-([^-]+-[^-]+)-([^-]+)\.pkg\.tar\.(zst|xz)$`) + +// packageCache is the node's cache: by package name, by version, the file's path; and each name's arch. +type packageCache struct { + files map[string]map[string]string + arch map[string]string +} + +// readCache lists the cache's package files. +func (m *Machine) readCache() (packageCache, error) { + c := packageCache{files: map[string]map[string]string{}, arch: map[string]string{}} + out, err := m.Out("find", CacheDir, "-mindepth", "1", "-maxdepth", "1", "-name", "*.pkg.tar.*", "-printf", "%f\n") + if err != nil { + return c, err + } + for _, l := range lines(out) { + f := cacheFile.FindStringSubmatch(strings.TrimSpace(l)) + if f == nil || !packageName.MatchString(f[1]) || !packageVersion.MatchString(f[2]) { + continue + } + if c.files[f[1]] == nil { + c.files[f[1]] = map[string]string{} + } + if _, ok := c.files[f[1]][f[2]]; !ok || f[4] == "zst" { + c.files[f[1]][f[2]] = path.Join(CacheDir, f[0]) + } + c.arch[f[1]] = f[3] + } + return c, nil +} + +// installedArch is an installed package's architecture from the local database, or "". +func (m *Machine) installedArch(name, version string) string { + desc, err := m.ReadFile(path.Join(PacmanDBDir, "local", name+"-"+version, "desc")) + if err != nil { + return "" + } + ls := strings.Split(string(desc), "\n") + for i, l := range ls { + if strings.TrimSpace(l) == "%ARCH%" && i+1 < len(ls) { + if a := strings.TrimSpace(ls[i+1]); repoName.MatchString(a) { + return a + } + } + } + return "" +} + +// source is where a package's record version comes from. +func source(name, version, repo string, arch string, cache *packageCache, recordAt time.Time) (string, string) { + if cache != nil { + if p, ok := cache.files[name][version]; ok { + return SourceCache, p + } + } + if arch != "" { + file := name + "-" + version + "-" + arch + ".pkg.tar.zst" + return SourceArchive, ArchiveURL + "/packages/" + name[:1] + "/" + name + "/" + file + } + day := recordAt.UTC().Format("2006/01/02") + return SourceRepository, ArchiveURL + "/repos/" + day + "/" + repo + "/os/$arch" +} + +// keyGetter reads one key: a Bucket the keeper writes, or a StateReader a verb reads. +type keyGetter interface { + Get(key string) (*stdio.StateEntry, error) +} + +// PlanRollback is the plan for returning node to the record of run, from the states: `records..` and +// `installed.`, which must be current. local says this is the node itself, whose cache and local database +// are read. +func PlanRollback(m *Machine, installed, records keyGetter, node, run string, removeAdded, local bool) (RollbackPlan, error) { + p := RollbackPlan{Node: node, Of: run, Items: []RollbackItem{}, Added: []PackageVersion{}, RemoveAdded: removeAdded} + if e, err := installed.Get(NotCurrentKey(node)); err != nil { + return p, err + } else if e != nil { + return p, fmt.Errorf("the packages of %s on the bus are not current: a plan from them would not be what is installed", node) + } + var rec Record + if e, err := records.Get(NodeKey(node, run)); err != nil { + return p, err + } else if e == nil { + return p, fmt.Errorf("%s has no record of run %s: it was never taken, or is older than record-days", node, run) + } else if err := json.Unmarshal(e.Value, &rec); err != nil { + return p, fmt.Errorf("the record of run %s is not readable: %w", run, err) + } + var now InstalledValue + if e, err := installed.Get(node); err != nil { + return p, err + } else if e == nil { + return p, fmt.Errorf("no packages of %s are on the bus", node) + } else if err := json.Unmarshal(e.Value, &now); err != nil { + return p, fmt.Errorf("installed.%s is not readable: %w", node, err) + } + p.RecordAt = rec.At + var cache *packageCache + if local { + c, err := m.readCache() + if err != nil { + return p, fmt.Errorf("reading the package cache: %w", err) + } + cache, p.CacheRead = &c, true + } + foreign := map[string]bool{} + nowBy := map[string]InstalledPackage{} + for _, ip := range now.Packages { + nowBy[ip.Name] = ip + if ip.Repo == RepoForeign { + foreign[ip.Name] = true + } + } + recBy := map[string]InstalledPackage{} + for _, ip := range rec.Installed.Packages { + recBy[ip.Name] = ip + if ip.Repo == RepoForeign { + foreign[ip.Name] = true + } + } + p.Foreign = len(foreign) + for _, want := range rec.Installed.Packages { + if foreign[want.Name] { + continue + } + have, installedNow := nowBy[want.Name] + if installedNow && have.Version == want.Version { + continue + } + arch := "" + if cache != nil { + arch = cache.arch[want.Name] + } + if arch == "" && local && installedNow { + arch = m.installedArch(have.Name, have.Version) + } + src, loc := source(want.Name, want.Version, want.Repo, arch, cache, rec.At) + p.Items = append(p.Items, RollbackItem{Name: want.Name, From: have.Version, To: want.Version, Source: src, Location: loc}) + } + for _, have := range now.Packages { + if _, inRecord := recBy[have.Name]; inRecord || foreign[have.Name] { + continue + } + p.Added = append(p.Added, PackageVersion{have.Name, have.Version}) + if removeAdded { + p.Items = append(p.Items, RollbackItem{Name: have.Name, From: have.Version, Source: SourceRemoved}) + } + } + sort.Slice(p.Items, func(i, j int) bool { return p.Items[i].Name < p.Items[j].Name }) + return p, nil +} + +// AskRollback writes an ask for a rollback of node to the record of run, for that node's keeper to take. +func AskRollback(asks Bucket, node, run string, removeAdded bool, now time.Time) (Ask, error) { + a := Ask{ID: AskID(now), Node: node, Act: ActRollback, At: now.UTC(), Of: run, RemoveAdded: removeAdded} + if _, err := asks.Create(NodeKey(node, a.ID), a); err != nil { + return Ask{}, fmt.Errorf("writing the ask: %w", err) + } + return a, nil +} + +// StartRollback runs a rollback's steps up to its transaction's start: the plan made again on the node, and the +// transaction started by the root helper with it. The end and the check follow as for an upgrade (Follow). +func (u *Upgrades) StartRollback(of string, removeAdded bool, ask string) (Run, error) { + now := u.Machine.Now().UTC() + r := &openRun{Run: Run{Node: u.Node, Run: RunID(now), Kind: RunRollback, From: FromAsk, Ask: ask, Of: of}} + runs, err := u.ownRuns() + if err != nil { + return r.Run, err + } + for _, o := range runs { + if o.Run.Run == r.Run.Run { + return r.Run, fmt.Errorf("run %s of %s is there already: one run a second", o.Run.Run, u.Node) + } + if !ended(o.Run) && o.From != FromLog { + return r.Run, u.refuse(r, fmt.Sprintf("run %s is still going (its last step is %s): one run at a time", o.Run.Run, lastStep(o.Run))) + } + } + if u.Machine.locked() { + return r.Run, u.refuse(r, "another pacman holds "+DBLock+": a transaction is running, or one was killed and left its lock") + } + plan, err := PlanRollback(u.Machine, u.Installed, u.Records, u.Node, of, removeAdded, true) + if err != nil { + return r.Run, u.refuse(r, err.Error()) + } + if len(plan.Items) == 0 { + return r.Run, u.refuse(r, "the node's packages are those of the record already: there is nothing to bring back") + } + unit := TransactionUnit(RunRollback, r.Run.Run) + if err := u.Root.Rollback(r.Run.Run, plan.Items); err != nil { + return r.Run, u.end(r, Step{Step: StepFailed, At: u.Machine.Now().UTC(), Error: "the transaction could not be started: " + err.Error()}) + } + if err := u.step(r, Step{Step: StepStarted, At: now, Unit: unit, Plan: plan.Items}); err != nil { + return r.Run, err + } + return r.Run, u.emit(EventRollbackStarted, RollbackStarted{Node: u.Node, Run: r.Run.Run, Of: of, Unit: unit}) +} + +// takeRollback carries out an ask for a rollback; the ask is deleted once a run names it. +func (u *Upgrades) takeRollback(key string, a Ask) error { + if a.Of == "" { + return errors.Join(fmt.Errorf("ask %s names no run to return to", key), u.Asks.Delete(key)) + } + r, err := u.StartRollback(a.Of, a.RemoveAdded, a.ID) + if len(r.Steps) == 0 { + return err + } + if derr := u.Asks.Delete(key); derr != nil { + err = errors.Join(err, fmt.Errorf("deleting the ask %s: %w", key, derr)) + } + return err +} + +// Rollback is pacman_rollback: the plan, read from the bus (and on the node itself its cache), and with apply +// the ask that has the node's keeper carry it out. +func (m *Machine) Rollback(node, run string, removeAdded, apply bool) (map[string]any, error) { + if node == "" { + node = m.Node + } + if node == "" { + return nil, errors.New("node is required: this bundle does not know its own") + } + if err := m.states(); err != nil { + return nil, err + } + plan, err := PlanRollback(m, m.States(StateInstalled), m.States(StateRecords), node, run, removeAdded, node == m.Node) + if err != nil { + return nil, err + } + out := map[string]any{"plan": plan} + if !apply { + return out, nil + } + if len(plan.Items) == 0 { + return nil, fmt.Errorf("the packages of %s are those of the record already: there is nothing to bring back", node) + } + a, err := AskRollback(upgradeState(StateAsks), node, run, removeAdded, m.Now()) + if err != nil { + return nil, err + } + out["asked"] = a + out["follow"] = "pacman_upgrades with node " + node + " lists the rollback run that names this ask; its keeper makes the plan again from its own cache" + return out, nil +} diff --git a/modules/pacman/cmd/pacman-tools/rollback_test.go b/modules/pacman/cmd/pacman-tools/rollback_test.go new file mode 100644 index 00000000..ff7ac532 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/rollback_test.go @@ -0,0 +1,195 @@ +package main + +// A rollback to a record (novox/hq ADR 0508 §7; acceptance 1841/AC8): the plan says each package now -> record with +// its source, the packages added and no foreign package; apply is an ask the node's keeper runs as one +// transaction, emitting rollback.started and rollback.finished and ending with the after-upgrade check. + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +var recordAt = time.Date(2026, 10, 3, 4, 0, 12, 0, time.UTC) + +// rollbackRecord is node-a's record of run 20261003T040012Z; rollbackNow its packages since. +var ( + rollbackRecord = Record{Node: "node-a", Run: "20261003T040012Z", At: recordAt, Installed: InstalledValue{At: recordAt, Packages: []InstalledPackage{ + {"bash", "5.3.15-1", true, "core"}, {"glibc", "2.42-1", false, "core"}, {"linux", "7.2.8.arch1-2", true, "core"}, + {"made-up-old-library", "1.0-1", false, "extra"}, {"made-up-foreign", "1-1", true, RepoForeign}, + }}} + rollbackNow = InstalledValue{At: recordAt.Add(time.Hour), Packages: []InstalledPackage{ + {"bash", "5.3.20-1", true, "core"}, {"glibc", "2.42-1", false, "core"}, {"linux", "7.3.1.arch1-1", true, "core"}, + {"made-up-foreign", "2-1", true, RepoForeign}, {"made-up-library", "1.0-1", false, "extra"}, + }} +) + +// rollbackCache is the node's package cache: bash's recorded version, and linux at its new version only. +const rollbackCache = "bash-5.3.15-1-x86_64.pkg.tar.zst\nbash-5.3.20-1-x86_64.pkg.tar.zst\nlinux-7.3.1.arch1-1-x86_64.pkg.tar.zst\n" + + "not-a-package-file\n" + +func rollbackStates(t *testing.T) (*fakeStates, *Machine, *[]call) { + t.Helper() + states := &fakeStates{} + states.store(StateRecords).Put(NodeKey("node-a", rollbackRecord.Run), rollbackRecord) + states.store(StateInstalled).Put("node-a", rollbackNow) + var calls []call + m := machine(byLine(map[string]Ran{ + "find " + CacheDir + " -mindepth 1 -maxdepth 1 -name *.pkg.tar.* -printf %f\n": {Stdout: rollbackCache}, + }, &calls), 1000) + m.Node, m.States = "node-a", readerOf(states) + return states, m, &calls +} + +func TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign(t *testing.T) { + _, m, calls := rollbackStates(t) + p, err := PlanRollback(m, m.States(StateInstalled), m.States(StateRecords), "node-a", rollbackRecord.Run, false, true) + if err != nil { + t.Fatal(err) + } + want := []RollbackItem{ + {"bash", "5.3.20-1", "5.3.15-1", SourceCache, CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"}, + {"linux", "7.3.1.arch1-1", "7.2.8.arch1-2", SourceArchive, "https://archive.archlinux.org/packages/l/linux/linux-7.2.8.arch1-2-x86_64.pkg.tar.zst"}, + {"made-up-old-library", "", "1.0-1", SourceRepository, "https://archive.archlinux.org/repos/2026/10/03/extra/os/$arch"}, + } + if len(p.Items) != len(want) { + t.Fatalf("items: %+v", p.Items) + } + for i := range want { + if p.Items[i] != want[i] { + t.Errorf("item %d: %+v, want %+v", i, p.Items[i], want[i]) + } + } + if len(p.Added) != 1 || p.Added[0] != (PackageVersion{"made-up-library", "1.0-1"}) || p.Foreign != 1 || !p.CacheRead { + t.Errorf("added %+v, foreign %d, cache %v", p.Added, p.Foreign, p.CacheRead) + } + for _, c := range *calls { + if strings.Contains(c.String(), "made-up-foreign") { + t.Errorf("a command named the foreign package: %s", c) + } + } + // asked to remove what was added: the added package is removed too. + p, _ = PlanRollback(m, m.States(StateInstalled), m.States(StateRecords), "node-a", rollbackRecord.Run, true, true) + if len(p.Items) != 4 || p.Items[2] != (RollbackItem{Name: "made-up-library", From: "1.0-1", Source: SourceRemoved}) { + t.Errorf("with remove_added: %+v", p.Items) + } +} + +func TestAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRepository(t *testing.T) { + _, m, calls := rollbackStates(t) + m.Node = "node-b" + r, err := m.Rollback("node-a", rollbackRecord.Run, false, false) + if err != nil { + t.Fatal(err) + } + p := r["plan"].(RollbackPlan) + if p.CacheRead || len(*calls) != 0 { + t.Errorf("another node's cache was read: %v", *calls) + } + for _, i := range p.Items { + if i.Source != SourceRepository || !strings.HasPrefix(i.Location, "https://archive.archlinux.org/repos/2026/10/03/") { + t.Errorf("%+v", i) + } + } +} + +func TestNoPlanFromPackagesNotCurrentOrWithoutARecord(t *testing.T) { + states, m, _ := rollbackStates(t) + if _, err := m.Rollback("node-a", "20250101T000000Z", false, false); err == nil || !strings.Contains(err.Error(), "no record") { + t.Errorf("no record: %v", err) + } + states.store(StateInstalled).Put(NotCurrentKey("node-a"), NotCurrent{Bytes: 300000, Since: recordAt}) + if _, err := m.Rollback("node-a", rollbackRecord.Run, false, false); err == nil || !strings.Contains(err.Error(), "not current") { + t.Errorf("not current: %v", err) + } +} + +func TestApplyWritesAnAskForTheNodesKeeper(t *testing.T) { + _, m, _ := rollbackStates(t) + asks := newBucket() + saved := upgradeState + upgradeState = func(name string) Bucket { return asks } + t.Cleanup(func() { upgradeState = saved }) + r, err := m.Rollback("", rollbackRecord.Run, true, true) + if err != nil { + t.Fatal(err) + } + a := r["asked"].(Ask) + var got Ask + if !asks.value(NodeKey("node-a", a.ID), &got) || got.Act != ActRollback || got.Of != rollbackRecord.Run || !got.RemoveAdded { + t.Errorf("the ask: %+v", got) + } +} + +func TestAnAskedRollbackIsOneTransactionWithItsEventsAndTheCheck(t *testing.T) { + k := newKeeperBed(t) + k.records.Put(NodeKey("node-a", rollbackRecord.Run), rollbackRecord) + k.installed.Put("node-a", rollbackNow) + inner := k.u.Machine.Run + k.u.Machine.Run = func(ctx context.Context, name string, args ...string) Ran { + if name == "find" { + return Ran{Stdout: rollbackCache} + } + return inner(ctx, name, args...) + } + a := Ask{ID: "20261004T120000Z-abc123", Node: "node-a", Act: ActRollback, At: k.now, Of: rollbackRecord.Run} + if err := k.takeAsk(t, a); err != nil { + t.Fatal(err) + } + var r Run + keys, _ := k.runs.Keys() + for _, key := range keys { + var x Run + k.runs.value(key, &x) + if x.Kind == RunRollback { + r = x + } + } + if r.Of != rollbackRecord.Run || r.Ask != a.ID || steps(r) != "started" || len(r.Steps[0].Plan) != 3 { + t.Fatalf("the rollback run: %+v", r) + } + if strings.Join(k.root.started, ";") != "rollback "+r.Run+" plan=bash:5.3.15-1:cache,linux:7.2.8.arch1-2:archive,made-up-old-library:1.0-1:repository" { + t.Errorf("the root helper was asked: %v", k.root.started) + } + if _, there := k.asks.values[NodeKey("node-a", a.ID)]; there { + t.Error("the ask is deleted once the run names it") + } + k.unitState, k.now = "LoadState=not-found\n", k.now.Add(time.Minute) + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + k.runs.value(NodeKey("node-a", r.Run), &r) + if steps(r) != "started finished checked-after" || r.Steps[2].Check == nil { + t.Errorf("steps: %s", steps(r)) + } + if got := strings.Join(k.events.names(), " "); got != "rollback.started rollback.finished upgrade.checked" { + t.Errorf("events: %s", got) + } + // the same record again: nothing to bring back once the packages are the record's. + k.installed.Put("node-a", rollbackRecord.Installed) + k.now = k.now.Add(time.Minute) + if err := k.takeAsk(t, Ask{ID: "20261004T120200Z-def456", Node: "node-a", Act: ActRollback, At: k.now, Of: rollbackRecord.Run}); err != nil { + t.Fatal(err) + } + var again Run + k.runs.value(NodeKey("node-a", "20261004T120200Z"), &again) + if steps(again) != "refused" || !strings.Contains(again.Steps[0].Why, "nothing to bring back") { + t.Errorf("a rollback with nothing to do: %+v", again) + } +} + +// takeAsk writes an ask and hands its change to the keeper, as the watch of `asks` would. +func (k *keeperBed) takeAsk(t *testing.T, a Ask) error { + t.Helper() + key := NodeKey(a.Node, a.ID) + rev, err := k.asks.Create(key, a) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(a) + return k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: key, Op: "put", Value: raw, Revision: rev}) +} diff --git a/modules/pacman/cmd/pacman-tools/root.go b/modules/pacman/cmd/pacman-tools/root.go new file mode 100644 index 00000000..df21a062 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/root.go @@ -0,0 +1,506 @@ +package main + +// The root helper (novox/hq ADR 0510): the module's acts that need root go through a second process of the same +// build, `pacman-root`, running as root with no bus account, never through a sudo rule. It serves a closed list +// of verbs on a unix socket only the keeper's account opens, the peer's account checked on every connection: +// +// - transaction: start a run's full upgrade as a unit of the service manager, named by the run, leaving out +// the packages the run holds back; +// - rollback: start a rollback's transaction as such a unit, carrying out the plan sent with the verb; +// - maps: the processes still mapping a file an upgrade replaced, by unit or container and executable; +// - journal: the last lines a transaction unit of the module logged. +// +// Every other verb is refused, and every argument that is not a unit name of the module's own transactions, a +// package name, a version, a run's id, or a plan item's location in the package cache or the Arch Linux +// Archive. The helper's program is root's (ADR 0510 §3): the node-engine places a process's program in a +// directory its own account owns, and that account is root. +// +// A rollback's unit runs this same program (`pacman-tools root-rollback `) on a plan file the helper +// wrote into a directory only root reads: it finds each package of the dated repository by its file's name, +// installs the plan's packages in one pacman transaction, then removes the ones the rollback was asked to +// remove. Nothing is changed when a package cannot be found. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "os/user" + "path" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +// The helper's verbs. +const ( + VerbTransaction = "transaction" + VerbRollback = "rollback" + VerbMaps = "maps" + VerbJournal = "journal" +) + +// The helper's environment, and the keeper's word for where the helper listens. +const ( + EnvRootSocket = "PACMAN_ROOT_SOCKET" // the helper's socket, for the helper and the keeper + EnvRootPeer = "PACMAN_ROOT_PEER" // the only account the helper answers: the keeper's + // RootSocketPath is the socket as the manifest names it: in a directory of the module's state the helper + // makes, root's. + RootSocketPath = "${dir:state}/root/helper.sock" +) + +// rootTimeout bounds one exchange with the helper: every verb answers at once (a transaction is started, not +// waited for). +const rootTimeout = 30 * time.Second + +// mostJournalLines is the most lines the journal verb reads. +const mostJournalLines = 400 + +// RootRequest is one verb sent to the helper, one per connection. +type RootRequest struct { + Verb string `json:"verb"` + Kind string `json:"kind,omitempty"` + Run string `json:"run,omitempty"` + Ignore []string `json:"ignore,omitempty"` + Plan []RollbackItem `json:"plan,omitempty"` + Unit string `json:"unit,omitempty"` + Lines int `json:"lines,omitempty"` +} + +// RootAnswer is the helper's answer: an error, or what the verb read. +type RootAnswer struct { + Error string `json:"error,omitempty"` + Lines []string `json:"lines,omitempty"` + Mapped []Mapped `json:"mapped,omitempty"` +} + +// RootHelper is the helper: what it runs, the process table it reads, where it keeps plans, its own program, +// and the one account it answers. +type RootHelper struct { + Run Runner + Proc ProcFS + ProcRoot string + PlanDir string + Self string + Peer int + // PeerOf is the account at the other end of a connection; nil reads it from the socket (SO_PEERCRED). + PeerOf func(net.Conn) (int, error) + Logf func(format string, a ...any) +} + +var ( + // runIDShape is a run's id (RunID): its start in UTC to the second. + runIDShape = regexp.MustCompile(`^[0-9]{8}T[0-9]{6}Z$`) + // ownUnit is a transaction unit of the module's own runs (TransactionUnit). + ownUnit = regexp.MustCompile(`^` + unitPrefix + `(` + RunUpgrade + `|` + RunRollback + `)-[0-9]{8}t[0-9]{6}z$`) + // archArg is a package's architecture in a file's name. + archArg = regexp.MustCompile(`^[a-z0-9_]+$`) + // datedRepository is the archive's repository as it stood on one day, its architecture left to pacman. + datedRepository = regexp.MustCompile(`^` + regexp.QuoteMeta(ArchiveURL) + `/repos/[0-9]{4}/[0-9]{2}/[0-9]{2}/[A-Za-z0-9_-]+/os/\$arch$`) +) + +func refused(format string, a ...any) RootAnswer { + return RootAnswer{Error: "refused: " + fmt.Sprintf(format, a...)} +} + +func checkRun(run string) error { + if !runIDShape.MatchString(run) { + return fmt.Errorf("%q is not a run's id", run) + } + return nil +} + +// checkItem refuses a plan item that is not a package brought back from the cache or the archive, or removed. +func checkItem(i RollbackItem) error { + if !packageName.MatchString(i.Name) { + return fmt.Errorf("%q is not a package's name", i.Name) + } + if i.Source == SourceRemoved { + if i.To != "" || i.Location != "" { + return fmt.Errorf("%s is removed, and names a version or a place to take it from", i.Name) + } + return nil + } + if !packageVersion.MatchString(i.To) { + return fmt.Errorf("%q is not a version of %s", i.To, i.Name) + } + switch i.Source { + case SourceCache: + f := cacheFile.FindStringSubmatch(path.Base(i.Location)) + if path.Clean(i.Location) != i.Location || path.Dir(i.Location) != CacheDir || f == nil || f[1] != i.Name || f[2] != i.To { + return fmt.Errorf("%q is not %s %s's file in the package cache", i.Location, i.Name, i.To) + } + case SourceArchive: + prefix := ArchiveURL + "/packages/" + i.Name[:1] + "/" + i.Name + "/" + i.Name + "-" + i.To + "-" + arch, ok := strings.CutPrefix(i.Location, prefix) + if !ok || !strings.HasSuffix(arch, ".pkg.tar.zst") || !archArg.MatchString(strings.TrimSuffix(arch, ".pkg.tar.zst")) { + return fmt.Errorf("%q is not %s %s's file in the archive", i.Location, i.Name, i.To) + } + case SourceRepository: + if !datedRepository.MatchString(i.Location) { + return fmt.Errorf("%q is not a dated repository of the archive", i.Location) + } + default: + return fmt.Errorf("%q is not a source a package is brought back from", i.Source) + } + return nil +} + +// Handle carries out one verb, or refuses it naming why; nothing runs for a refused verb. +func (h *RootHelper) Handle(req RootRequest) RootAnswer { + switch req.Verb { + case VerbTransaction: + if req.Kind != RunUpgrade { + return refused("a transaction is a full upgrade, not %q", req.Kind) + } + if err := checkRun(req.Run); err != nil { + return refused("%v", err) + } + for _, n := range req.Ignore { + if !packageName.MatchString(n) { + return refused("%q is not a package's name to leave out", n) + } + } + args := append([]string{"pacman"}, UpgradeArgs...) + if len(req.Ignore) > 0 { + args = append(args, "--ignore", strings.Join(req.Ignore, ",")) + } + return h.startUnit(TransactionUnit(RunUpgrade, req.Run), "pacman full upgrade of run "+req.Run, args) + case VerbRollback: + if err := checkRun(req.Run); err != nil { + return refused("%v", err) + } + if len(req.Plan) == 0 { + return refused("a rollback with no plan") + } + for _, i := range req.Plan { + if err := checkItem(i); err != nil { + return refused("%v", err) + } + } + raw, err := json.Marshal(req.Plan) + if err != nil { + return RootAnswer{Error: err.Error()} + } + if err := os.MkdirAll(h.PlanDir, 0o700); err != nil { + return RootAnswer{Error: "the plan's directory: " + err.Error()} + } + file := filepath.Join(h.PlanDir, req.Run+".json") + if err := os.WriteFile(file, raw, 0o600); err != nil { + return RootAnswer{Error: "writing the plan: " + err.Error()} + } + return h.startUnit(TransactionUnit(RunRollback, req.Run), "pacman rollback of run "+req.Run, []string{h.Self, "root-rollback", file}) + case VerbMaps: + mapped, err := MappedDeleted(h.Proc, h.ProcRoot) + if err != nil { + return RootAnswer{Error: "reading the process table: " + err.Error()} + } + return RootAnswer{Mapped: mapped} + case VerbJournal: + if !ownUnit.MatchString(req.Unit) { + return refused("%q is not a transaction unit of the module", req.Unit) + } + if req.Lines < 1 || req.Lines > mostJournalLines { + return refused("a journal is read 1 to %d lines at a time, not %d", mostJournalLines, req.Lines) + } + r := h.Run(context.Background(), "journalctl", "--no-pager", "-o", "cat", "-n", strconv.Itoa(req.Lines), "-u", req.Unit) + if r.Status != 0 || r.Err != "" { + return RootAnswer{Error: failure("journalctl", "journalctl", r).Error()} + } + return RootAnswer{Lines: lines(r.Stdout)} + } + return refused("%q is not a verb of the root helper (%s, %s, %s, %s)", req.Verb, VerbTransaction, VerbRollback, VerbMaps, VerbJournal) +} + +// startUnit starts command as a transient unit of the service manager and answers once it started. +func (h *RootHelper) startUnit(unit, description string, command []string) RootAnswer { + args := append([]string{"--unit=" + unit, "--description=" + description + ", started by the mesh's pacman keeper", "--quiet"}, command...) + r := h.Run(context.Background(), "systemd-run", args...) + if r.Status != 0 || r.Err != "" { + return RootAnswer{Error: failure("systemd-run", "systemd-run", r).Error()} + } + return RootAnswer{} +} + +// peerCred is the account at the other end of a unix socket. +func peerCred(c net.Conn) (int, error) { + uc, ok := c.(*net.UnixConn) + if !ok { + return -1, errors.New("not a unix socket") + } + raw, err := uc.SyscallConn() + if err != nil { + return -1, err + } + uid := -1 + var gerr error + if err := raw.Control(func(fd uintptr) { + cred, err := unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) + if err != nil { + gerr = err + return + } + uid = int(cred.Uid) + }); err != nil { + return -1, err + } + return uid, gerr +} + +// Serve answers each connection on l, one request each, until ctx ends: a peer that is not the keeper's +// account is told so and nothing it sent is read. +func (h *RootHelper) Serve(ctx context.Context, l net.Listener) error { + go func() { <-ctx.Done(); l.Close() }() + peerOf := h.PeerOf + if peerOf == nil { + peerOf = peerCred + } + for { + c, err := l.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + go h.serveOne(c, peerOf) + } +} + +func (h *RootHelper) serveOne(c net.Conn, peerOf func(net.Conn) (int, error)) { + defer c.Close() + _ = c.SetDeadline(time.Now().Add(rootTimeout)) + answer := func(a RootAnswer) { _ = json.NewEncoder(c).Encode(a) } + uid, err := peerOf(c) + if err != nil || uid != h.Peer { + if h.Logf != nil { + h.Logf("[pacman] root helper: a connection from account %d refused: only the keeper's (%d) is answered", uid, h.Peer) + } + answer(refused("only the keeper's account is answered")) + return + } + var req RootRequest + if err := json.NewDecoder(io.LimitReader(c, 4<<20)).Decode(&req); err != nil { + answer(refused("no request: %v", err)) + return + } + a := h.Handle(req) + if h.Logf != nil { + said := "carried out" + if a.Error != "" { + said = a.Error + } + h.Logf("[pacman] root helper: %s %s %s: %s", req.Verb, req.Kind, req.Run+req.Unit, said) + } + answer(a) +} + +// rootClient is the keeper's side: the RootActs over the helper's socket. +type rootClient struct{ Socket string } + +func (c rootClient) ask(req RootRequest) (RootAnswer, error) { + conn, err := net.DialTimeout("unix", c.Socket, rootTimeout) + if err != nil { + return RootAnswer{}, fmt.Errorf("the root helper cannot be reached at %s: %w", c.Socket, err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(rootTimeout)) + if err := json.NewEncoder(conn).Encode(req); err != nil { + return RootAnswer{}, fmt.Errorf("asking the root helper: %w", err) + } + var a RootAnswer + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&a); err != nil { + return RootAnswer{}, fmt.Errorf("the root helper's answer: %w", err) + } + if a.Error != "" { + return a, fmt.Errorf("the root helper: %s", a.Error) + } + return a, nil +} + +func (c rootClient) Transaction(kind, run string, ignore []string) error { + _, err := c.ask(RootRequest{Verb: VerbTransaction, Kind: kind, Run: run, Ignore: ignore}) + return err +} + +func (c rootClient) Rollback(run string, plan []RollbackItem) error { + _, err := c.ask(RootRequest{Verb: VerbRollback, Run: run, Plan: plan}) + return err +} + +func (c rootClient) Journal(unit string, n int) ([]string, error) { + a, err := c.ask(RootRequest{Verb: VerbJournal, Unit: unit, Lines: n}) + return a.Lines, err +} + +func (c rootClient) Maps() ([]Mapped, error) { + a, err := c.ask(RootRequest{Verb: VerbMaps}) + if a.Mapped == nil && err == nil { + a.Mapped = []Mapped{} + } + return a.Mapped, err +} + +// rootHelper is `pacman-tools root`: it listens on its socket, made the keeper's alone, until stopped. +func rootHelper() int { + logf := func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) } + if os.Getuid() != 0 { + logf("[pacman] root helper: it runs as root, and runs as account %d", os.Getuid()) + return 2 + } + sock, peerName := os.Getenv(EnvRootSocket), os.Getenv(EnvRootPeer) + if sock == "" || peerName == "" { + logf("[pacman] root helper was started without %s or %s", EnvRootSocket, EnvRootPeer) + return 2 + } + peer, err := user.Lookup(peerName) + if err != nil { + logf("[pacman] root helper: the keeper's account %s: %v", peerName, err) + return 1 + } + uid, err := strconv.Atoi(peer.Uid) + if err != nil { + logf("[pacman] root helper: account %s has no number: %v", peerName, err) + return 1 + } + self, err := os.Executable() + if err != nil { + logf("[pacman] root helper: its own program: %v", err) + return 1 + } + dir := filepath.Dir(sock) + if err := os.MkdirAll(dir, 0o711); err != nil || os.Chown(dir, 0, 0) != nil || os.Chmod(dir, 0o711) != nil { + logf("[pacman] root helper: its directory %s cannot be made root's: %v", dir, err) + return 1 + } + _ = os.Remove(sock) + l, err := net.Listen("unix", sock) + if err != nil { + logf("[pacman] root helper: listening on %s: %v", sock, err) + return 1 + } + if err := os.Chown(sock, uid, 0); err != nil || os.Chmod(sock, 0o600) != nil { + logf("[pacman] root helper: the socket cannot be made the keeper's alone: %v", err) + return 1 + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + h := &RootHelper{Run: ExecRunner, Proc: osProc{}, ProcRoot: "/proc", PlanDir: filepath.Join(dir, "plans"), Self: self, Peer: uid, Logf: logf} + logf("[pacman] root helper listening on %s for account %s (%d)", sock, peerName, uid) + if err := h.Serve(ctx, l); err != nil { + logf("[pacman] root helper stopped: %v", err) + return 1 + } + return 0 +} + +// machineArch is this machine's architecture as Arch Linux names it. +func machineArch() string { + switch runtime.GOARCH { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + } + return runtime.GOARCH +} + +// archiveHas says whether the archive holds a file, by asking for its head. +func archiveHas(url string) bool { + c := &http.Client{Timeout: 30 * time.Second} + resp, err := c.Head(url) + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == http.StatusOK +} + +// rollbackUnit carries out a rollback plan inside its unit: every package of a dated repository found by its +// file's name (this machine's architecture, then any), the plan's packages installed in one transaction, then +// the packages to remove removed. Nothing is changed when a package is not found. +func rollbackUnit(file, arch string, run func(string, ...string) int, probe func(string) bool, log io.Writer) int { + raw, err := os.ReadFile(file) + if err != nil { + fmt.Fprintf(log, "error: reading the plan: %v\n", err) + return 1 + } + var plan []RollbackItem + if err := json.Unmarshal(raw, &plan); err != nil { + fmt.Fprintf(log, "error: the plan is not one: %v\n", err) + return 1 + } + var targets, remove []string + for _, i := range plan { + if err := checkItem(i); err != nil { + fmt.Fprintf(log, "error: %v\n", err) + return 1 + } + switch i.Source { + case SourceRemoved: + remove = append(remove, i.Name) + case SourceRepository: + base := strings.Replace(i.Location, "$arch", arch, 1) + found := "" + for _, a := range []string{arch, "any"} { + if u := base + "/" + i.Name + "-" + i.To + "-" + a + ".pkg.tar.zst"; probe(u) { + found = u + break + } + } + if found == "" { + fmt.Fprintf(log, "error: %s %s is not in %s: nothing was changed\n", i.Name, i.To, base) + return 1 + } + targets = append(targets, found) + default: + targets = append(targets, i.Location) + } + } + if len(targets) > 0 { + if code := run("pacman", append([]string{"-U", "--noconfirm", "--"}, targets...)...); code != 0 { + return code + } + } + if len(remove) > 0 { + return run("pacman", append([]string{"-R", "--noconfirm", "--"}, remove...)...) + } + return 0 +} + +// rootRollback is `pacman-tools root-rollback `, the rollback unit's command: pacman's output goes to the +// unit's journal. +func rootRollback(args []string) int { + if len(args) != 1 || os.Getuid() != 0 { + fmt.Fprintln(os.Stderr, "error: root-rollback takes one plan file, and runs as root in its unit") + return 2 + } + run := func(name string, a ...string) int { + cmd := exec.Command(name, a...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + cmd.Env = append(os.Environ(), "LC_ALL=C") + if err := cmd.Run(); err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + return exit.ExitCode() + } + fmt.Fprintf(os.Stderr, "error: %s: %v\n", name, err) + return 1 + } + return 0 + } + return rollbackUnit(args[0], machineArch(), run, archiveHas, os.Stderr) +} diff --git a/modules/pacman/cmd/pacman-tools/root_test.go b/modules/pacman/cmd/pacman-tools/root_test.go new file mode 100644 index 00000000..8d9d2962 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/root_test.go @@ -0,0 +1,312 @@ +package main + +// The root helper (novox/hq ADR 0510; acceptance 1841/AC10 as reworded on the issue): each listed verb with valid +// arguments is carried out; every other verb, a unit name outside the module's own, a package name with other +// characters and a run of no run's shape are refused; a connection from an account other than the keeper's is +// refused; the manifest declares `pacman-root` as root with no bus and places nothing under the sudo +// configuration. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// rootBed is a helper over a recording runner, a stand-in process table and a plan directory of its own. +type rootBed struct { + h *RootHelper + ran [][]string +} + +func newRootBed(t *testing.T) *rootBed { + b := &rootBed{} + b.h = &RootHelper{ + Run: func(_ context.Context, name string, args ...string) Ran { + b.ran = append(b.ran, append([]string{name}, args...)) + if name == "journalctl" { + return Ran{Stdout: "error: failed to commit transaction\n"} + } + return Ran{} + }, + Proc: stubProc{"/proc": {"1"}, "/proc/1/maps": {"7f00 r-xp 0 0:1 1 /usr/lib/libc.so.6 (deleted)"}, "/proc/1/cgroup": {"0::/system.slice/sshd.service"}, "/proc/1/exe": {"/usr/bin/sshd"}}, + ProcRoot: "/proc", + PlanDir: t.TempDir(), + Self: "/var/lib/mesh/daemons/pacman-root/pacman-tools", + } + return b +} + +// stubProc is a process table in memory: a directory lists its names, a file holds its one line, a link its target. +type stubProc map[string][]string + +func (s stubProc) ReadDir(dir string) ([]string, error) { return s[dir], nil } +func (s stubProc) ReadFile(name string) ([]byte, error) { + v, ok := s[name] + if !ok { + return nil, os.ErrNotExist + } + return []byte(strings.Join(v, "\n")), nil +} +func (s stubProc) Readlink(name string) (string, error) { + v, ok := s[name] + if !ok { + return "", os.ErrNotExist + } + return v[0], nil +} + +const aRun = "20261024T040000Z" + +// testSocket is a socket in the abstract namespace, named for the test: a path under a test's temporary +// directory can pass the 108 bytes a socket's path may hold. +func testSocket(t *testing.T) string { + return fmt.Sprintf("@pacman-root-test-%d-%s", os.Getpid(), t.Name()) +} + +func TestTheRootHelperStartsAnUpgradeAsTheRunsUnit(t *testing.T) { + b := newRootBed(t) + a := b.h.Handle(RootRequest{Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"linux", "linux-headers"}}) + if a.Error != "" { + t.Fatalf("refused: %s", a.Error) + } + want := []string{"systemd-run", "--unit=" + TransactionUnit(RunUpgrade, aRun), "--description=pacman full upgrade of run " + aRun + ", started by the mesh's pacman keeper", + "--quiet", "pacman", "-Syu", "--noconfirm", "--ignore", "linux,linux-headers"} + if len(b.ran) != 1 || !reflect.DeepEqual(b.ran[0], want) { + t.Fatalf("ran %q, want %q", b.ran, want) + } +} + +func TestTheRootHelperStartsARollbackWithThePlanInTheRequest(t *testing.T) { + b := newRootBed(t) + plan := []RollbackItem{ + {Name: "bash", From: "5.3.20-1", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"}, + {Name: "glibc", From: "2.43-2", To: "2.43-1", Source: SourceArchive, Location: ArchiveURL + "/packages/g/glibc/glibc-2.43-1-x86_64.pkg.tar.zst"}, + {Name: "zlib", From: "1.3.2-1", To: "1.3.1-2", Source: SourceRepository, Location: ArchiveURL + "/repos/2026/10/04/core/os/$arch"}, + {Name: "made-up-library", From: "1.0-1", Source: SourceRemoved}, + } + a := b.h.Handle(RootRequest{Verb: VerbRollback, Run: aRun, Plan: plan}) + if a.Error != "" { + t.Fatalf("refused: %s", a.Error) + } + file := filepath.Join(b.h.PlanDir, aRun+".json") + want := []string{"systemd-run", "--unit=" + TransactionUnit(RunRollback, aRun), "--description=pacman rollback of run " + aRun + ", started by the mesh's pacman keeper", + "--quiet", b.h.Self, "root-rollback", file} + if len(b.ran) != 1 || !reflect.DeepEqual(b.ran[0], want) { + t.Fatalf("ran %q, want %q", b.ran, want) + } + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var got []RollbackItem + if err := json.Unmarshal(raw, &got); err != nil || !reflect.DeepEqual(got, plan) { + t.Fatalf("the plan file holds %s (%v)", raw, err) + } + if fi, _ := os.Stat(file); fi.Mode().Perm() != 0o600 { + t.Errorf("the plan file is mode %v, want 0600: only root reads what root carries out", fi.Mode().Perm()) + } +} + +func TestTheRootHelperReadsTheReplacedFilesMappings(t *testing.T) { + b := newRootBed(t) + a := b.h.Handle(RootRequest{Verb: VerbMaps}) + want := []Mapped{{Unit: "sshd.service", Executable: "/usr/bin/sshd"}} + if a.Error != "" || !reflect.DeepEqual(a.Mapped, want) { + t.Fatalf("maps answered %+v, want %+v", a, want) + } +} + +func TestTheRootHelperReadsATransactionUnitsJournal(t *testing.T) { + b := newRootBed(t) + unit := TransactionUnit(RunUpgrade, aRun) + a := b.h.Handle(RootRequest{Verb: VerbJournal, Unit: unit, Lines: 40}) + if a.Error != "" || !reflect.DeepEqual(a.Lines, []string{"error: failed to commit transaction"}) { + t.Fatalf("journal answered %+v", a) + } + want := []string{"journalctl", "--no-pager", "-o", "cat", "-n", "40", "-u", unit} + if !reflect.DeepEqual(b.ran[0], want) { + t.Fatalf("ran %q, want %q", b.ran[0], want) + } +} + +func TestTheRootHelperRefusesEverythingOffItsList(t *testing.T) { + good := RollbackItem{Name: "bash", From: "5.3.20-1", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"} + with := func(f func(*RollbackItem)) []RollbackItem { i := good; f(&i); return []RollbackItem{i} } + cases := map[string]RootRequest{ + "another verb": {Verb: "shell", Run: aRun}, + "an empty verb": {}, + "another kind of transaction": {Verb: VerbTransaction, Kind: "remove", Run: aRun}, + "a rollback as a transaction": {Verb: VerbTransaction, Kind: RunRollback, Run: aRun}, + "a run of no run's shape": {Verb: VerbTransaction, Kind: RunUpgrade, Run: "../../etc"}, + "a run with an option in it": {Verb: VerbTransaction, Kind: RunUpgrade, Run: "--now"}, + "an ignored name with a space": {Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"linux headers"}}, + "an ignored name as an option": {Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"--root=/tmp"}}, + "an ignored name with a comma": {Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"linux,glibc"}}, + "an ignored name in capitals": {Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"Linux"}}, + "a journal of another unit": {Verb: VerbJournal, Unit: "sshd.service", Lines: 10}, + "a journal of a look-alike unit": {Verb: VerbJournal, Unit: "mesh-pacman-upgrade-x;reboot", Lines: 10}, + "a journal of too many lines": {Verb: VerbJournal, Unit: TransactionUnit(RunUpgrade, aRun), Lines: 100000}, + "a rollback of no plan": {Verb: VerbRollback, Run: aRun}, + "a rollback of a bad run": {Verb: VerbRollback, Run: "x", Plan: []RollbackItem{good}}, + "a plan item with a bad name": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.Name = "bash;rm" })}, + "a plan item with a bad version": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.To = "1 2" })}, + "a cache file outside the cache": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.Location = "/tmp/bash-5.3.15-1-x86_64.pkg.tar.zst" })}, + "a cache file climbing out": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.Location = CacheDir + "/../../../tmp/bash-5.3.15-1-x86_64.pkg.tar.zst" })}, + "a cache file of another package": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.Location = CacheDir + "/evil-1-1-x86_64.pkg.tar.zst" })}, + "an archive URL elsewhere": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { + i.Source, i.Location = SourceArchive, "https://example.org/packages/b/bash/bash-5.3.15-1-x86_64.pkg.tar.zst" + })}, + "a repository URL elsewhere": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { + i.Source, i.Location = SourceRepository, ArchiveURL+"/repos/2026/10/04/../../../../x" + })}, + "another source": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { i.Source = "file" })}, + "a removal with a location": {Verb: VerbRollback, Run: aRun, Plan: with(func(i *RollbackItem) { + i.Source, i.To = SourceRemoved, "" + })}, + } + for name, req := range cases { + b := newRootBed(t) + if a := b.h.Handle(req); a.Error == "" { + t.Errorf("%s: carried out (%+v)", name, a) + } + if len(b.ran) != 0 { + t.Errorf("%s: ran %q although refused", name, b.ran) + } + } +} + +// serveOnce starts the helper on a socket of a test directory, allowing peer, and answers one request. +func serveOnce(t *testing.T, peer int, req RootRequest) (RootAnswer, error) { + t.Helper() + b := newRootBed(t) + b.h.Peer = peer + sock := testSocket(t) + l, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = b.h.Serve(ctx, l) }() + return rootClient{Socket: sock}.ask(req) +} + +func TestTheRootHelperAnswersTheKeepersAccountOverItsSocket(t *testing.T) { + a, err := serveOnce(t, os.Getuid(), RootRequest{Verb: VerbMaps}) + if err != nil || a.Error != "" || len(a.Mapped) != 1 { + t.Fatalf("the keeper's own account was answered %+v, %v", a, err) + } +} + +func TestTheRootHelperRefusesAnyOtherAccount(t *testing.T) { + a, err := serveOnce(t, os.Getuid()+1, RootRequest{Verb: VerbMaps}) + if err == nil && a.Error == "" { + t.Fatalf("another account was answered: %+v", a) + } + if a.Mapped != nil { + t.Fatalf("another account was handed the mappings: %+v", a) + } +} + +func TestTheClientCarriesTheRootActsAndSaysARefusal(t *testing.T) { + b := newRootBed(t) + b.h.Peer = os.Getuid() + sock := testSocket(t) + l, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = b.h.Serve(ctx, l) }() + var acts RootActs = rootClient{Socket: sock} + if err := acts.Transaction(RunUpgrade, aRun, nil); err != nil { + t.Fatalf("transaction: %v", err) + } + if err := acts.Transaction(RunUpgrade, "bad", nil); err == nil || !strings.Contains(err.Error(), "run") { + t.Fatalf("a bad run was not refused by name: %v", err) + } + if lines, err := acts.Journal(TransactionUnit(RunUpgrade, aRun), 5); err != nil || len(lines) != 1 { + t.Fatalf("journal: %q, %v", lines, err) + } + if _, err := (rootClient{Socket: testSocket(t) + "-none"}).Maps(); err == nil || !strings.Contains(err.Error(), "root helper") { + t.Fatalf("a helper that is not there is not said: %v", err) + } +} + +func TestTheRootHelperIsRootWithNoBusAndNoSudoRule(t *testing.T) { + m := manifest(t) + r := m.resource(t, "root-helper") + if r["type"] != "process" || r["name"] != "pacman-root" || r["artifact"] != "tools" || + strings.Join(strs(r["run"]), " ") != "./"+binaryName+" root" { + t.Fatalf("the root helper: %v", r) + } + if _, ok := r["user"]; ok { + t.Errorf("the root helper runs as root (ADR 0510 §1, §3), and names a user: %v", r["user"]) + } + if _, ok := r["bus"]; ok { + t.Errorf("the root helper has no bus account (ADR 0510 §1): %v", r["bus"]) + } + env, _ := r["env"].(map[string]any) + if env[EnvRootSocket] != RootSocketPath || env[EnvRootPeer] != "pacman-keeper" { + t.Errorf("the root helper's environment names its socket and the keeper's account: %v", env) + } + k := m.resource(t, "keeper") + kenv, _ := k["env"].(map[string]any) + if kenv[EnvRootSocket] != RootSocketPath { + t.Errorf("the keeper is told the helper's socket: %v", kenv) + } + for _, res := range m.Resources { + if p, _ := res["path"].(string); strings.Contains(p, "sudoers") { + t.Errorf("a file under the sudo configuration is placed: %s (ADR 0510 §1)", p) + } + } +} + +func TestTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder(t *testing.T) { + dir := t.TempDir() + plan := []RollbackItem{ + {Name: "bash", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"}, + {Name: "zlib", To: "1.3.1-2", Source: SourceRepository, Location: ArchiveURL + "/repos/2026/10/04/core/os/$arch"}, + {Name: "made-up-library", From: "1.0-1", Source: SourceRemoved}, + } + raw, _ := json.Marshal(plan) + file := filepath.Join(dir, aRun+".json") + if err := os.WriteFile(file, raw, 0o600); err != nil { + t.Fatal(err) + } + var ran [][]string + run := func(name string, args ...string) int { ran = append(ran, append([]string{name}, args...)); return 0 } + probed := []string{} + probe := func(url string) bool { probed = append(probed, url); return strings.HasSuffix(url, "-any.pkg.tar.zst") } + if code := rollbackUnit(file, "x86_64", run, probe, os.Stderr); code != 0 { + t.Fatalf("exit %d", code) + } + want := [][]string{ + {"pacman", "-U", "--noconfirm", "--", CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst", ArchiveURL + "/repos/2026/10/04/core/os/x86_64/zlib-1.3.1-2-any.pkg.tar.zst"}, + {"pacman", "-R", "--noconfirm", "--", "made-up-library"}, + } + if !reflect.DeepEqual(ran, want) { + t.Fatalf("ran %q, want %q", ran, want) + } + if len(probed) != 2 || !strings.HasSuffix(probed[0], "-x86_64.pkg.tar.zst") { + t.Errorf("the machine's architecture is tried first, then any: %q", probed) + } +} + +func TestTheRollbackUnitChangesNothingWhenAPackageIsNotFound(t *testing.T) { + file := filepath.Join(t.TempDir(), aRun+".json") + raw, _ := json.Marshal([]RollbackItem{{Name: "zlib", To: "1.3.1-2", Source: SourceRepository, Location: ArchiveURL + "/repos/2026/10/04/core/os/$arch"}}) + _ = os.WriteFile(file, raw, 0o600) + var ran [][]string + run := func(name string, args ...string) int { ran = append(ran, append([]string{name}, args...)); return 0 } + if code := rollbackUnit(file, "x86_64", run, func(string) bool { return false }, &strings.Builder{}); code == 0 || len(ran) != 0 { + t.Fatalf("exit %d, ran %q: a package the archive does not hold fails the unit before any change", code, ran) + } +} diff --git a/modules/pacman/cmd/pacman-tools/round_integration_test.go b/modules/pacman/cmd/pacman-tools/round_integration_test.go new file mode 100644 index 00000000..a6855d9b --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/round_integration_test.go @@ -0,0 +1,180 @@ +//go:build integration + +package main + +// The round over a real bus (novox/hq ADR 0508 §5, §6; acceptance 1841/AC15): the keepers of two made-up nodes, +// each whole (its own bus account, its states, its watches) against a JetStream server of the test's own, with +// fake machines and a fake root helper each. The first node of the order opens the week's round and runs; the +// second starts only once the first's after-upgrade check passed; a failed check stops the round there. +// Compiled by the merge gate's `go vet -tags integration`; run by the merge check, never by hand. + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// countingRoot is a node's root helper: it counts the transactions it was asked to start. +type countingRoot struct { + mu sync.Mutex + started []string +} + +func (r *countingRoot) Transaction(kind, run string, ignore []string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.started = append(r.started, kind+" "+run) + return nil +} +func (r *countingRoot) Rollback(string, []RollbackItem) error { return nil } +func (r *countingRoot) Journal(string, int) ([]string, error) { return nil, nil } +func (r *countingRoot) Maps() ([]Mapped, error) { return []Mapped{}, nil } +func (r *countingRoot) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.started) +} + +// roundNode is one made-up node's keeper on the test's bus. +type roundNode struct { + root *countingRoot + signals chan Signal +} + +// startRoundNode starts node's keeper; its machine's clock begins at the Saturday's opening, its transaction +// unit is gone (ended well) whenever asked, and its after-upgrade check passes unless passes says otherwise. +func startRoundNode(t *testing.T, ctx context.Context, url, node string, order []string, passes bool) *roundNode { + t.Helper() + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\nzsh 5.9-1\n", "zsh\n") + base := db.runner() + run := func(c context.Context, name string, args ...string) Ran { + line := strings.TrimSpace(name + " " + strings.Join(args, " ")) + switch { + case strings.HasPrefix(line, "systemctl list-units --state=failed"), line == "dkms status": + return Ran{} + case strings.HasPrefix(line, "systemctl show "+unitPrefix): + return Ran{Stdout: "LoadState=not-found\n"} + } + return base(c, name, args...) + } + m := machine(run, 1000) + began := time.Now() + m.Now = func() time.Time { return t0.Add(time.Since(began)) } + m.ReadFile = func(path string) ([]byte, error) { + if path == OSRelease { + return []byte("7.2.8-arch1-2\n"), nil + } + return nil, errNoFile + } + n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)} + k := &Keeper{Config: KeeperConfig{UpgradeOrder: order, RecordDays: 180, RoundStartMinutes: 120}, Machine: m, Clock: realClock{}, + Watcher: fakeWatcher{n.signals}, Logf: func(f string, a ...any) { t.Logf(node+": "+f, a...) }, MakeHooks: keeperHooks, + Root: n.root, Check: func(Run, *Record) Check { return Check{Passed: passes} }} + account := process.Account{URL: url, Node: node, Module: "pacman", User: node + ".pacman.keeper", Password: "pw"} + go func() { _ = runKeeper(ctx, account, k) }() + return n +} + +func roundBus(t *testing.T) (string, map[string]jetstream.KeyValue) { + t.Helper() + url := newTestBus(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + buckets := map[string]jetstream.KeyValue{} + for _, name := range States { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pacman_" + name, History: 8}) + if err != nil { + t.Fatal(err) + } + buckets[name] = kv + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "pacman-events", Subjects: []string{"mesh.mod.pacman.event.>"}}); err != nil { + t.Fatal(err) + } + return url, buckets +} + +func eventually(t *testing.T, what string, within time.Duration, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(within); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func roundNodeOnBus(kv jetstream.KeyValue, round, node string) RoundNode { + var rn RoundNode + if e, err := kv.Get(context.Background(), RoundNodeKey(round, node)); err == nil { + _ = json.Unmarshal(e.Value(), &rn) + } + return rn +} + +func TestTheRoundGoesNodeByNodeOnARealBus(t *testing.T) { + url, buckets := roundBus(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + order := []string{"node-a", "node-b"} + a := startRoundNode(t, ctx, url, "node-a", order, true) + b := startRoundNode(t, ctx, url, "node-b", order, true) + round := RoundID(t0) + + eventually(t, "node-a opened the round and started its run", 20*time.Second, func() bool { + return a.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-a").Run != "" + }) + time.Sleep(time.Second) + if b.root.count() != 0 { + t.Fatalf("node-b started before node-a passed: %v", b.root.started) + } + // node-a's transaction ends: its lock is released, its run is finished and checked. + a.signals <- LockReleased + eventually(t, "node-b started once node-a passed", 20*time.Second, func() bool { + return b.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != "" + }) + if a.root.count() != 1 { + t.Errorf("node-a ran again: %v", a.root.started) + } +} + +func TestAFailedCheckStopsTheRoundOnARealBus(t *testing.T) { + url, buckets := roundBus(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + order := []string{"node-a", "node-b"} + a := startRoundNode(t, ctx, url, "node-a", order, false) + b := startRoundNode(t, ctx, url, "node-b", order, true) + round := RoundID(t0) + eventually(t, "node-a started its run", 20*time.Second, func() bool { return a.root.count() == 1 }) + a.signals <- LockReleased + runA := "" + eventually(t, "node-a's run checked", 20*time.Second, func() bool { + runA = roundNodeOnBus(buckets[StateRounds], round, "node-a").Run + if runA == "" { + return false + } + e, err := buckets[StateRuns].Get(context.Background(), NodeKey("node-a", runA)) + return err == nil && strings.Contains(string(e.Value()), `"checked-after"`) + }) + time.Sleep(2 * time.Second) + if b.root.count() != 0 || roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != "" { + t.Fatalf("node-b started after node-a's check failed: %v", b.root.started) + } +} diff --git a/modules/pacman/cmd/pacman-tools/rounds.go b/modules/pacman/cmd/pacman-tools/rounds.go new file mode 100644 index 00000000..dcb10823 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/rounds.go @@ -0,0 +1,565 @@ +package main + +// The week's round (novox/hq ADR 0508 §5, §6): the nodes upgrade one at a time in the order of the mesh-level key +// `upgrade-order`, each after the node before it passed its after-upgrade check. On Saturday at 04:00 local time +// the first node of the order opens the week's round in `rounds` (a key created once, so one round a week); every +// node's keeper watches `rounds` and `runs` and starts its own run when its turn came. A failed run or check +// stops the round at that node. Only the operator passes a node: `pacman_round` with pass and a reason asks the +// operator through the operator's channel (ADR 0259), and the keeper of the node waiting on it lets the round go +// on only on the operator's Approve, read from the channel's own record. With no order, no round opens. + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// Module is this module's name: the asker of its asks on the operator's channel. +const Module = "pacman" + +// MessengerAsksBucket is the operator channel's records bucket, where the keeper reads the answer to a pass. +const MessengerAsksBucket = "messenger_asks" + +// The operator's answers to a pass, besides PassApproved and PassDeclined: an ask nobody answered in time. +const PassExpired = "expired" + +// passLasts is how long a pass waits for the operator's answer. +const passLasts = 12 * time.Hour + +// refusedRetry is how long after a run of the round was refused (it started nothing) the node tries again. +const refusedRetry = time.Hour + +// passCheckEvery is how often a keeper reads the channel's record of a pass that waits for its answer. +const passCheckEvery = time.Minute + +// defaultRoundStartMinutes is round-start-minutes' default in the manifest. +const defaultRoundStartMinutes = 120 + +// RoundOpening is the Saturday at 04:00 of the ISO week that holds at, in at's own time zone. +func RoundOpening(at time.Time) time.Time { + day := int(at.Weekday()+6) % 7 // Monday 0 … Sunday 6 + monday := time.Date(at.Year(), at.Month(), at.Day()-day, 0, 0, 0, 0, at.Location()) + return monday.AddDate(0, 0, 5).Add(4 * time.Hour) +} + +// PassOperatorAsk is the ask on the operator's channel for a pass: composed from the pass alone, so the tool that +// publishes it and the keeper that reads its answer compose the same ask, with the same digest. +func PassOperatorAsk(round, node, next, reason, id string, at time.Time) asks.Ask { + headline := "Upgrade round: pass " + node + if len([]rune(headline)) > asks.HeadlineLength { + headline = "Upgrade round: pass a node" + } + binds, _ := asks.ActDigest(passAct(round, node, reason, id)) + return asks.Ask{ + ID: id, + Headline: headline, + Explanation: fmt.Sprintf("The week's upgrade round %s stands at %s. Passing it lets %s and the nodes after it upgrade; "+ + "%s is not upgraded in this round.\nThe reason given: %s", round, node, next, node, reason), + Options: []asks.Option{ + {ID: "pass", Label: "Pass it", Does: "the round goes on past " + node, Level: asks.Approve, Binds: binds}, + {ID: "keep", Label: "Keep it waiting", Does: "the round stays at " + node, Level: asks.Acknowledge}, + }, + Who: asks.Operator, + Expires: at.UTC().Add(passLasts), + OnExpiry: "the round stays at " + node, + } +} + +func passAct(round, node, reason, id string) asks.Act { + return asks.Act{"verb": Module + ".pass", "round": round, "node": node, "reason": reason, "ask": id} +} + +// roundView is one round as the bus holds it: the round, each node's place, and each node's run in it. +type roundView struct { + Round Round + Nodes map[string]RoundNode + revs map[string]uint64 + Runs map[string]Run +} + +// latestRound is the newest round's id, or none. +func latestRound(rounds Bucket) (string, error) { + keys, err := rounds.Keys() + if err != nil { + return "", fmt.Errorf("listing rounds: %w", err) + } + latest := "" + for _, k := range keys { + if !strings.Contains(k, ".") && k > latest { + latest = k + } + } + return latest, nil +} + +func readRound(rounds, runs Bucket, id string) (*roundView, error) { + e, err := rounds.Get(id) + if err != nil || e == nil { + return nil, err + } + v := &roundView{Nodes: map[string]RoundNode{}, revs: map[string]uint64{}, Runs: map[string]Run{}} + if err := json.Unmarshal(e.Value, &v.Round); err != nil { + return nil, fmt.Errorf("round %s: %w", id, err) + } + for _, node := range v.Round.Order { + e, err := rounds.Get(RoundNodeKey(id, node)) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var rn RoundNode + if err := json.Unmarshal(e.Value, &rn); err != nil { + return nil, fmt.Errorf("round %s, node %s: %w", id, node, err) + } + v.Nodes[node], v.revs[node] = rn, e.Revision + if rn.Run == "" { + continue + } + re, err := runs.Get(NodeKey(node, rn.Run)) + if err != nil { + return nil, err + } + if re != nil { + var r Run + if err := json.Unmarshal(re.Value, &r); err != nil { + return nil, fmt.Errorf("run %s of %s: %w", rn.Run, node, err) + } + v.Runs[node] = r + } + } + return v, nil +} + +// checkedAt is when a run's after-upgrade check passed, or zero. +func checkedAt(r Run) time.Time { + for _, s := range r.Steps { + if s.Step == StepCheckedAfter && s.Check != nil && s.Check.Passed { + return s.At + } + } + return time.Time{} +} + +// passedAt is when a node let the round go on: its check passed, or the operator approved its pass; zero when +// it has not. +func (v *roundView) passedAt(node string) time.Time { + if rn := v.Nodes[node]; rn.Pass != nil && rn.Pass.Answer == PassApproved && rn.Pass.Answered != nil { + return *rn.Pass.Answered + } + if r, ok := v.Runs[node]; ok { + return checkedAt(r) + } + return time.Time{} +} + +// turnSince is when a node's turn came: the round's opening for the first, the passing of the node before it +// for the rest; zero when it has not come. +func (v *roundView) turnSince(node string) time.Time { + for i, n := range v.Round.Order { + if n != node { + continue + } + if i == 0 { + return v.Round.Opened + } + return v.passedAt(v.Round.Order[i-1]) + } + return time.Time{} +} + +// next is the node after node in the order, or none. +func (v *roundView) next(node string) string { + for i, n := range v.Round.Order { + if n == node && i+1 < len(v.Round.Order) { + return v.Round.Order[i+1] + } + } + return "" +} + +// putRoundNode writes a node's place in a round from the revision read, so a pass and a run written by two +// keepers never overwrite each other: a lost race is read again and changed again. +func putRoundNode(rounds Bucket, round, node string, change func(*RoundNode)) error { + key := RoundNodeKey(round, node) + for attempt := 0; attempt < 5; attempt++ { + e, err := rounds.Get(key) + if err != nil { + return err + } + rn := RoundNode{Round: round, Node: node} + if e != nil { + if err := json.Unmarshal(e.Value, &rn); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + change(&rn) + _, err = rounds.Update(key, rn, e.Revision) + } else { + change(&rn) + _, err = rounds.Create(key, rn) + } + if !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("%s was written by another keeper five times in a row", key) +} + +// RoundStep does what the node's keeper has to do for the rounds now, and answers when it should be called +// again (zero: only when `rounds` or `runs` change). The keeper calls it at its start, at the time it answered, +// and whenever `rounds` or `runs` change. The first node of the order opens the week's round once it is +// Saturday 04:00; a node whose turn came starts its run; a node waiting on a pass reads the operator's answer. +func (u *Upgrades) RoundStep() (time.Time, error) { + now := u.Machine.Now() + order := u.Config.UpgradeOrder + var next time.Time + soonest := func(t time.Time) { + if !t.IsZero() && (next.IsZero() || t.Before(next)) { + next = t + } + } + if len(order) == 0 { + return time.Time{}, nil // no order: no round (the health says so) + } + var errs []error + if order[0] == u.Node { + opening := RoundOpening(now) + if now.Before(opening) { + soonest(opening) + } else { + soonest(opening.AddDate(0, 0, 7)) + id := RoundID(now) + round := Round{Round: id, Opened: now.UTC(), OpenedBy: u.Node, Order: append([]string(nil), order...)} + if _, err := u.Rounds.Create(id, round); err != nil && !errors.Is(err, stdio.ErrChanged) { + errs = append(errs, fmt.Errorf("opening round %s: %w", id, err)) + } + } + } + id, err := latestRound(u.Rounds) + if err != nil || id == "" { + return next, errors.Join(append(errs, err)...) + } + v, err := readRound(u.Rounds, u.Runs, id) + if err != nil || v == nil { + return next, errors.Join(append(errs, err)...) + } + // A pass of the node before this one that waits for the operator's answer. + for i, node := range v.Round.Order { + if i+1 >= len(v.Round.Order) || v.Round.Order[i+1] != u.Node { + continue + } + if rn := v.Nodes[node]; rn.Pass != nil && rn.Pass.Answer == "" { + answered, err := u.readPass(v, node) + if err != nil { + errs = append(errs, err) + } + if !answered { + soonest(now.Add(passCheckEvery)) + } else if v, err = readRound(u.Rounds, u.Runs, id); err != nil { + return next, errors.Join(append(errs, err)...) + } + } + } + t, err := u.takeTurn(v, now) + soonest(t) + return next, errors.Join(append(errs, err)...) +} + +// takeTurn starts this node's run in the round when its turn came and it has none, and again an hour after a +// run of it was refused (it started nothing); it answers when to look again. +func (u *Upgrades) takeTurn(v *roundView, now time.Time) (time.Time, error) { + if !contains(v.Round.Order, u.Node) || v.turnSince(u.Node).IsZero() || !v.passedAt(u.Node).IsZero() { + return time.Time{}, nil + } + if rn := v.Nodes[u.Node]; rn.Pass != nil && rn.Pass.Answer == "" { + return time.Time{}, nil // the operator is asked to pass this node: it waits for the answer + } + own, err := u.ownRuns() + if err != nil { + return time.Time{}, err + } + var latest *Run + for i := range own { + if own[i].Round == v.Round.Round && (latest == nil || own[i].Run.Run > latest.Run) { + latest = &own[i].Run + } + } + if latest != nil { + if v.Nodes[u.Node].Run != latest.Run { + // a keeper that went away between starting its run and naming it in the round + if err := putRoundNode(u.Rounds, v.Round.Round, u.Node, func(rn *RoundNode) { rn.Run = latest.Run }); err != nil { + return time.Time{}, err + } + } + if lastStep(*latest) != StepRefused { + return time.Time{}, nil // running, or ended: a failure stops the round here + } + retry := latest.Steps[len(latest.Steps)-1].At.Add(refusedRetry) + if now.Before(retry) { + return retry, nil + } + } + r, err := u.StartUpgrade(FromRound, v.Round.Round, "") + if len(r.Steps) == 0 { + return time.Time{}, err + } + if perr := putRoundNode(u.Rounds, v.Round.Round, u.Node, func(rn *RoundNode) { rn.Run = r.Run }); perr != nil { + err = errors.Join(err, perr) + } + if lastStep(r) == StepRefused { + return r.Steps[len(r.Steps)-1].At.Add(refusedRetry), err + } + return time.Time{}, err +} + +// readPass reads the operator's answer to a pass of node from the channel's record, and writes it into the +// round once there is one. It answers whether the pass is answered. +func (u *Upgrades) readPass(v *roundView, node string) (bool, error) { + rn := v.Nodes[node] + p := rn.Pass + if u.AskRecord == nil { + return false, errors.New("no reader of the operator channel's records: a pass cannot be answered") + } + e, err := u.AskRecord(p.Ask) + if err != nil { + return false, fmt.Errorf("reading the operator's answer to the pass of %s: %w", node, err) + } + if e == nil { + return false, nil + } + var rec struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + } + if err := json.Unmarshal(e.Value, &rec); err != nil { + return false, fmt.Errorf("the channel's record of the pass of %s: %w", node, err) + } + if rec.State == "" || rec.State == "open" || rec.Warrant == nil { + return false, nil + } + w := *rec.Warrant + answer := PassDeclined + switch { + case w.Outcome == asks.OutcomeExpired: + answer = PassExpired + case w.Outcome == asks.OutcomeChosen: + ask := PassOperatorAsk(v.Round.Round, node, v.next(node), p.Reason, p.Ask, p.Asked) + option, err := w.For(Module, ask) + if err == nil { + err = option.Performs(passAct(v.Round.Round, node, p.Reason, p.Ask)) + } + if err != nil { + return false, fmt.Errorf("the answer to the pass of %s is not the operator's to this pass: %w", node, err) + } + if option.ID == "pass" { + answer = PassApproved + } + } + at := u.Machine.Now().UTC() + return true, putRoundNode(u.Rounds, v.Round.Round, node, func(rn *RoundNode) { + if rn.Pass != nil && rn.Pass.Ask == p.Ask && rn.Pass.Answer == "" { + rn.Pass.Answer, rn.Pass.Answered = answer, &at + } + }) +} + +// takePass is a pass ask taken by the keeper of the node waiting on the passed node: the pass is written into +// the round, waiting for the operator's answer (the tool asked the operator). +func (u *Upgrades) takePass(a Ask) error { + v, err := readRound(u.Rounds, u.Runs, a.Round) + if err != nil { + return err + } + switch { + case v == nil: + return fmt.Errorf("ask %s passes %s in round %s, which is not open", a.ID, a.Pass, a.Round) + case v.next(a.Pass) != u.Node: + return fmt.Errorf("ask %s passes %s, on which %s does not wait", a.ID, a.Pass, u.Node) + } + return putRoundNode(u.Rounds, a.Round, a.Pass, func(rn *RoundNode) { + if rn.Pass == nil || rn.Pass.Ask != a.ID { // taken again after a restart: the answer already read stays + rn.Pass = &Pass{Reason: a.Reason, Ask: a.ID, Asked: a.At.UTC()} + } + }) +} + +// NodeInRound is where one node stands in a round, as pacman_round and the health say it. +type NodeInRound struct { + Node string `json:"node"` + State string `json:"state"` + Run string `json:"run,omitempty"` + Since *time.Time `json:"since,omitempty"` + WaitsFor string `json:"waits_for,omitempty"` + Pass *Pass `json:"pass,omitempty"` +} + +// The states of a node in a round. +const ( + InRoundWaiting = "waiting" // the node before it has not passed + InRoundTurn = "turn" // its turn came, its run has not started + InRoundStalled = "stalled" // its turn came longer than round-start-minutes ago, and it has not started + InRoundRunning = "running" // its run goes on + InRoundPassed = "passed" // its check passed, or the operator passed it + InRoundFailed = "failed" // its run failed or its check did not pass: the round stops here + InRoundRefused = "refused" // its run did not start (it is tried again) + InRoundOff = "off-order" // not in the round's order +) + +// RoundState is where every node of the order stands in a round at now. +func RoundState(v *roundView, now time.Time, startMinutes int) []NodeInRound { + if startMinutes < 1 { + startMinutes = defaultRoundStartMinutes + } + out := []NodeInRound{} + for i, node := range v.Round.Order { + s := NodeInRound{Node: node, Run: v.Nodes[node].Run, Pass: v.Nodes[node].Pass} + at := func(t time.Time) *time.Time { + if t.IsZero() { + return nil + } + t = t.UTC() + return &t + } + r, hasRun := v.Runs[node] + switch { + case !v.passedAt(node).IsZero(): + s.State, s.Since = InRoundPassed, at(v.passedAt(node)) + case hasRun && lastStep(r) == StepRefused: + s.State, s.Since = InRoundRefused, at(r.Steps[len(r.Steps)-1].At) + case hasRun && lastStep(r) == StepCheckedAfter: + s.State, s.Since = InRoundFailed, at(r.Steps[len(r.Steps)-1].At) + case hasRun: + s.State = InRoundRunning + if len(r.Steps) > 0 { + s.Since = at(r.Steps[0].At) + } + case !v.turnSince(node).IsZero(): + since := v.turnSince(node) + s.State, s.Since = InRoundTurn, at(since) + if now.Sub(since) > time.Duration(startMinutes)*time.Minute { + s.State = InRoundStalled + } + default: + s.State = InRoundWaiting + for j := i - 1; j >= 0; j-- { + if v.passedAt(v.Round.Order[j]).IsZero() { + s.WaitsFor = v.Round.Order[j] + } + } + if s.WaitsFor != "" { + // since the node it waits for stopped or its turn came + prev := out[indexOf(v.Round.Order, s.WaitsFor)] + s.Since = prev.Since + } + } + out = append(out, s) + } + return out +} + +func indexOf(list []string, s string) int { + for i, x := range list { + if x == s { + return i + } + } + return -1 +} + +// seatPublish publishes on a seat for the tools bundle; a test replaces it. +var seatPublish = stdio.SeatPublish + +// ListRounds answers pacman_round's read: the newest rounds, or one, each with every node's state. +func ListRounds(rounds, runs Bucket, round string, now time.Time, startMinutes, limit int) (map[string]any, error) { + keys, err := rounds.Keys() + if err != nil { + return nil, err + } + var ids []string + for _, k := range keys { + if !strings.Contains(k, ".") && (round == "" || k == round) { + ids = append(ids, k) + } + } + if round != "" && len(ids) == 0 { + return nil, fmt.Errorf("no round %s", round) + } + sort.Sort(sort.Reverse(sort.StringSlice(ids))) + if len(ids) > limit { + ids = ids[:limit] + } + listed := []map[string]any{} + for _, id := range ids { + v, err := readRound(rounds, runs, id) + if err != nil || v == nil { + return nil, err + } + listed = append(listed, map[string]any{"round": id, "opened": v.Round.Opened, "opened_by": v.Round.OpenedBy, + "order": v.Round.Order, "nodes": RoundState(v, now, startMinutes)}) + } + return map[string]any{"rounds": listed}, nil +} + +// AskPass asks the operator to let a round go on past a node: the ask on the operator's channel, and the ask in +// `asks` for the keeper of the node waiting on it, which reads the answer. A reason is required. +func AskPass(rounds, runs, asksState Bucket, round, node, reason string, now time.Time) (map[string]any, error) { + if strings.TrimSpace(reason) == "" { + return nil, errors.New("a pass needs a reason: the operator is asked to let the round go on past a node, and is told why") + } + if round == "" { + id, err := latestRound(rounds) + if err != nil { + return nil, err + } + round = id + } + v, err := readRound(rounds, runs, round) + if err != nil { + return nil, err + } + if v == nil { + return nil, fmt.Errorf("no round %s is open", round) + } + if !contains(v.Round.Order, node) { + return nil, fmt.Errorf("%s is not in round %s's order (%s)", node, round, strings.Join(v.Round.Order, ", ")) + } + if !v.passedAt(node).IsZero() { + return nil, fmt.Errorf("%s has passed already in round %s", node, round) + } + next := v.next(node) + if next == "" { + return nil, fmt.Errorf("%s is the last of the order: no node waits on it", node) + } + a := Ask{ID: AskID(now), Node: next, Act: ActPass, At: now.UTC(), Round: round, Pass: node, Reason: reason} + opAsk := PassOperatorAsk(round, node, next, reason, a.ID, a.At) + if err := opAsk.Check(now); err != nil { + return nil, err + } + if err := seatPublish(asks.AskSubject(Module), opAsk, opAsk.ID); err != nil { + return nil, fmt.Errorf("the operator could not be asked: %w; nothing was passed", err) + } + if _, err := asksState.Create(NodeKey(next, a.ID), a); err != nil { + return nil, fmt.Errorf("the operator was asked, and the ask for %s's keeper could not be written: %w; an Approve passes nothing", next, err) + } + return map[string]any{"asked": opAsk.ID, "round": round, "pass": node, "waiting": next, + "next": "the round goes on past " + node + " only when the operator approves on their channel; pacman_round says where it stands"}, nil +} + +// roundStartMinutes is round-start-minutes as the tools bundle reads it: from the keeper's config file, which +// the manifest names to the bundle too (EnvConfig); its default when the file cannot be read. +var roundStartMinutes = func() int { + raw, err := os.ReadFile(os.Getenv(EnvConfig)) + var c KeeperConfig + if err != nil || json.Unmarshal(raw, &c) != nil || c.RoundStartMinutes < 1 { + return defaultRoundStartMinutes + } + return c.RoundStartMinutes +} diff --git a/modules/pacman/cmd/pacman-tools/rounds_test.go b/modules/pacman/cmd/pacman-tools/rounds_test.go new file mode 100644 index 00000000..26a54748 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/rounds_test.go @@ -0,0 +1,305 @@ +package main + +// The week's round (novox/hq ADR 0508 §5, §6; acceptance 1841/AC6 and 1841/AC7), for three made-up nodes over +// states in memory. + +import ( + "encoding/json" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// roundBed is three keepers over one bed's states, each with the order node-a, node-b, node-c. +type roundBed struct { + *keeperBed + nodes map[string]*Upgrades +} + +func newRoundBed(t *testing.T, order ...string) *roundBed { + k := newKeeperBed(t) + k.now = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC) // a Saturday + b := &roundBed{keeperBed: k, nodes: map[string]*Upgrades{}} + var updates UpdatesValue + k.updates.value("node-a", &updates) + for _, n := range []string{"node-a", "node-b", "node-c"} { + u := k.keeper(k.u.Machine) + u.Node, u.Config.UpgradeOrder, u.Config.RoundStartMinutes = n, order, 120 + if _, err := k.installed.Put(n, nodeAPackages); err != nil { + t.Fatal(err) + } + if _, err := k.updates.Put(n, updates); err != nil { + t.Fatal(err) + } + b.nodes[n] = u + } + return b +} + +func (b *roundBed) step(t *testing.T, node string) time.Time { + t.Helper() + next, err := b.nodes[node].RoundStep() + if err != nil { + t.Fatalf("%s: %v", node, err) + } + return next +} + +// end ends node's running transaction as unitState says and follows it. +func (b *roundBed) end(t *testing.T, node, unitState string) { + t.Helper() + b.unitState = unitState + if err := b.nodes[node].Follow(); err != nil { + t.Fatal(err) + } + b.now = b.now.Add(5 * time.Minute) +} + +func (b *roundBed) runsOf(node string) []Run { + keys, _ := b.runs.Keys() + var out []Run + for _, k := range keys { + if strings.HasPrefix(k, node+".") { + var r Run + b.runs.value(k, &r) + out = append(out, r) + } + } + return out +} + +func (b *roundBed) state(t *testing.T) map[string]NodeInRound { + t.Helper() + v, err := readRound(b.rounds, b.runs, "2026-W43") + if err != nil || v == nil { + t.Fatalf("round: %v", err) + } + out := map[string]NodeInRound{} + for _, s := range RoundState(v, b.now, 120) { + out[s.Node] = s + } + return out +} + +func TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn(t *testing.T) { + if o := RoundOpening(time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)); !o.Equal(time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC)) { + t.Fatalf("the opening of the week of Thursday 22 October: %s", o) + } + b := newRoundBed(t, "node-a", "node-b", "node-c") + if next := b.step(t, "node-a"); !next.Equal(time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC)) { + t.Errorf("before 04:00 the first node waits for 04:00: %s", next) + } + if keys, _ := b.rounds.Keys(); len(keys) != 0 { + t.Fatalf("a round before 04:00: %v", keys) + } + b.step(t, "node-b") + if keys, _ := b.rounds.Keys(); len(keys) != 0 { + t.Fatalf("only the first of the order opens the round: %v", keys) + } + + b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + if next := b.step(t, "node-a"); !next.Equal(time.Date(2026, 10, 31, 4, 0, 0, 0, time.UTC)) { + t.Errorf("after opening, next Saturday: %s", next) + } + var round Round + if !b.rounds.value("2026-W43", &round) || round.OpenedBy != "node-a" || strings.Join(round.Order, " ") != "node-a node-b node-c" { + t.Fatalf("the round: %+v", round) + } + if r := b.runsOf("node-a"); len(r) != 1 || r[0].Round != "2026-W43" || r[0].From != FromRound || lastStep(r[0]) != StepStarted { + t.Fatalf("the first node starts its run in the round: %+v", r) + } + b.now = b.now.Add(time.Minute) + b.step(t, "node-a") + if w := b.rounds.writesTo("2026-W43"); len(w) != 2 || len(b.runsOf("node-a")) != 1 { + t.Errorf("the round is created once and the run started once: %v", w) + } + b.step(t, "node-b") + if len(b.runsOf("node-b")) != 0 || b.state(t)["node-b"].State != InRoundWaiting { + t.Fatalf("node-b waits while node-a runs: %+v", b.state(t)["node-b"]) + } + + b.end(t, "node-a", "LoadState=not-found\n") + if s := b.state(t)["node-a"]; s.State != InRoundPassed { + t.Fatalf("node-a: %+v", s) + } + b.step(t, "node-c") + if len(b.runsOf("node-c")) != 0 { + t.Fatal("node-c started before node-b") + } + b.step(t, "node-b") + if r := b.runsOf("node-b"); len(r) != 1 || r[0].Round != "2026-W43" { + t.Fatalf("node-b's turn after node-a's check passed: %+v", r) + } + + // node-b's transaction fails: the round stops at node-b, and node-c waits. + b.end(t, "node-b", "LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\n") + b.step(t, "node-c") + st := b.state(t) + if len(b.runsOf("node-c")) != 0 || st["node-b"].State != InRoundFailed || st["node-c"].State != InRoundWaiting || st["node-c"].WaitsFor != "node-b" { + t.Errorf("a failed check stops the round: %+v %+v", st["node-b"], st["node-c"]) + } +} + +func TestNoOrderNoRound(t *testing.T) { + b := newRoundBed(t) + b.now = time.Date(2026, 10, 24, 5, 0, 0, 0, time.UTC) + if next := b.step(t, "node-a"); !next.IsZero() { + t.Errorf("no wake without an order: %s", next) + } + if keys, _ := b.rounds.Keys(); len(keys) != 0 || len(b.runsOf("node-a")) != 0 { + t.Errorf("a round without an order: %v", keys) + } +} + +func TestANodeOffAtItsTurnRunsAtItsNextStartAndAStalledTurnIsSaid(t *testing.T) { + b := newRoundBed(t, "node-a", "node-b", "node-c") + b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + b.step(t, "node-a") + b.end(t, "node-a", "LoadState=not-found\n") + passed := b.now.Add(-5 * time.Minute) + b.now = passed.Add(121 * time.Minute) // node-b is off + st := b.state(t) + if st["node-b"].State != InRoundStalled || st["node-b"].Since == nil || !st["node-b"].Since.Equal(passed) { + t.Errorf("node-b's turn came at %s and it has not started: %+v", passed, st["node-b"]) + } + if st["node-c"].WaitsFor != "node-b" || st["node-c"].State != InRoundWaiting { + t.Errorf("node-c names the node it waits for: %+v", st["node-c"]) + } + b.step(t, "node-b") // node-b starts again + if len(b.runsOf("node-b")) != 1 { + t.Error("a node off at its turn runs at its next start") + } +} + +// warrant is the operator channel's record of an ask answered with option. +func warrant(a asks.Ask, option string, level asks.Level, at time.Time) json.RawMessage { + w := asks.Warrant{Ask: a.ID, Asker: Module, Outcome: asks.OutcomeChosen, Option: option, Level: level, + By: &asks.Person{Who: asks.Operator, Kind: "made-up-channel", Identity: "1", Verified: "user id"}, At: at, AskDigest: a.Digest()} + raw, _ := json.Marshal(map[string]any{"state": "chosen", "warrant": w}) + return raw +} + +func TestOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason(t *testing.T) { + b := newRoundBed(t, "node-a", "node-b", "node-c") + b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + b.step(t, "node-a") + b.end(t, "node-a", "LoadState=not-found\n") + b.step(t, "node-b") + b.end(t, "node-b", "LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\n") + + var published []asks.Ask + var subjects []string + saved := seatPublish + seatPublish = func(subject string, body any, id string) error { + subjects, published = append(subjects, subject), append(published, body.(asks.Ask)) + return nil + } + defer func() { seatPublish = saved }() + if _, err := AskPass(b.rounds, b.runs, b.asks, "", "node-b", " ", b.now); err == nil || len(published) != 0 { + t.Fatalf("a pass without a reason: %v", err) + } + out, err := AskPass(b.rounds, b.runs, b.asks, "", "node-b", "its disk is being replaced", b.now) + if err != nil { + t.Fatal(err) + } + if len(published) != 1 || subjects[0] != "mesh.seat.operator-channel.accept.ask.pacman" || out["waiting"] != "node-c" { + t.Fatalf("the operator is asked on their channel: %v %v", subjects, out) + } + opAsk := published[0] + if err := opAsk.Check(b.now); err != nil || !strings.Contains(opAsk.Explanation, "its disk is being replaced") { + t.Errorf("the ask: %v %+v", err, opAsk) + } + keys, _ := b.asks.Keys() + if len(keys) != 1 || keys[0] != NodeKey("node-c", opAsk.ID) { + t.Fatalf("the ask for node-c's keeper: %v", keys) + } + var a Ask + b.asks.value(keys[0], &a) + raw, _ := json.Marshal(a) + if err := b.nodes["node-c"].TakeAsk(stdio.StateChange{State: StateAsks, Key: keys[0], Op: "put", Value: raw}); err != nil { + t.Fatal(err) + } + if st := b.state(t)["node-b"]; st.Pass == nil || st.Pass.Reason != "its disk is being replaced" || st.Pass.Answer != "" { + t.Fatalf("the pass waits in the round: %+v", st.Pass) + } + + record := map[string]json.RawMessage{} + b.nodes["node-c"].AskRecord = func(id string) (*stdio.StateEntry, error) { + if v, ok := record[id]; ok { + return &stdio.StateEntry{Key: Module + "." + id, Value: v}, nil + } + return nil, nil + } + if next := b.step(t, "node-c"); len(b.runsOf("node-c")) != 0 || !next.Equal(b.now.Add(passCheckEvery)) { + t.Fatalf("no answer yet: node-c waits and looks again in a minute (%s)", next) + } + // An answer to another ask, or to this one as it was not shown, passes nothing. + other := opAsk + other.Explanation = "something else" + record[opAsk.ID] = warrant(other, "pass", asks.Approve, b.now) + if _, err := b.nodes["node-c"].RoundStep(); err == nil || len(b.runsOf("node-c")) != 0 { + t.Fatalf("a warrant for another ask: %v", err) + } + record[opAsk.ID] = warrant(opAsk, "pass", asks.Approve, b.now) + b.step(t, "node-c") + if st := b.state(t)["node-b"]; st.State != InRoundPassed || st.Pass.Answer != PassApproved { + t.Errorf("the operator's Approve passes node-b: %+v", st) + } + if r := b.runsOf("node-c"); len(r) != 1 || r[0].Round != "2026-W43" { + t.Errorf("node-c's turn after the pass: %+v", r) + } +} + +func TestAPassTheOperatorKeepsWaitingPassesNothing(t *testing.T) { + b := newRoundBed(t, "node-a", "node-b") + b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + b.step(t, "node-a") // node-a runs and never ends + saved := seatPublish + var opAsk asks.Ask + seatPublish = func(_ string, body any, _ string) error { opAsk = body.(asks.Ask); return nil } + defer func() { seatPublish = saved }() + if _, err := AskPass(b.rounds, b.runs, b.asks, "2026-W43", "node-a", "it hangs", b.now); err != nil { + t.Fatal(err) + } + keys, _ := b.asks.Keys() + var a Ask + b.asks.value(keys[0], &a) + raw, _ := json.Marshal(a) + if err := b.nodes["node-b"].TakeAsk(stdio.StateChange{State: StateAsks, Key: keys[0], Op: "put", Value: raw}); err != nil { + t.Fatal(err) + } + b.nodes["node-b"].AskRecord = func(string) (*stdio.StateEntry, error) { + return &stdio.StateEntry{Value: warrant(opAsk, "keep", asks.Acknowledge, b.now)}, nil + } + b.step(t, "node-b") + if st := b.state(t); st["node-a"].Pass.Answer != PassDeclined || len(b.runsOf("node-b")) != 0 { + t.Errorf("kept waiting: %+v", st["node-a"]) + } + if _, err := AskPass(b.rounds, b.runs, b.asks, "2026-W43", "node-b", "x", b.now); err == nil { + t.Error("the last of the order has no node waiting on it") + } + if _, err := AskPass(b.rounds, b.runs, b.asks, "2026-W43", "node-z", "x", b.now); err == nil { + t.Error("a node not in the order") + } +} + +func TestPacmanRoundReadsTheRounds(t *testing.T) { + b := newRoundBed(t, "node-a", "node-b") + b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + b.step(t, "node-a") + out, err := ListRounds(b.rounds, b.runs, "", b.now, 120, 4) + if err != nil { + t.Fatal(err) + } + listed := out["rounds"].([]map[string]any) + nodes := listed[0]["nodes"].([]NodeInRound) + if len(listed) != 1 || listed[0]["round"] != "2026-W43" || nodes[0].State != InRoundRunning || nodes[1].WaitsFor != "node-a" { + t.Errorf("%+v", listed) + } + if _, err := ListRounds(b.rounds, b.runs, "2026-W40", b.now, 120, 4); err == nil { + t.Error("a round that is not there") + } +} diff --git a/modules/pacman/cmd/pacman-tools/runs_test.go b/modules/pacman/cmd/pacman-tools/runs_test.go new file mode 100644 index 00000000..d339b695 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/runs_test.go @@ -0,0 +1,217 @@ +package main + +// Runs on the bus and the log read once (novox/hq ADR 0508 §1; acceptance 1841/AC1 for the writing of steps, +// 1841/AC2 for the log and pacman_upgrades). + +import ( + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func capturedLog(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("testdata/pacman-log-three-full-upgrades/pacman.log") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// The captured log holds three full upgrades among other commands: one that ended before any transaction, one +// whose transaction completed, and a later one that upgraded one package. +func TestEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { + runs := PastUpgrades("node-a", capturedLog(t)) + if len(runs) != 3 { + t.Fatalf("%d runs, want the three full upgrades and none of the other commands: %+v", len(runs), runs) + } + for _, r := range runs { + if r.Node != "node-a" || r.Kind != RunUpgrade || r.From != FromLog || len(r.Steps) != 2 || r.Steps[0].Step != StepStarted { + t.Errorf("a run read from the log: %+v", r) + } + } + first, second, third := runs[0], runs[1], runs[2] + if first.Run != "20261004T130648Z" || first.Steps[1].Step != StepFinished || first.Steps[1].Why != noTransaction || + len(first.Steps[1].Changes.Changed) != 0 { + t.Errorf("an upgrade with no transaction says so: %+v", first) + } + end := second.Steps[1] + if second.Run != "20261004T130724Z" || end.Step != StepFinished || !end.At.Equal(time.Date(2026, 10, 4, 13, 22, 50, 0, time.UTC)) { + t.Errorf("the second upgrade finished when its transaction completed: %+v", second) + } + var names []string + for _, c := range end.Changes.Changed { + names = append(names, c.Name) + } + if strings.Join(names, " ") != "bash glibc iana-etc readline tzdata wireshark-qt xfsprogs xpad xterm zathura-pdf-mupdf" { + t.Errorf("what it changed, sorted: %v", names) + } + if c := end.Changes.Changed[0]; c.From != "5.3.15-1" || c.To != "5.3.20-1" { + t.Errorf("bash: %+v", c) + } + if len(end.Changes.Removed) != 0 { + t.Errorf("the removal run between them is no part of an upgrade: %+v", end.Changes.Removed) + } + if c := third.Steps[1].Changes.Changed; third.Run != "20261004T132817Z" || len(c) != 1 || c[0].Name != "dunst" { + t.Errorf("the third upgrade: %+v", third) + } +} + +func TestATransactionWithoutAnEndOrThatFailedIsAFailedRun(t *testing.T) { + log := "[2026-10-10T04:00:00+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + + "[2026-10-10T04:00:01+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-10T04:00:05+0000] [ALPM] transaction started\n" + + "[2026-10-10T04:00:06+0000] [ALPM] upgraded linux (7.2.8.arch1-2 -> 7.2.9.arch1-1)\n" + + "[2026-10-10T04:00:09+0000] [ALPM] transaction failed\n" + + "[2026-10-11T04:00:00+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + + "[2026-10-11T04:00:01+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-11T04:00:05+0000] [ALPM] transaction started\n" + runs := PastUpgrades("node-b", log) + if len(runs) != 2 { + t.Fatalf("%+v", runs) + } + if s := runs[0].Steps[1]; s.Step != StepFailed || s.Error != "transaction failed" || len(s.Changes.Changed) != 1 { + t.Errorf("a failed transaction: %+v", s) + } + if s := runs[1].Steps[1]; s.Step != StepFailed || !strings.Contains(s.Error, "no end") { + t.Errorf("a transaction the log holds no end of: %+v", s) + } +} + +func TestTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing(t *testing.T) { + m := machine(nil, 1000) + reads := 0 + log := capturedLog(t) + m.ReadFile = func(path string) ([]byte, error) { + if path != PacmanLog { + t.Fatalf("read %s", path) + } + reads++ + return []byte(log), nil + } + runs := newBucket() + other := Run{Node: "node-b", Run: "20261001T040000Z", Kind: RunUpgrade, From: FromRound} + if _, err := runs.Create(NodeKey("node-b", other.Run), other); err != nil { + t.Fatal(err) + } + u := &Upgrades{Node: "node-a", Machine: m, Runs: runs} + n, err := u.ImportLog() + if err != nil || n != 3 || reads != 1 { + t.Fatalf("first start: %d runs, %d reads, %v", n, reads, err) + } + var r Run + if !runs.value("node-a.20261004T132817Z", &r) || r.From != FromLog { + t.Errorf("the run is kept at ., marked as read from the log: %+v", r) + } + before := len(runs.writes) + n, err = u.ImportLog() + if err != nil || n != 0 || reads != 1 || len(runs.writes) != before { + t.Errorf("a second start: %d runs, %d reads, %d writes, %v", n, reads, len(runs.writes)-before, err) + } +} + +// A run's key is created at its first step and then only updated from the revision its writer holds: a step +// is written once, and a run another writer changed meanwhile is refused, never overwritten. +func TestAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten(t *testing.T) { + runs := newBucket() + u := &Upgrades{Node: "node-a", Runs: runs} + r := &openRun{Run: Run{Node: "node-a", Run: "20261024T040000Z", Kind: RunUpgrade, From: FromAsk}} + at := time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) + for _, s := range []string{StepRecorded, StepCheckedBefore} { + if err := u.step(r, Step{Step: s, At: at}); err != nil { + t.Fatal(err) + } + } + if w := strings.Join(runs.writes, ", "); w != "create node-a.20261024T040000Z, update node-a.20261024T040000Z" { + t.Errorf("writes: %s", w) + } + var kept Run + runs.value("node-a.20261024T040000Z", &kept) + if len(kept.Steps) != 2 || kept.Steps[1].Step != StepCheckedBefore { + t.Errorf("kept: %+v", kept) + } + if _, err := runs.Put("node-a.20261024T040000Z", kept); err != nil { + t.Fatal(err) + } + if err := u.step(r, Step{Step: StepStarted, At: at}); !errors.Is(err, stdio.ErrChanged) || len(r.Steps) != 2 { + t.Errorf("a step over another writer's change: %v, %d steps held", err, len(r.Steps)) + } +} + +func TestPacmanUpgradesAnswersAllOneNodeDaysAndOneRun(t *testing.T) { + runs := newBucket() + put := func(node, id string, steps ...Step) { + r := Run{Node: node, Run: id, Kind: RunUpgrade, From: FromRound, Round: "2026-W43", Steps: steps} + if _, err := runs.Create(NodeKey(node, id), r); err != nil { + t.Fatal(err) + } + } + at := time.Date(2026, 10, 24, 4, 30, 0, 0, time.UTC) + passed := &Check{Passed: true} + put("node-a", "20261024T040000Z", Step{Step: StepRecorded, At: at}, Step{Step: StepFinished, At: at}, Step{Step: StepCheckedAfter, At: at, Check: passed}) + put("node-b", "20261024T050000Z", Step{Step: StepRecorded, At: at}, Step{Step: StepStarted, At: at}) + put("node-b", "20261017T050000Z", Step{Step: StepRefused, At: at, Why: "installed is not current"}) + all, err := ListUpgrades(runs, "", "", "", "", 50) + if err != nil { + t.Fatal(err) + } + listed := all["runs"].([]RunSummary) + var order []string + for _, s := range listed { + order = append(order, s.Node+"/"+s.Run+"/"+s.Outcome+"/"+s.Checked) + } + if strings.Join(order, " ") != "node-b/20261024T050000Z/running/ node-a/20261024T040000Z/finished/passed node-b/20261017T050000Z/refused/" { + t.Fatalf("all runs, newest first: %v", order) + } + if listed[2].Error != "installed is not current" { + t.Errorf("a refused run says why: %+v", listed[2]) + } + one, _ := ListUpgrades(runs, "node-b", "", "", "", 50) + if one["count"] != 2 { + t.Errorf("one node: %v", one) + } + days, _ := ListUpgrades(runs, "", "2026-10-18", "2026-10-24", "", 50) + if days["count"] != 2 { + t.Errorf("between two days, the last one included: %v", days) + } + limited, _ := ListUpgrades(runs, "", "", "", "", 1) + if limited["count"] != 1 || limited["truncated"] != true { + t.Errorf("limited: %v", limited) + } + whole, err := ListUpgrades(runs, "node-a", "", "", "20261024T040000Z", 50) + if err != nil || len(whole["run"].(Run).Steps) != 3 { + t.Errorf("one run whole: %v %v", whole, err) + } + if _, err := ListUpgrades(runs, "node-a", "", "", "20261024T050000Z", 50); err == nil { + t.Error("a run the node does not have was answered") + } + if _, err := ListUpgrades(runs, "", "", "", "20261024T040000Z", 50); err == nil { + t.Error("a run without its node was answered") + } +} + +func TestTheUpgradesToolReadsTheRunsState(t *testing.T) { + runs := newBucket() + saved := upgradeState + upgradeState = func(name string) Bucket { + if name != StateRuns { + t.Fatalf("read %s", name) + } + return runs + } + defer func() { upgradeState = saved }() + for _, tool := range tools(machine(nil, 1000)) { + if tool.Name == "pacman_upgrades" { + out, err := tool.Run(map[string]any{"node": "node-a"}) + if err != nil || out.(map[string]any)["count"] != 0 { + t.Errorf("%v %v", out, err) + } + return + } + } + t.Fatal("no pacman_upgrades") +} diff --git a/modules/pacman/cmd/pacman-tools/shapes.go b/modules/pacman/cmd/pacman-tools/shapes.go new file mode 100644 index 00000000..3c364538 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/shapes.go @@ -0,0 +1,474 @@ +package main + +// The keeper's bus shapes (novox/hq ADR 0507, ADR 0508): the value of every key of the module's states and the +// body of every event its process emits, in one place, so the process that writes them and the verbs that read +// them agree by type. Each is a struct of named fields only: no map and no free-form value, so nothing but what +// is named here goes on the bus (ADR 0507 §1: package names, versions, unit and container names, file paths and +// exit statuses, never a process's command line or environment). A time in a value is the time of the thing it +// describes (a transaction, a step, an opening), never the time it was read: a key is written only when what it +// describes changed (ADR 0289 §1), and no field changes on a read alone (ADR 0289 §4). When a key last changed +// is the bus's to say (its revision), not a field's. +// +// A key is `` or `.`: a node name and an id are each one key token ([A-Za-z0-9_-]), so the +// dot only ever separates them. + +import ( + "fmt" + "time" +) + +// The module's states (manifest `state` and the keeper's `bus` `state`). +const ( + // StateInstalled holds each node's installed packages: key `` an InstalledValue, and key + // `.not-current` a NotCurrent while the node's packages are over the bus's cap (ADR 0507 §2). + StateInstalled = "installed" + // StateUpdates holds what a full upgrade would change: key `` an UpdatesValue (ADR 0507 §4). + StateUpdates = "updates" + // StateRuns holds every upgrade and rollback: key `.` a Run, kept (ADR 0508 §1). + StateRuns = "runs" + // StateRecords holds the package record taken before a run: key `.` a Record, deleted after + // the config value `record-days` (ADR 0508 §2). + StateRecords = "records" + // StateRounds holds the weekly rounds: key `` a Round, created once by the first node of the order, + // and key `.` that node's RoundNode (ADR 0508 §5, §6). + StateRounds = "rounds" + // StateAsks holds the acts asked of a node: key `.` an Ask, taken by that node's keeper alone + // (ADR 0508 §8). + StateAsks = "asks" +) + +// States are the module's states, in the manifest's order. +var States = []string{StateInstalled, StateUpdates, StateRuns, StateRecords, StateRounds, StateAsks} + +// The events the keeper emits (the keeper's `bus` `emits`, and the manifest's `emits`). +const ( + EventPackageChanged = "package.changed" // PackageChanged (ADR 0507 §3) + EventRecorded = "upgrade.recorded" // UpgradeRecorded (ADR 0508 §1, §2) + EventKernelHeld = "upgrade.kernel-held" // KernelHeld (ADR 0508 §3) + EventStarted = "upgrade.started" // UpgradeStarted + EventFinished = "upgrade.finished" // UpgradeFinished + EventFailed = "upgrade.failed" // UpgradeFailed + EventChecked = "upgrade.checked" // UpgradeChecked (ADR 0508 §4) + EventRollbackStarted = "rollback.started" // RollbackStarted (ADR 0508 §7) + EventRollbackFinished = "rollback.finished" // RollbackFinished +) + +// Events are the events the keeper emits, each with the type of its body. +var Events = []struct { + Name string + Body any +}{ + {EventPackageChanged, PackageChanged{}}, + {EventRecorded, UpgradeRecorded{}}, + {EventKernelHeld, KernelHeld{}}, + {EventStarted, UpgradeStarted{}}, + {EventFinished, UpgradeFinished{}}, + {EventFailed, UpgradeFailed{}}, + {EventChecked, UpgradeChecked{}}, + {EventRollbackStarted, RollbackStarted{}}, + {EventRollbackFinished, RollbackFinished{}}, +} + +// Key names. + +// NotCurrentKey is the key in `installed` that marks a node's packages as not current. +func NotCurrentKey(node string) string { return node + ".not-current" } + +// NodeKey is a key `.`: a run in `runs` or `records`, an ask in `asks`. +func NodeKey(node, id string) string { return node + "." + id } + +// RoundNodeKey is a node's key in a round: `.` in `rounds`. +func RoundNodeKey(round, node string) string { return round + "." + node } + +// RunID names a run by the moment it began, in UTC: `20261024T040012Z`. A run read from the package manager's +// log is named by the log's time of its start the same way. +func RunID(began time.Time) string { return began.UTC().Format("20060102T150405Z") } + +// RoundID names the round of the ISO week that holds the given moment, in its own time zone: `2026-W43`. The +// round opens on Saturday 04:00 local time, so the local date decides the week. +func RoundID(at time.Time) string { + year, week := at.ISOWeek() + return fmt.Sprintf("%04d-W%02d", year, week) +} + +// installed + +// InstalledValue is `installed.`: every installed package, sorted by name, and the time of the transaction +// that left the database so (ADR 0507 §2). Written compactly: one short entry per package. +type InstalledValue struct { + At time.Time `json:"at"` + Packages []InstalledPackage `json:"packages"` +} + +// InstalledPackage is one installed package: its version, whether it was installed explicitly (else as a +// dependency), and its repository, or RepoForeign when no repository holds it (ADR 0507 §7). +type InstalledPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Explicit bool `json:"explicit"` + Repo string `json:"repo"` +} + +// RepoForeign marks a package from outside the repositories. +const RepoForeign = "foreign" + +// NotCurrent is `installed..not-current`: the node's packages came to more than the bus's cap on a value, +// so `installed.` keeps the last value written. Written once, when it first happens, and deleted at the +// next successful write of `installed.`. +type NotCurrent struct { + Bytes int `json:"bytes"` + Since time.Time `json:"since"` +} + +// updates + +// UpdatesValue is `updates.`: every package a full upgrade would change, sorted by name, and whether the +// node reached its mirrors (ADR 0507 §4). A failed read writes Reachable false and keeps the last list, never an +// empty list in its place. +type UpdatesValue struct { + Reachable bool `json:"reachable"` + Packages []VersionChange `json:"packages"` +} + +// VersionChange is one package going from one version to another. +type VersionChange struct { + Name string `json:"name"` + From string `json:"from"` + To string `json:"to"` +} + +// PackageVersion is one package at one version. +type PackageVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// runs + +// The kinds of run. +const ( + RunUpgrade = "upgrade" + RunRollback = "rollback" +) + +// What started a run. +const ( + FromRound = "round" // its turn in the week's round + FromAsk = "ask" // an ask: pacman_upgrade, or pacman_rollback with apply + FromLog = "log" // read from the package manager's log at the keeper's first start (ADR 0508 §1) +) + +// The steps of a run, in the order they are written. A run that does not start ends at StepRefused; a +// transaction ends at StepFinished or StepFailed, and the after-upgrade check follows either. +const ( + StepRecorded = "recorded" // the record is written and read back (upgrade only) + StepCheckedBefore = "checked-before" // the DKMS gate ran; Held names what it held + StepRefused = "refused" // the run did not start; Why says why + StepStarted = "started" // the transaction's unit started + StepFinished = "finished" // the transaction exited 0 + StepFailed = "failed" // the transaction failed + StepCheckedAfter = "checked-after" // the after-upgrade check +) + +// Run is `runs..`: one upgrade or rollback of a node, written at each step and at no other time +// (ADR 0508 §1). Round names the round it ran in; Ask the ask that started it (the keeper deletes the ask once +// the run names it); Of, for a rollback, the run whose record it returns to. +type Run struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Ask string `json:"ask,omitempty"` + Of string `json:"of,omitempty"` + Steps []Step `json:"steps"` +} + +// Step is one step of a run with its time and outcome; only the fields of its kind are set. +type Step struct { + Step string `json:"step"` + At time.Time `json:"at"` + // StepRefused: why the run did not start. + Why string `json:"why,omitempty"` + // StepCheckedBefore: the kernels held from this run. + Held []HeldKernel `json:"held,omitempty"` + // StepStarted: the transaction's unit, the packages left out of it (held kernels and their headers), and + // for a rollback its plan. + Unit string `json:"unit,omitempty"` + Ignored []string `json:"ignored,omitempty"` + Plan []RollbackItem `json:"plan,omitempty"` + // StepFinished and StepFailed: the exit status, the last error line, and what the transaction changed, + // read from its lines in the package manager's log. + Exit *int `json:"exit,omitempty"` + Error string `json:"error,omitempty"` + Changes *Changes `json:"changes,omitempty"` + // StepCheckedAfter: the after-upgrade check. + Check *Check `json:"check,omitempty"` +} + +// HeldKernel is a kernel package left out of a run because an installed DKMS module's own bound excludes its +// new version (ADR 0508 §3): `Linux-Maximum` in the module's META, or `BUILD_EXCLUSIVE_KERNEL` in its dkms.conf. +type HeldKernel struct { + Package string `json:"package"` + Version string `json:"version"` + Module string `json:"module"` + Bound string `json:"bound"` +} + +// Changes is what a transaction changed, each list sorted by name. +type Changes struct { + Added []PackageVersion `json:"added"` + Removed []PackageVersion `json:"removed"` + Changed []VersionChange `json:"changed"` +} + +// Check is the after-upgrade check (ADR 0508 §4). Passed when the transaction succeeded, no unit newly failed +// and every DKMS module is built for every installed kernel; the rest is said and does not stop the round. +type Check struct { + Passed bool `json:"passed"` + // The .pacnew and .pacsave files the transaction left, by path. + Pacnew []string `json:"pacnew"` + Pacsave []string `json:"pacsave"` + // The distribution's news since the node's previous upgrade. + News []NewsTitle `json:"news"` + // Why the news could not be read, when it could not: no news is then not said to be none. + NewsError string `json:"news_error,omitempty"` + // The units and containers whose processes still map a file the upgrade replaced. + Mapped []Mapped `json:"mapped"` + // Why the mapped files could not be read, when they could not. + MappedError string `json:"mapped_error,omitempty"` + // The units failed now that were not failed in the record. + NewlyFailed []string `json:"newly_failed"` + // Whether a restart is needed: the running kernel is not the installed version of its package. + RestartNeeded bool `json:"restart_needed"` + RunningKernel string `json:"running_kernel"` + InstalledKernel string `json:"installed_kernel"` + // The DKMS modules not built for an installed kernel. + Unbuilt []DKMSBuild `json:"unbuilt"` + // Why the DKMS builds could not be read, when they could not; the check then does not pass. + DKMSError string `json:"dkms_error,omitempty"` +} + +// NewsTitle is one item of the distribution's news, without its text. +type NewsTitle struct { + Title string `json:"title"` + Link string `json:"link"` + Published string `json:"published"` +} + +// Mapped is a process still mapping a replaced file, named by its unit or its container and its executable's +// path; never its command line or environment. +type Mapped struct { + Unit string `json:"unit,omitempty"` + Container string `json:"container,omitempty"` + Executable string `json:"executable"` +} + +// DKMSBuild is one DKMS module at one version for one kernel. +type DKMSBuild struct { + Module string `json:"module"` + Version string `json:"version"` + Kernel string `json:"kernel"` +} + +// RollbackItem is one package a rollback brings back (ADR 0508 §7): from its version now to the record's, and +// where that version comes from. +type RollbackItem struct { + Name string `json:"name"` + From string `json:"from"` + To string `json:"to"` + Source string `json:"source"` + Location string `json:"location"` +} + +// Where a rollback takes a package's version from, in the order it is looked for. +const ( + SourceCache = "cache" // the node's package cache: a file path + SourceArchive = "archive" // the Arch Linux Archive's copy of that exact version: a URL + SourceRepository = "repository" // the archive's repository as it stood on the record's date: a URL +) + +// records + +// Record is `records..`: the node's packages taken before a run and read back (ADR 0508 §2): a copy of +// `installed.`, the running kernel (/proc/sys/kernel/osrelease), the units failed at that moment, and the +// installed DKMS modules. +type Record struct { + Node string `json:"node"` + Run string `json:"run"` + At time.Time `json:"at"` + Installed InstalledValue `json:"installed"` + Kernel string `json:"kernel"` + FailedUnits []string `json:"failed_units"` + DKMS []DKMSModule `json:"dkms"` +} + +// DKMSModule is one installed DKMS module and the kernels it is built for. +type DKMSModule struct { + Module string `json:"module"` + Version string `json:"version"` + Kernels []string `json:"kernels"` +} + +// rounds + +// Round is `rounds.`: the week's round, created once (a create, so one round per week) by the first node +// of the order when it opens it, with the order as it stood then (ADR 0508 §5). +type Round struct { + Round string `json:"round"` + Opened time.Time `json:"opened"` + OpenedBy string `json:"opened_by"` + Order []string `json:"order"` +} + +// RoundNode is `rounds..`: the node's place in the round. Run is the run it started in its turn, +// written by the node itself; Pass, the operator's pass of a node that stopped the round, written by the keeper +// of the node waiting on it (ADR 0508 §6). Both write through a compare-and-set. +type RoundNode struct { + Round string `json:"round"` + Node string `json:"node"` + Run string `json:"run,omitempty"` + Pass *Pass `json:"pass,omitempty"` +} + +// Pass is a pass of a node asked of the operator through the operator's channel (ADR 0259): the reason, the +// channel's ask, and its answer once given (Answer empty while it waits). +type Pass struct { + Reason string `json:"reason"` + Ask string `json:"ask"` + Asked time.Time `json:"asked"` + Answer string `json:"answer,omitempty"` + Answered *time.Time `json:"answered,omitempty"` +} + +// The operator's answers to a pass. +const ( + PassApproved = "approved" + PassDeclined = "declined" +) + +// asks + +// The acts an ask asks. +const ( + ActUpgrade = "upgrade" // pacman_upgrade + ActRollback = "rollback" // pacman_rollback with apply + ActPass = "pass" // pacman_round with pass +) + +// Ask is `asks..`: an act asked of a node, taken by that node's keeper alone, which answers in `runs` +// (an upgrade, a rollback) or in `rounds` (a pass) and then deletes the ask (ADR 0508 §8). A rollback names the +// run whose record it returns to and whether the packages the upgrade added are removed; a pass names the +// round, the node passed and the reason, and is asked of the node waiting on it. +type Ask struct { + ID string `json:"id"` + Node string `json:"node"` + Act string `json:"act"` + At time.Time `json:"at"` + Of string `json:"of,omitempty"` + RemoveAdded bool `json:"remove_added,omitempty"` + Round string `json:"round,omitempty"` + Pass string `json:"pass,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// events + +// PackageChanged is `package.changed`: a write of `installed.` that changed it (ADR 0507 §3). +type PackageChanged struct { + Node string `json:"node"` + At time.Time `json:"at"` + Added []PackageVersion `json:"added"` + Removed []PackageVersion `json:"removed"` + Changed []VersionChange `json:"changed"` +} + +// UpgradeRecorded is `upgrade.recorded`: the run's record is written and read back. +type UpgradeRecorded struct { + Node string `json:"node"` + Run string `json:"run"` + Packages int `json:"packages"` + Kernel string `json:"kernel"` +} + +// KernelHeld is `upgrade.kernel-held`: kernels left out of a run, each with the module and bound that hold it. +type KernelHeld struct { + Node string `json:"node"` + Run string `json:"run"` + Held []HeldKernel `json:"held"` +} + +// UpgradeStarted is `upgrade.started`: a run's transaction started. +type UpgradeStarted struct { + Node string `json:"node"` + Run string `json:"run"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Unit string `json:"unit"` + Ignored []string `json:"ignored"` +} + +// UpgradeFinished is `upgrade.finished`: a run's transaction exited 0, with what it changed. +type UpgradeFinished struct { + Node string `json:"node"` + Run string `json:"run"` + Changes Changes `json:"changes"` +} + +// UpgradeFailed is `upgrade.failed`: a run that did not start (Step StepRefused, with why in Error) or whose +// transaction failed (Step StepFailed, with its exit status and last error line). +type UpgradeFailed struct { + Node string `json:"node"` + Run string `json:"run"` + Step string `json:"step"` + Exit *int `json:"exit,omitempty"` + Error string `json:"error"` +} + +// UpgradeChecked is `upgrade.checked`: the after-upgrade check of a run, an upgrade's or a rollback's; the +// check whole is in the run. +type UpgradeChecked struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + Passed bool `json:"passed"` +} + +// RollbackStarted is `rollback.started`: a rollback's transaction started, returning to the record of run Of. +type RollbackStarted struct { + Node string `json:"node"` + Run string `json:"run"` + Of string `json:"of"` + Unit string `json:"unit"` +} + +// RollbackFinished is `rollback.finished`: a rollback's transaction ended, with its exit status and, when it +// failed, its last error line. +type RollbackFinished struct { + Node string `json:"node"` + Run string `json:"run"` + Of string `json:"of"` + Exit int `json:"exit"` + Error string `json:"error,omitempty"` +} + +// The keeper's config file + +// The keeper's environment, named by the manifest's process resource. +const ( + EnvBroker = "PACMAN_KEEPER_BROKER" // the keeper's bus account file + EnvConfig = "PACMAN_KEEPER_CONFIG" // the keeper's config file, a KeeperConfig +) + +// KeeperConfig is the keeper's config file: the module's config values the mesh merges into it (ADR 0348 §5). +// The process reads it at its start; the mesh restarts the process when it changes (the manifest's +// `restart-on`). +type KeeperConfig struct { + // UpgradeOrder is the nodes in the order they upgrade; the first opens each week's round. Empty: no round. + UpgradeOrder []string `json:"upgrade-order"` + // RecordDays is how long a record is kept, in days. + RecordDays int `json:"record-days"` + // RoundStartMinutes is how long a node whose turn came may wait before starting, in minutes, before the + // round is said to stand still. + RoundStartMinutes int `json:"round-start-minutes"` +} diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status b/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status new file mode 100644 index 00000000..d19fd59a --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-status-installed-and-added/status @@ -0,0 +1,4 @@ +openrazer-driver/3.12.4, 7.2.8-arch1-2, x86_64: installed +vboxhost/7.2.20_OSE, 7.2.8-arch1-2, x86_64: installed +xone/0.5.8, 7.2.8-arch1-2, x86_64: installed +xpad/0.4: added diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META new file mode 100644 index 00000000..8133bf1e --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/META @@ -0,0 +1,10 @@ +Meta: 1 +Name: zfs +Branch: 1.0 +Version: 2.4.4 +Release: 1 +Release-Tags: relext +License: CDDL +Author: OpenZFS +Linux-Maximum: 7.2 +Linux-Minimum: 4.18 diff --git a/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf new file mode 100644 index 00000000..50136d8a --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/dkms-zfs-linux-maximum-7.2/dkms.conf @@ -0,0 +1,56 @@ +PACKAGE_NAME="zfs" +PACKAGE_VERSION="2.4.4" +PACKAGE_CONFIG="/etc/sysconfig/zfs" +NO_WEAK_MODULES="yes" +PRE_BUILD="configure + --disable-dependency-tracking + --prefix=/usr + --with-config=kernel + --with-linux="${kernel_source_dir}" + --with-linux-obj="${kernel_source_dir}" + $( + [[ -n \"${ICP_ROOT}\" ]] && \ + { + echo --with-qat=\"${ICP_ROOT}\" + } + ) + $( + [[ -r ${PACKAGE_CONFIG} ]] \ + && source ${PACKAGE_CONFIG} \ + && shopt -q -s extglob \ + && \ + { + if [[ ${ZFS_DKMS_ENABLE_DEBUG,,} == @(y|yes) ]] + then + echo --enable-debug + fi + if [[ ${ZFS_DKMS_ENABLE_DEBUGINFO,,} == @(y|yes) ]] + then + echo --enable-debuginfo + fi + } + ) +" +POST_BUILD="scripts/dkms.postbuild + -n ${PACKAGE_NAME} + -v ${PACKAGE_VERSION} + -a ${arch} + -k ${kernelver} + -t ${dkms_tree} +" +AUTOINSTALL="yes" +MAKE[0]="make -C module/" +STRIP[0]="$( + [[ -r ${PACKAGE_CONFIG} ]] \ + && source ${PACKAGE_CONFIG} \ + && shopt -q -s extglob \ + && [[ ${ZFS_DKMS_DISABLE_STRIP,,} == @(y|yes) ]] \ + && echo -n no +)" +STRIP[1]="${STRIP[0]}" +BUILT_MODULE_NAME[0]="zfs" +BUILT_MODULE_LOCATION[0]="module/" +DEST_MODULE_LOCATION[0]="/extra" +BUILT_MODULE_NAME[1]="spl" +BUILT_MODULE_LOCATION[1]="module/" +DEST_MODULE_LOCATION[1]="/extra" diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log b/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log new file mode 100644 index 00000000..b35df8bd --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-log-three-full-upgrades/pacman.log @@ -0,0 +1,63 @@ +[2026-10-04T14:52:51+0200] [ALPM] installed logrotate (3.22.0-1) +[2026-10-04T14:52:51+0200] [ALPM] transaction completed +[2026-10-04T14:52:51+0200] [ALPM] running '30-systemd-daemon-reload-system.hook'... +[2026-10-04T14:54:22+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:06:13+0200] [PACMAN] Running 'pacman -Syu --noconfirm' +[2026-10-04T15:06:13+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:06:48+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:07:23+0200] [PACMAN] Running 'pacman -Rns --noconfirm lib32-libpcap' +[2026-10-04T15:07:23+0200] [ALPM] transaction started +[2026-10-04T15:07:24+0200] [ALPM] removed lib32-libpcap (1.10.6-1) +[2026-10-04T15:07:24+0200] [ALPM] removed lib32-libnl (3.12.0-1) +[2026-10-04T15:07:24+0200] [ALPM] transaction completed +[2026-10-04T15:07:24+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:07:24+0200] [PACMAN] Running 'pacman -Syu --noconfirm' +[2026-10-04T15:07:24+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:07:24+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:16:51+0200] [ALPM] running '60-mkinitcpio-remove.hook'... +[2026-10-04T15:16:52+0200] [ALPM] running '71-dkms-remove.hook'... +[2026-10-04T15:16:52+0200] [ALPM-SCRIPTLET] ==> dkms remove --no-depmod xone/0.5.8 -k 7.1.9-arch1-2 +[2026-10-04T15:16:52+0200] [ALPM-SCRIPTLET] ==> depmod 7.1.9-arch1-2 +[2026-10-04T15:16:55+0200] [ALPM] running 'ghc-unregister.hook'... +[2026-10-04T15:16:57+0200] [ALPM] running 'texinfo-remove.hook'... +[2026-10-04T15:16:58+0200] [ALPM] transaction started +[2026-10-04T15:16:58+0200] [ALPM] upgraded tzdata (2026c-1 -> 2026e-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded iana-etc (20260530-1 -> 20260617-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded glibc (2.44+r24+g16be1518495f-1 -> 2.44+r50+g1848099f063e-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded readline (8.3.003-1 -> 8.3.6-1) +[2026-10-04T15:16:58+0200] [ALPM] upgraded bash (5.3.15-1 -> 5.3.20-1) +[2026-10-04T15:22:46+0200] [ALPM] upgraded wireshark-qt (4.7.2-1 -> 4.7.3-1) +[2026-10-04T15:22:46+0200] [ALPM] upgraded xfsprogs (7.1.1-1 -> 7.2.0-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded xpad (5.8.0-1 -> 5.9.0-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded xterm (410-1 -> 411-1) +[2026-10-04T15:22:47+0200] [ALPM] upgraded zathura-pdf-mupdf (2026.07.18-1 -> 2026.07.18-4) +[2026-10-04T15:22:50+0200] [ALPM] transaction completed +[2026-10-04T15:22:52+0200] [ALPM] running '10-glibc-locale-gen.hook'... +[2026-10-04T15:22:52+0200] [ALPM-SCRIPTLET] Generating locales... +[2026-10-04T15:22:53+0200] [ALPM-SCRIPTLET] en_US.UTF-8... done +[2026-10-04T15:22:53+0200] [ALPM-SCRIPTLET] Generation complete. +[2026-10-04T15:22:53+0200] [ALPM] running '11-glibc-ldconfig.hook'... +[2026-10-04T15:24:49+0200] [ALPM] running 'texinfo-install.hook'... +[2026-10-04T15:24:51+0200] [ALPM] running 'update-desktop-database.hook'... +[2026-10-04T15:24:51+0200] [ALPM] running 'update-vlc-plugin-cache.hook'... +[2026-10-04T15:24:52+0200] [ALPM] running 'xorg-mkfontscale.hook'... +[2026-10-04T15:28:15+0200] [PACMAN] Running 'pacman -Syu' +[2026-10-04T15:28:15+0200] [PACMAN] synchronizing package lists +[2026-10-04T15:28:17+0200] [PACMAN] starting full system upgrade +[2026-10-04T15:30:26+0200] [ALPM] transaction started +[2026-10-04T15:30:26+0200] [ALPM] upgraded dunst (1.13.2-2 -> 1.13.2-3) +[2026-10-04T15:30:26+0200] [ALPM] transaction completed +[2026-10-04T15:30:26+0200] [ALPM] running '30-systemd-daemon-reload-user.hook'... +[2026-10-04T15:30:26+0200] [ALPM-SCRIPTLET] Failed to connect to system scope bus via local transport: Connection refused +[2026-10-04T15:30:26+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-04T15:30:26+0200] [ALPM] running 'i3status-rust.hook'... +[2026-10-04T16:00:43+0200] [PACMAN] Running 'pacman -R --noconfirm ntp' +[2026-10-04T16:00:43+0200] [ALPM] transaction started +[2026-10-04T16:00:45+0200] [ALPM] removed ntp (4.2.8.p18-6) +[2026-10-04T16:00:45+0200] [ALPM] transaction completed +[2026-10-04T16:00:45+0200] [ALPM] running '30-systemd-daemon-reload-system.hook'... +[2026-10-04T16:00:45+0200] [ALPM] running '35-systemd-update.hook'... +[2026-10-05T10:04:50+0200] [PACMAN] Running 'pacman -S --noconfirm --ask 4 lemurs' +[2026-10-05T10:04:51+0200] [ALPM] transaction started +[2026-10-05T10:04:51+0200] [ALPM] removed lemurs-git (0.4.0.r17.g7151336-2) +[2026-10-05T10:04:51+0200] [ALPM] installed lemurs (0.4.0-1) diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates new file mode 100644 index 00000000..40c952dc --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates @@ -0,0 +1,4 @@ +glibc 2.42-1 -> 2.42-2 +zsh 5.9-1 -> 5.9-2 PASSWORD=hunter2 +env FOO=bar -> /bin/sh +:: Synchronizing package databases... diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q new file mode 100644 index 00000000..a8c96383 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q @@ -0,0 +1,9 @@ +glibc 2.42-1 +sudo pacman -Syu --noconfirm +yay 12.0-1 +LD_PRELOAD=/tmp/hook.so 1.0-1 +zsh 5.9-1 +bash 5.2-1 HOME=/root +evil 1.0;rm-rf +PATH=/usr/bin:/bin +warning: could not get file information for etc/secret diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq new file mode 100644 index 00000000..815d6570 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq @@ -0,0 +1,3 @@ +yay +zsh +TOKEN=abc diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl new file mode 100644 index 00000000..269f745e --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl @@ -0,0 +1,3 @@ +core glibc 2.42-1 [installed] +extra zsh 5.9-1 [installed] +extra SECRET=x 1-1 diff --git a/modules/pacman/cmd/pacman-tools/transaction.go b/modules/pacman/cmd/pacman-tools/transaction.go new file mode 100644 index 00000000..df04b961 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/transaction.go @@ -0,0 +1,393 @@ +package main + +// An upgrade's transaction through an ask (novox/hq ADR 0508 §1, §8): a hand upgrade writes an ask in `asks` at +// `.`; that node's keeper alone takes it and runs the steps: the record, the DKMS gate, the +// transaction as a unit of the service manager started by the root helper (ADR 0510), its end, and the +// after-upgrade check, each written to the run and emitted. A keeper started again finds an unfinished run in +// `runs` and follows it to its end. + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// journalLines is how much of a failed transaction's journal is read for its last error. +const journalLines = 40 + +// thisNode is the node the tools bundle runs on, as the runtime names it; a test replaces it. +var thisNode = func() string { return os.Getenv("MESH_NODE") } + +// AskID names an ask by the moment it was asked and a few random digits, so two asks in one second differ. +func AskID(at time.Time) string { + b := make([]byte, 3) + _, _ = rand.Read(b) + return RunID(at) + "-" + hex.EncodeToString(b) +} + +// AskUpgrade writes an ask for a full upgrade of a node, for that node's keeper to take: what pacman_upgrade +// does (ADR 0508 §8). +func AskUpgrade(asks Bucket, node string, now time.Time) (Ask, error) { + if node == "" { + return Ask{}, errors.New("no node to ask: the runtime names none (MESH_NODE), and none was given") + } + a := Ask{ID: AskID(now), Node: node, Act: ActUpgrade, At: now.UTC()} + if _, err := asks.Create(NodeKey(node, a.ID), a); err != nil { + return Ask{}, fmt.Errorf("writing the ask: %w", err) + } + return a, nil +} + +// lastStep is a run's last step, or none. +func lastStep(r Run) string { + if len(r.Steps) == 0 { + return "" + } + return r.Steps[len(r.Steps)-1].Step +} + +// ended says whether a run has nothing more to do. +func ended(r Run) bool { + switch lastStep(r) { + case StepRefused, StepCheckedAfter: + return true + } + return false +} + +// ownRuns are the node's runs, oldest first, each with the revision of its key. +func (u *Upgrades) ownRuns() ([]openRun, error) { + keys, err := u.Runs.Keys() + if err != nil { + return nil, fmt.Errorf("listing runs: %w", err) + } + var out []openRun + for _, k := range keys { + if !strings.HasPrefix(k, u.Node+".") { + continue + } + e, err := u.Runs.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var r Run + if err := json.Unmarshal(e.Value, &r); err != nil { + return nil, fmt.Errorf("run %s: %w", k, err) + } + out = append(out, openRun{Run: r, rev: e.Revision}) + } + return out, nil +} + +// refuse ends a run that does not start, saying why. +func (u *Upgrades) refuse(r *openRun, why string) error { + if err := u.step(r, Step{Step: StepRefused, At: u.Machine.Now().UTC(), Why: why}); err != nil { + return err + } + return u.emit(EventFailed, UpgradeFailed{Node: r.Node, Run: r.Run.Run, Step: StepRefused, Error: why}) +} + +// StartUpgrade runs a full upgrade's steps up to its transaction's start: the record, the DKMS gate and the +// transaction's unit. The rest follows when the transaction ends (Follow). A run that cannot start is +// written as refused, saying why; only a run that could not be written at all is an error. +func (u *Upgrades) StartUpgrade(from, round, ask string) (Run, error) { + now := u.Machine.Now().UTC() + r := &openRun{Run: Run{Node: u.Node, Run: RunID(now), Kind: RunUpgrade, From: from, Round: round, Ask: ask}} + err := u.startUpgrade(r) + return r.Run, err +} + +func (u *Upgrades) startUpgrade(r *openRun) error { + runs, err := u.ownRuns() + if err != nil { + return err + } + for _, o := range runs { + if o.Run.Run == r.Run.Run { + return fmt.Errorf("run %s of %s is there already: one run a second", o.Run.Run, u.Node) + } + if !ended(o.Run) { + return u.refuse(r, fmt.Sprintf("run %s is still going (its last step is %s): one run at a time", o.Run.Run, lastStep(o.Run))) + } + } + if u.Machine.locked() { + return u.refuse(r, "another pacman holds "+DBLock+": a transaction is running, or one was killed and left its lock") + } + rec, why := u.TakeRecord(r.Run.Run) + if why != "" { + return u.refuse(r, why) + } + if err := u.step(r, Step{Step: StepRecorded, At: u.Machine.Now().UTC()}); err != nil { + return err + } + var errs []error + if err := u.emit(EventRecorded, UpgradeRecorded{Node: u.Node, Run: r.Run.Run, Packages: len(rec.Installed.Packages), Kernel: rec.Kernel}); err != nil { + errs = append(errs, err) + } + held, ignore, why := u.Gate() + if why != "" { + return errors.Join(append(errs, u.refuse(r, why))...) + } + if err := u.step(r, Step{Step: StepCheckedBefore, At: u.Machine.Now().UTC(), Held: held}); err != nil { + return errors.Join(append(errs, err)...) + } + if len(held) > 0 { + if err := u.emit(EventKernelHeld, KernelHeld{Node: u.Node, Run: r.Run.Run, Held: held}); err != nil { + errs = append(errs, err) + } + } + return errors.Join(append(errs, u.start(r, ignore))...) +} + +// start starts the run's transaction through the root helper and writes it started, or failed when it could +// not start, followed by the after-upgrade check. +func (u *Upgrades) start(r *openRun, ignore []string) error { + at := u.Machine.Now().UTC() + unit := TransactionUnit(r.Kind, r.Run.Run) + if err := u.Root.Transaction(r.Kind, r.Run.Run, ignore); err != nil { + return u.end(r, Step{Step: StepFailed, At: u.Machine.Now().UTC(), Error: "the transaction could not be started: " + err.Error()}) + } + if err := u.step(r, Step{Step: StepStarted, At: at, Unit: unit, Ignored: ignore}); err != nil { + return err + } + if r.Kind == RunRollback { + return u.emit(EventRollbackStarted, RollbackStarted{Node: u.Node, Run: r.Run.Run, Of: r.Of, Unit: unit}) + } + return u.emit(EventStarted, UpgradeStarted{Node: u.Node, Run: r.Run.Run, From: r.From, Round: r.Round, Unit: unit, Ignored: ignore}) +} + +// end writes how the transaction ended, emits it, and runs the after-upgrade check. +func (u *Upgrades) end(r *openRun, s Step) error { + if err := u.step(r, s); err != nil { + return err + } + var errs []error + exit := 0 + if s.Exit != nil { + exit = *s.Exit + } + switch { + case r.Kind == RunRollback: + errs = append(errs, u.emit(EventRollbackFinished, RollbackFinished{Node: u.Node, Run: r.Run.Run, Of: r.Of, Exit: exit, Error: s.Error})) + case s.Step == StepFinished: + changes := Changes{} + if s.Changes != nil { + changes = *s.Changes + } + errs = append(errs, u.emit(EventFinished, UpgradeFinished{Node: u.Node, Run: r.Run.Run, Changes: changes})) + default: + errs = append(errs, u.emit(EventFailed, UpgradeFailed{Node: u.Node, Run: r.Run.Run, Step: StepFailed, Exit: s.Exit, Error: s.Error})) + } + return errors.Join(append(errs, u.checkAfter(r))...) +} + +// checkAfter writes the after-upgrade check (ADR 0508 §4) and emits it. +func (u *Upgrades) checkAfter(r *openRun) error { + var rec *Record + key := NodeKey(u.Node, r.Run.Run) + if r.Kind == RunRollback { + key = NodeKey(u.Node, r.Of) + } + if e, err := u.Records.Get(key); err == nil && e != nil { + var got Record + if json.Unmarshal(e.Value, &got) == nil { + rec = &got + } + } + check := u.Check + if check == nil { + check = u.AfterCheck + } + c := check(r.Run, rec) + if err := u.step(r, Step{Step: StepCheckedAfter, At: u.Machine.Now().UTC(), Check: &c}); err != nil { + return err + } + return u.emit(EventChecked, UpgradeChecked{Node: u.Node, Run: r.Run.Run, Kind: r.Kind, Passed: c.Passed}) +} + +// transactionSucceeded says whether a run's transaction finished. +func transactionSucceeded(r Run) bool { + for _, s := range r.Steps { + if s.Step == StepFinished { + return true + } + } + return false +} + +// unitEnd reads a transaction unit's state: whether it ended, and its exit status. A transient unit that ended +// well is let go by the service manager (not found); one that failed stays, failed. +func (u *Upgrades) unitEnd(unit string) (done bool, exit int, err error) { + p, err := u.Machine.unitProps(unit, "LoadState", "ActiveState", "Result", "ExecMainStatus") + if err != nil { + return false, 0, err + } + switch { + case p["LoadState"] == "not-found": + return true, 0, nil + case p["ActiveState"] == "active" || p["ActiveState"] == "activating" || p["ActiveState"] == "deactivating" || p["ActiveState"] == "reloading": + return false, 0, nil + } + exit, _ = strconv.Atoi(p["ExecMainStatus"]) + if exit == 0 && p["Result"] != "success" { + exit = 1 + } + return true, exit, nil +} + +// transactionBlock is the run's transaction as pacman's log tells it: the first command of its kind pacman +// logged since the transaction was started; nil when the log holds none. +func (u *Upgrades) transactionBlock(kind string, since time.Time) *logBlock { + text, err := u.Machine.ReadFile(PacmanLog) + if err != nil { + return nil + } + since = since.Truncate(time.Second) + for _, b := range logBlocks(string(text)) { + if b.Start.Before(since) { + continue + } + if kind == RunUpgrade && !b.Full || kind == RunRollback && !strings.Contains(b.Command, " -U") { + continue + } + return &b + } + return nil +} + +// transactionChanges is what the run's transaction changed, read from its lines in pacman's log. +func (u *Upgrades) transactionChanges(kind string, since time.Time) *Changes { + if b := u.transactionBlock(kind, since); b != nil { + c := b.Changes + return &c + } + return nil +} + +// follow writes the end of a run whose transaction has ended, and its check; a transaction still running is +// left to the next call. +func (u *Upgrades) follow(r *openRun) error { + started := r.Steps[len(r.Steps)-1] + done, exit, err := u.unitEnd(started.Unit) + if err != nil { + return fmt.Errorf("reading %s: %w", started.Unit, err) + } + if !done { + return nil + } + s := Step{Step: StepFinished, At: u.Machine.Now().UTC(), Exit: &exit, Changes: u.transactionChanges(r.Kind, started.At)} + if exit != 0 { + s.Step = StepFailed + lines, err := u.Root.Journal(started.Unit, journalLines) + if err != nil { + s.Error = "(its journal could not be read: " + err.Error() + ")" + } else { + s.Error = lastError(lines) + } + } + return u.end(r, s) +} + +// Follow carries every unfinished run of the node on as far as it can go now: a transaction that ended is +// written finished or failed and checked; a run stopped between its steps by a keeper that went away is taken +// up where it stopped. The keeper calls it at its start and whenever pacman's database lock is released. +func (u *Upgrades) Follow() error { + runs, err := u.ownRuns() + if err != nil { + return err + } + var errs []error + for i := range runs { + r := &runs[i] + if ended(r.Run) || r.From == FromLog { + continue + } + switch lastStep(r.Run) { + case StepStarted: + errs = append(errs, u.follow(r)) + case StepFinished, StepFailed: + errs = append(errs, u.checkAfter(r)) + case StepRecorded, StepCheckedBefore, "": + errs = append(errs, u.resumeBeforeStart(r)) + } + } + return errors.Join(errs...) +} + +// resumeBeforeStart takes up a run whose keeper went away before it wrote the transaction started: when the +// run's unit is there, or pacman's log holds an upgrade since the run's last step, the transaction did start +// and is followed; otherwise nothing started, and the run is refused, so said. +func (u *Upgrades) resumeBeforeStart(r *openRun) error { + unit := TransactionUnit(r.Kind, r.Run.Run) + since := time.Time{} + if n := len(r.Steps); n > 0 { + since = r.Steps[n-1].At + } + p, err := u.Machine.unitProps(unit, "LoadState") + if err != nil { + return fmt.Errorf("reading %s: %w", unit, err) + } + if p["LoadState"] == "not-found" && u.transactionChanges(r.Kind, since) == nil { + return u.refuse(r, "the keeper stopped before it started the transaction, and nothing was started") + } + if err := u.step(r, Step{Step: StepStarted, At: since, Unit: unit}); err != nil { + return err + } + return u.follow(r) +} + +// TakeAsk takes one change of `asks`: an ask for this node, not taken yet, is carried out and then deleted. An +// ask for another node is that node's keeper's. An ask a run already names was taken by a keeper that went +// away before deleting it, and is deleted. The keeper watches `asks` at `.*` and hands each change here. +func (u *Upgrades) TakeAsk(c stdio.StateChange) error { + if c.Op != "put" || !strings.HasPrefix(c.Key, u.Node+".") { + return nil + } + var a Ask + if err := json.Unmarshal(c.Value, &a); err != nil { + return fmt.Errorf("ask %s is not readable: %w", c.Key, err) + } + if a.Node != u.Node { + return fmt.Errorf("ask %s names node %q, not this one", c.Key, a.Node) + } + runs, err := u.ownRuns() + if err != nil { + return err + } + for _, r := range runs { + if r.Ask == a.ID { + return u.Asks.Delete(c.Key) + } + } + switch a.Act { + case ActUpgrade: + r, err := u.StartUpgrade(FromAsk, "", a.ID) + if len(r.Steps) == 0 { + return err // no run names the ask yet: it stays, and is taken again + } + if derr := u.Asks.Delete(c.Key); derr != nil { + err = errors.Join(err, fmt.Errorf("deleting the ask %s: %w", c.Key, derr)) + } + return err + case ActRollback: + return u.takeRollback(c.Key, a) + case ActPass: + if err := u.takePass(a); err != nil { + return err + } + return u.Asks.Delete(c.Key) + default: + return fmt.Errorf("ask %s asks %q, which this keeper does not carry out yet", c.Key, a.Act) + } +} diff --git a/modules/pacman/cmd/pacman-tools/transaction_test.go b/modules/pacman/cmd/pacman-tools/transaction_test.go new file mode 100644 index 00000000..7902d29e --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/transaction_test.go @@ -0,0 +1,427 @@ +package main + +// An upgrade through an ask, and a keeper started again following its run (novox/hq ADR 0508 §1, §8; +// acceptance 1841/AC1 and 1841/AC9). + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// fakeRoot is the root helper: it records the transactions it was asked to start. +type fakeRoot struct { + started []string + fail error + journal []string + mapped []Mapped + mapsErr error +} + +func (f *fakeRoot) Transaction(kind, run string, ignore []string) error { + if f.fail != nil { + return f.fail + } + f.started = append(f.started, kind+" "+run+" ignore="+strings.Join(ignore, ",")) + return nil +} + +func (f *fakeRoot) Rollback(run string, plan []RollbackItem) error { + if f.fail != nil { + return f.fail + } + var names []string + for _, i := range plan { + names = append(names, i.Name+":"+i.To+":"+i.Source) + } + f.started = append(f.started, RunRollback+" "+run+" plan="+strings.Join(names, ",")) + return nil +} + +func (f *fakeRoot) Journal(unit string, n int) ([]string, error) { return f.journal, nil } + +func (f *fakeRoot) Maps() ([]Mapped, error) { return f.mapped, f.mapsErr } + +// keeperBed is one node's keeper over states in memory and a machine of fakes: a unit's state, the lock and +// pacman's log are set by the test as the transaction goes. +type keeperBed struct { + u *Upgrades + root *fakeRoot + events *eventLog + installed, updates, runs, records, asks *memBucket + rounds *memBucket + now time.Time + unitState string // systemctl show's answer for the run's unit + locked bool + log string +} + +const upgradeLog = "[2026-10-04T12:00:01+0000] [PACMAN] Running 'pacman -Syu --noconfirm --ignore linux,linux-headers'\n" + + "[2026-10-04T12:00:02+0000] [PACMAN] starting full system upgrade\n" + + "[2026-10-04T12:00:09+0000] [ALPM] transaction started\n" + + "[2026-10-04T12:00:10+0000] [ALPM] upgraded bash (5.3.15-1 -> 5.3.20-1)\n" + + "[2026-10-04T12:00:10+0000] [ALPM] installed made-up-library (1.0-1)\n" + + "[2026-10-04T12:00:11+0000] [ALPM] transaction completed\n" + +func newKeeperBed(t *testing.T) *keeperBed { + k := &keeperBed{root: &fakeRoot{}, events: &eventLog{}, installed: newBucket(), updates: newBucket(), runs: newBucket(), + records: newBucket(), asks: newBucket(), rounds: newBucket(), now: time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC), unitState: "LoadState=not-found\n"} + saved := fetch + fetch = func(string) ([]byte, error) { return nil, errors.New("offline") } + t.Cleanup(func() { fetch = saved }) + if _, err := k.installed.Put("node-a", nodeAPackages); err != nil { + t.Fatal(err) + } + if _, err := k.updates.Put("node-a", UpdatesValue{Reachable: true, Packages: []VersionChange{ + {Name: "bash", From: "5.3.15-1", To: "5.3.20-1"}, + {Name: "linux", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"}, + {Name: "linux-headers", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"}, + }}); err != nil { + t.Fatal(err) + } + sources := capturedZFSSource(t) + files := func(paths ...string) Ran { return Ran{Stdout: strings.Join(paths, "\n") + "\n"} } + m := machine(fake(func(c call) Ran { + line := c.String() + switch { + case line == "systemctl list-units --state=failed --plain --no-legend --no-pager": + return Ran{} + case line == "dkms status": + return Ran{Stdout: zfsStatus} + case line == "pacman -Qq": + return Ran{Stdout: "bash\nlinux\n"} + case line == "pacman -Qlq -- linux": + return files("/usr/lib/modules/7.2.8-arch1-2/vmlinuz") + case line == "pacman -Qlq -- linux-headers": + return files("/usr/lib/modules/7.2.8-arch1-2/build/Makefile") + case strings.HasPrefix(line, "systemctl show "+unitPrefix): + return Ran{Stdout: k.unitState} + } + return Ran{Status: 99, Stderr: "unexpected command: " + line} + }, nil), 1000) + m.Now = func() time.Time { return k.now } + m.ReadFile = func(path string) ([]byte, error) { + switch { + case path == OSRelease: + return []byte("7.2.8-arch1-2\n"), nil + case path == DBLock && k.locked: + return []byte{}, nil + case path == PacmanLog: + return []byte(k.log), nil + } + if s, ok := sources[path]; ok { + return []byte(s), nil + } + return nil, errNoFile + } + k.u = k.keeper(m) + return k +} + +// keeper is a keeper for node-a over the bed's states: a second call is the keeper started again. +func (k *keeperBed) keeper(m *Machine) *Upgrades { + return &Upgrades{Node: "node-a", Machine: m, Root: k.root, Events: k.events, Installed: k.installed, Updates: k.updates, + Runs: k.runs, Records: k.records, Rounds: k.rounds, Asks: k.asks} +} + +// ask writes an ask for node and hands its change to the keeper, as the watch of `asks` would. +func (k *keeperBed) ask(t *testing.T, node, act string) (string, error) { + t.Helper() + a := Ask{ID: RunID(k.now) + "-abc123", Node: node, Act: act, At: k.now} + key := NodeKey(node, a.ID) + rev, err := k.asks.Create(key, a) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(a) + return key, k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: key, Op: "put", Value: raw, Revision: rev}) +} + +func (k *keeperBed) run(t *testing.T) Run { + t.Helper() + keys, _ := k.runs.Keys() + if len(keys) != 1 { + t.Fatalf("runs: %v", keys) + } + var r Run + k.runs.value(keys[0], &r) + return r +} + +func steps(r Run) string { + var s []string + for _, st := range r.Steps { + s = append(s, st.Step) + } + return strings.Join(s, " ") +} + +func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) { + k := newKeeperBed(t) + key, err := k.ask(t, "node-a", ActUpgrade) + if err != nil { + t.Fatal(err) + } + r := k.run(t) + if r.Run != "20261004T120000Z" || r.From != FromAsk || r.Ask != "20261004T120000Z-abc123" || r.Kind != RunUpgrade { + t.Errorf("the run: %+v", r) + } + if steps(r) != "recorded checked-before started" { + t.Fatalf("steps before the transaction ends: %s", steps(r)) + } + if h := r.Steps[1].Held; len(h) != 1 || h[0].Package != "linux" { + t.Errorf("the gate's step names the held kernel: %+v", h) + } + if s := r.Steps[2]; s.Unit != "mesh-pacman-upgrade-20261004t120000z" || strings.Join(s.Ignored, ",") != "linux,linux-headers" { + t.Errorf("started: %+v", s) + } + if strings.Join(k.root.started, ";") != "upgrade 20261004T120000Z ignore=linux,linux-headers" { + t.Errorf("the root helper was asked: %v", k.root.started) + } + if _, there := k.asks.values[key]; there { + t.Error("the ask is deleted once the run names it") + } + var rec Record + if !k.records.value("node-a.20261004T120000Z", &rec) { + t.Error("no record was taken") + } + + // The transaction ends: the lock is released and the keeper follows the run. + k.unitState, k.log, k.now = "LoadState=not-found\n", upgradeLog, k.now.Add(15*time.Second) + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + r = k.run(t) + if steps(r) != "recorded checked-before started finished checked-after" { + t.Fatalf("steps: %s", steps(r)) + } + end := r.Steps[3] + if end.Exit == nil || *end.Exit != 0 || end.Changes == nil || len(end.Changes.Changed) != 1 || end.Changes.Changed[0].Name != "bash" || + len(end.Changes.Added) != 1 { + t.Errorf("finished: %+v %+v", end, end.Changes) + } + if c := r.Steps[4].Check; c == nil || !c.Passed { + t.Errorf("the check: %+v", c) + } + if got := strings.Join(k.events.names(), " "); got != "upgrade.recorded upgrade.kernel-held upgrade.started upgrade.finished upgrade.checked" { + t.Errorf("events: %s", got) + } + if w := k.runs.writesTo("node-a."); len(w) != 5 || w[0] != "create node-a.20261004T120000Z" { + t.Errorf("each step written once: %v", w) + } + if err := k.u.Follow(); err != nil || len(k.runs.writesTo("node-a.")) != 5 { + t.Errorf("an ended run is not written again: %v", err) + } +} + +func TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing.T) { + k := newKeeperBed(t) + if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { + t.Fatal(err) + } + k.unitState = "LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\n" + k.root.journal = []string{"checking for file conflicts...", "error: failed to commit transaction (conflicting files)", "Errors occurred, no packages were upgraded."} + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + r := k.run(t) + end := r.Steps[3] + if steps(r) != "recorded checked-before started failed checked-after" || *end.Exit != 1 || + end.Error != "error: failed to commit transaction (conflicting files)" || r.Steps[4].Check.Passed { + t.Errorf("%s %+v %+v", steps(r), end, r.Steps[4].Check) + } + if got := k.events.names(); got[len(got)-2] != EventFailed || got[len(got)-1] != EventChecked { + t.Errorf("events: %v", got) + } +} + +func TestATransactionThatCannotStartFailsAndIsChecked(t *testing.T) { + k := newKeeperBed(t) + k.root.fail = errors.New("the root helper's socket is not there") + if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { + t.Fatal(err) + } + r := k.run(t) + if steps(r) != "recorded checked-before failed checked-after" || !strings.Contains(r.Steps[2].Error, "socket is not there") { + t.Errorf("%s %+v", steps(r), r.Steps[2]) + } +} + +func TestAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) { + k := newKeeperBed(t) + key, err := k.ask(t, "node-b", ActUpgrade) + if err != nil { + t.Fatal(err) + } + if keys, _ := k.runs.Keys(); len(keys) != 0 || len(k.root.started) != 0 { + t.Errorf("node-a's keeper took node-b's ask: %v %v", keys, k.root.started) + } + if _, there := k.asks.values[key]; !there { + t.Error("another node's ask is left for its keeper") + } + if err := k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: "node-a.x", Op: "delete"}); err != nil { + t.Errorf("a delete is no ask: %v", err) + } +} + +func TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) { + k := newKeeperBed(t) + done := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, Ask: RunID(k.now) + "-abc123", + Steps: []Step{{Step: StepRefused, At: k.now, Why: "x"}}} + if _, err := k.runs.Create(NodeKey("node-a", done.Run), done); err != nil { + t.Fatal(err) + } + key, err := k.ask(t, "node-a", ActUpgrade) + if err != nil { + t.Fatal(err) + } + if keys, _ := k.runs.Keys(); len(keys) != 1 || len(k.root.started) != 0 { + t.Errorf("taken twice: %v", keys) + } + if _, there := k.asks.values[key]; there { + t.Error("the ask a run names is deleted") + } +} + +func TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T) { + k := newKeeperBed(t) + k.locked = true + if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { + t.Fatal(err) + } + r := k.run(t) + if steps(r) != "refused" || !strings.Contains(r.Steps[0].Why, DBLock) || len(k.root.started) != 0 { + t.Errorf("locked: %s %+v", steps(r), r.Steps) + } + if e := k.events.sent; len(e) != 1 || e[0].Name != EventFailed || e[0].Body.(UpgradeFailed).Step != StepRefused { + t.Errorf("a refused run is said: %+v", e) + } + + k = newKeeperBed(t) + going := Run{Node: "node-a", Run: "20261004T110000Z", Kind: RunUpgrade, From: FromRound, Steps: []Step{{Step: StepStarted, At: k.now}}} + if _, err := k.runs.Create(NodeKey("node-a", going.Run), going); err != nil { + t.Fatal(err) + } + if _, err := k.u.StartUpgrade(FromAsk, "", "a1"); err != nil { + t.Fatal(err) + } + var r2 Run + k.runs.value("node-a.20261004T120000Z", &r2) + if steps(r2) != "refused" || !strings.Contains(r2.Steps[0].Why, "20261004T110000Z is still going") { + t.Errorf("another run going: %+v", r2) + } +} + +func TestAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) { + k := newKeeperBed(t) + if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { + t.Fatal(err) + } + // The keeper goes away while the transaction runs; a new one starts and finds it still running. + again := k.keeper(k.u.Machine) + k.unitState = "LoadState=loaded\nActiveState=active\nResult=success\nExecMainStatus=0\n" + before := len(k.runs.writes) + if err := again.Follow(); err != nil || len(k.runs.writes) != before { + t.Fatalf("a running transaction is left running: %v, %d writes", err, len(k.runs.writes)-before) + } + k.unitState, k.log = "LoadState=not-found\n", upgradeLog + if err := again.Follow(); err != nil { + t.Fatal(err) + } + if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" { + t.Errorf("followed to its end: %s", steps(r)) + } +} + +func TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) { + // Stopped after the gate, before the transaction started: nothing started, so the run is refused. + k := newKeeperBed(t) + stopped := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, + Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}}} + if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil { + t.Fatal(err) + } + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + if r := k.run(t); steps(r) != "recorded checked-before refused" || !strings.Contains(r.Steps[2].Why, "nothing was started") { + t.Errorf("%s %+v", steps(r), r.Steps) + } + + // Stopped after the transaction started and before it was written: its log says it ran. + k = newKeeperBed(t) + if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil { + t.Fatal(err) + } + k.log = upgradeLog + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" || r.Steps[2].Unit != "mesh-pacman-upgrade-20261004t115900z" { + t.Errorf("%s %+v", steps(r), r.Steps) + } + + // Stopped after the transaction ended and before it was checked. + k = newKeeperBed(t) + zero := 0 + unchecked := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, + Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}, {Step: StepStarted, At: k.now}, {Step: StepFinished, At: k.now, Exit: &zero}}} + if _, err := k.runs.Create(NodeKey("node-a", unchecked.Run), unchecked); err != nil { + t.Fatal(err) + } + if err := k.u.Follow(); err != nil { + t.Fatal(err) + } + if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" { + t.Errorf("%s", steps(r)) + } +} + +func TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) { + asks := newBucket() + savedState, savedNode, savedFetch := upgradeState, thisNode, fetch + upgradeState = func(name string) Bucket { + if name != StateAsks { + t.Fatalf("wrote %s", name) + } + return asks + } + thisNode = func() string { return "node-a" } + fetch = func(string) ([]byte, error) { return nil, errors.New("offline") } + defer func() { upgradeState, thisNode, fetch = savedState, savedNode, savedFetch }() + var calls []call + m := machine(fake(func(c call) Ran { return Ran{} }, &calls), 1000) + for _, tool := range tools(m) { + if tool.Name != "pacman_upgrade" { + continue + } + out, err := tool.Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + a := out.(map[string]any)["asked"].(Ask) + if a.Node != "node-a" || a.Act != ActUpgrade || !strings.HasPrefix(a.ID, "20261004T120000Z-") { + t.Errorf("ask: %+v", a) + } + if keys, _ := asks.Keys(); len(keys) != 1 || keys[0] != NodeKey("node-a", a.ID) { + t.Errorf("asks: %v", keys) + } + for _, c := range calls { + if c.name == "sudo" || c.name == "systemd-run" { + t.Errorf("the tool acted itself: %s", c) + } + } + if _, err := tool.Run(map[string]any{"node": "node-b"}); err != nil { + t.Fatal(err) + } + if keys, _ := asks.Keys(); len(keys) != 2 || !strings.HasPrefix(keys[1], "node-b.") { + t.Errorf("another node's ask: %v", keys) + } + return + } + t.Fatal("no pacman_upgrade") +} diff --git a/modules/pacman/cmd/pacman-tools/updates_test.go b/modules/pacman/cmd/pacman-tools/updates_test.go new file mode 100644 index 00000000..6dc0e220 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/updates_test.go @@ -0,0 +1,147 @@ +package main + +// What an upgrade would change (novox/hq ADR 0507 §4; acceptance 1840/AC4): read at the keeper's start, after a +// transaction, after the mirror list is rewritten and a day after the last read; written only when the list or +// reachable changed; a failed read says reachable false and keeps the last list. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +func updatesOf(t *testing.T, s *fakeStore) UpdatesValue { + t.Helper() + e, _ := s.Get("node-a") + if e == nil { + t.Fatal("no updates.node-a") + } + var v UpdatesValue + if err := json.Unmarshal(e.Value, &v); err != nil { + t.Fatal(err) + } + return v +} + +func (d *pacmanDB) checkupdatesRuns() int { + d.mu.Lock() + defer d.mu.Unlock() + n := 0 + for _, c := range d.commands { + if c == "checkupdates" { + n++ + } + } + return n +} + +func TestUpdatesAreWrittenOnlyWhenTheListOrReachableChanged(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Stdout: "zsh 5.9-1 -> 5.9-2\nlinux 6.17.1-1 -> 6.17.2-1\n"}} + db.set("glibc 2.42-1\n", "") + k, states, _, _, _ := newTestKeeper(db) + up := states.store(StateUpdates) + k.refreshUpdates() + if w := up.Writes(); strings.Join(w, ",") != "put node-a" { + t.Fatalf("writes: %v", w) + } + if v := updatesOf(t, up); !v.Reachable || fmt.Sprint(v.Packages) != "[{linux 6.17.1-1 6.17.2-1} {zsh 5.9-1 5.9-2}]" { + t.Fatalf("%+v", v) + } + // the same answer: nothing written. + k.refreshUpdates() + if w := up.Writes(); len(w) != 0 { + t.Fatalf("the same list was written again: %v", w) + } + // the mirrors cannot be reached: reachable false, the last list kept. + db.setUpdates(Ran{Status: 1, Stderr: "==> ERROR: Cannot fetch updates\n"}) + k.refreshUpdates() + if w := up.Writes(); len(w) != 1 { + t.Fatalf("writes: %v", w) + } + if v := updatesOf(t, up); v.Reachable || len(v.Packages) != 2 { + t.Fatalf("a failed read keeps the list and says reachable false: %+v", v) + } + k.refreshUpdates() + if w := up.Writes(); len(w) != 0 { + t.Fatalf("still unreachable, and written again: %v", w) + } + // reached again with nothing to do (status 2): an empty list, reachable. + db.setUpdates(Ran{Status: 2}) + k.refreshUpdates() + if v := updatesOf(t, up); !v.Reachable || v.Packages == nil || len(v.Packages) != 0 { + t.Fatalf("%+v", v) + } +} + +func TestAFirstReadThatFailsSaysUnreachableNeverAnEmptyListAsAnAnswer(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 1, Stderr: "==> ERROR: Cannot fetch updates\n"}} + k, states, _, _, _ := newTestKeeper(db) + k.refreshUpdates() + if v := updatesOf(t, states.store(StateUpdates)); v.Reachable { + t.Fatalf("%+v", v) + } + raw, _ := states.store(StateUpdates).Get("node-a") + if string(raw.Value) != `{"reachable":false,"packages":[]}` { + t.Errorf("%s", raw.Value) + } +} + +// Read at the start, after a transaction, after the mirror list is rewritten, and a day after the last read +// with nothing else asking; never in between. +func TestUpdatesAreReadAtStartAfterATransactionAfterTheMirrorsAndDaily(t *testing.T) { + db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} + db.set("glibc 2.42-1\n", "") + k, states, _, clock, w := newTestKeeper(db) + var mu sync.Mutex + var seen []string + k.Hooks.AfterTransaction = []func(*Keeper, Transaction) error{func(*Keeper, Transaction) error { + mu.Lock() + seen = append(seen, "transaction") + mu.Unlock() + return nil + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- k.Run(ctx) }() + reads := func(want int) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if db.checkupdatesRuns() == want { + return + } + } + t.Fatalf("checkupdates ran %d times, want %d", db.checkupdatesRuns(), want) + } + reads(1) + db.setUpdates(Ran{Stdout: "zsh 5.9-1 -> 5.9-2\n"}) + clock.Advance(time.Hour) + w.c <- MirrorsRewritten + reads(2) + db.setUpdates(Ran{Status: 2}) + w.c <- LockReleased + reads(3) + waitFor(t, &mu, &seen, "transaction") + clock.Advance(UpdatesEvery - time.Minute) + time.Sleep(50 * time.Millisecond) + if n := db.checkupdatesRuns(); n != 3 { + t.Fatalf("read before a day passed: %d", n) + } + db.setUpdates(Ran{Stdout: "glibc 2.42-1 -> 2.42-2\n"}) + clock.Advance(time.Minute) + reads(4) + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if e, _ := states.store(StateUpdates).Get("node-a"); e != nil && strings.Contains(string(e.Value), "glibc") { + break + } + } + if w := states.store(StateUpdates).Writes(); len(w) != 4 { + t.Errorf("each read that changed the list wrote it, and only those: %v", w) + } + cancel() + <-done +} diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go index 90180130..f73f1095 100644 --- a/modules/pacman/cmd/pacman-tools/upgrade_health.go +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -1,18 +1,30 @@ package main +// The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's +// process reads this node's latest run, the week's round and any held kernel from the module's states, never a +// unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It +// is unhealthy, naming why, when: +// +// - `installed.` is marked not current (ADR 0507 §2); +// - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5); +// - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes; +// - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock; +// - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3); +// - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not +// started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6). +// +// A state that cannot be read is unhealthy: failure is loud. + import ( + "encoding/json" + "errors" "fmt" + "os" + "sort" "strings" "time" ) -// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node -// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager -// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of -// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, -// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an -// upgrade passes. - // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. type Health struct { Healthy bool `json:"healthy"` @@ -20,66 +32,190 @@ type Health struct { Found []string `json:"found,omitempty"` } -// WeeklyUnit is the service the weekly timer starts. -const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" - -// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on -// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit -// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run -// can start; it is said, not killed. +// RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on +// purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one +// (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said, +// not killed. const RunLimit = 2 * time.Hour -// systemdTime is how systemctl shows a timestamp, in the machine's local time. -const systemdTime = "Mon 2006-01-02 15:04:05 MST" +// healthTime is how the health writes a moment: the machine's local time. +const healthTime = "2006-01-02 15:04 MST" -// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit -// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A -// unit that cannot be read is unhealthy: failure is loud. -func (m *Machine) UpgradeHealth() Health { - p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp") - if err != nil { - return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} +// keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the +// bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order. +var keeperConfig = func() KeeperConfig { + var c KeeperConfig + if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil { + _ = json.Unmarshal(raw, &c) } - switch { - case p["LoadState"] == "not-found": - return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} - case p["ActiveState"] == "activating" || p["ActiveState"] == "active": - return m.runningHealth(p["ExecMainStartTimestamp"]) - case p["ActiveState"] != "failed" && p["Result"] == "success": - if at := p["ExecMainExitTimestamp"]; at != "" { - return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} - } - return Health{Healthy: true, Why: "no weekly upgrade has run yet"} - } - why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) - if at := p["ExecMainExitTimestamp"]; at != "" { - why += " at " + at - } - found := []string{} - if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { - why += ": " + said - found = append(found, said) - } - return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} + return c } -// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the -// start and the last line the journal holds. -func (m *Machine) runningHealth(started string) Health { - at, err := time.ParseInLocation(systemdTime, started, time.Local) +// readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing. +type readOnly struct{ StateReader } + +var errReadOnly = errors.New("the health writes nothing") + +func (readOnly) Create(string, any) (uint64, error) { return 0, errReadOnly } +func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly } +func (readOnly) Put(string, any) (uint64, error) { return 0, errReadOnly } +func (readOnly) Delete(string) error { return errReadOnly } + +// UpgradeHealth is the health of this node's upgrades, from the module's states. +func (m *Machine) UpgradeHealth() Health { + if h, said := m.InstalledHealth(); said { + return h + } + if m.States == nil || m.Node == "" { + return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"} + } + cfg := keeperConfig() + if len(cfg.UpgradeOrder) == 0 { + return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " + + "upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"} + } + runs := readOnly{m.States(StateRuns)} + latest, err := latestOwnRun(runs, m.Node) if err != nil { - return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)} + return Health{Why: "the node's runs could not be read: " + err.Error()} } - if m.Now().Sub(at) <= RunLimit { - return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started} + if latest != nil { + if h, said := m.runHealth(*latest); said { + return h + } } - why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit) - found := []string{} - if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") { - why += "; its last line: " + strings.TrimSpace(ls[0]) - found = append(found, strings.TrimSpace(ls[0])) + if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said { + return h } - return Health{Why: why, Found: found} + if latest == nil { + return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"} + } + return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)} +} + +// latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history. +func latestOwnRun(runs Bucket, node string) (*Run, error) { + keys, err := runs.Keys() + if err != nil { + return nil, err + } + sort.Sort(sort.Reverse(sort.StringSlice(keys))) + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := runs.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var r Run + if err := json.Unmarshal(e.Value, &r); err != nil { + return nil, fmt.Errorf("run %s: %w", k, err) + } + if r.From == FromLog || len(r.Steps) == 0 { + continue + } + return &r, nil + } + return nil, nil +} + +// runHealth judges the node's latest run; said is false when the run has nothing to say against the node. +func (m *Machine) runHealth(r Run) (Health, bool) { + var held []HeldKernel + var started *Step + for i, s := range r.Steps { + switch s.Step { + case StepCheckedBefore: + held = s.Held + case StepStarted: + started = &r.Steps[i] + } + } + what := r.Kind + " run " + r.Run + last := r.Steps[len(r.Steps)-1] + switch lastStep(r) { + case StepRefused: + return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true + case StepCheckedAfter: + if last.Check == nil || !last.Check.Passed { + why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime)) + for _, s := range r.Steps { + if s.Step == StepFailed { + why += ": its transaction failed: " + s.Error + } + } + if last.Check != nil { + for _, u := range last.Check.Unbuilt { + why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel) + } + } + return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true + } + case StepStarted: + if started != nil && m.Now().Sub(started.At) > RunLimit { + return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start", + what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true + } + } + if len(held) > 0 { + var said []string + for _, h := range held { + said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound)) + } + return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true + } + return Health{}, false +} + +// roundHealth judges this node's place in the newest round; said is false when the round has nothing to say +// against it. +func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) { + id, err := latestRound(rounds) + if err != nil { + return Health{Why: "the rounds could not be read: " + err.Error()}, true + } + if id == "" { + return Health{}, false + } + v, err := readRound(rounds, runs, id) + if err != nil { + return Health{Why: "round " + id + " could not be read: " + err.Error()}, true + } + if v == nil || !contains(v.Round.Order, m.Node) { + return Health{}, false + } + if startMinutes < 1 { + startMinutes = defaultRoundStartMinutes + } + byNode := map[string]NodeInRound{} + for _, s := range RoundState(v, m.Now(), startMinutes) { + byNode[s.Node] = s + } + since := func(s NodeInRound) string { + if s.Since == nil { + return "an unknown time" + } + return s.Since.Local().Format(healthTime) + } + me := byNode[m.Node] + switch me.State { + case InRoundStalled: + return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true + case InRoundWaiting: + if w, ok := byNode[me.WaitsFor]; ok { + switch w.State { + case InRoundStalled: + return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true + case InRoundFailed: + return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true + } + } + } + return Health{}, false } // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health_test.go b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go new file mode 100644 index 00000000..86690236 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go @@ -0,0 +1,147 @@ +package main + +// The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and +// issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a +// unit's age, so a laptop that slept through a run is judged by what the bus holds. + +import ( + "strings" + "testing" + "time" +) + +// the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go) +var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) + +func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) { + t.Helper() + states := &fakeStates{} + m := machine(nil, 1000) + m.Node, m.States = "node-b", readerOf(states) + saved := keeperConfig + keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} } + t.Cleanup(func() { keeperConfig = saved }) + return m, states +} + +func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) } + +func passedRun(node string, at time.Time) Run { + return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{ + {Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at}, + {Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}} +} + +func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { + m, _ := healthBed(t, nil) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") { + t.Fatalf("%+v", h) + } +} + +func TestANodeWithNoRunYetIsHealthy(t *testing.T) { + m, _ := healthBed(t, []string{"node-a", "node-b"}) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("%+v", h) + } +} + +// issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from +// the log is history and judges nothing. +func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { + m, states := healthBed(t, []string{"node-a", "node-b"}) + exit := 1 + failedAt := healthNow.Add(-7 * 24 * time.Hour) + putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{ + {Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"}, + {Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}}) + putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog, + Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}}) + putRun(states, passedRun("node-a", healthNow.Add(-time.Minute))) + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) { + t.Fatalf("a failed run: %+v", h) + } + putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute))) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("a later run passed: %+v", h) + } +} + +func TestAFailedCheckIsUnhealthy(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + r := passedRun("node-b", healthNow.Add(-time.Hour)) + r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}} + putRun(states, r) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") { + t.Fatalf("%+v", h) + } +} + +func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + started := healthNow.Add(-3 * time.Hour) + putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}}) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) { + t.Fatalf("3 h: %+v", h) + } + m2, states2 := healthBed(t, []string{"node-b"}) + started = healthNow.Add(-10 * time.Minute) + putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}}) + if h := m2.UpgradeHealth(); !h.Healthy { + t.Fatalf("10 min: %+v", h) + } +} + +// 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it. +func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + r := passedRun("node-b", healthNow.Add(-2*time.Hour)) + r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}} + putRun(states, r) + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") { + t.Fatalf("%+v", h) + } + putRun(states, passedRun("node-b", healthNow.Add(-time.Hour))) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("a later run held nothing: %+v", h) + } +} + +// 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it +// waits for and since when; the stalled node's own health says it too. +func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { + order := []string{"node-a", "node-b", "node-c"} + m, states := healthBed(t, order) + round := RoundID(healthNow) + opened := healthNow.Add(-3 * time.Hour) + states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order}) + putRun(states, passedRun("node-a", opened)) + states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)}) + // node-a passed 2 h 54 min ago; node-b never started. + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "has not started") { + t.Fatalf("the stalled node: %+v", h) + } + m.Node = "node-c" + h = m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") { + t.Fatalf("the node waiting on it: %+v", h) + } +} + +func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { + order := []string{"node-a", "node-b"} + m, states := healthBed(t, order) + round := RoundID(healthNow) + at := healthNow.Add(-time.Hour) + states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order}) + r := passedRun("node-a", at) + r.Steps[len(r.Steps)-1].Check = &Check{Passed: false} + putRun(states, r) + states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run}) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") { + t.Fatalf("%+v", h) + } +} diff --git a/modules/pacman/cmd/pacman-tools/upgrades.go b/modules/pacman/cmd/pacman-tools/upgrades.go new file mode 100644 index 00000000..1ed15457 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrades.go @@ -0,0 +1,409 @@ +package main + +// Upgrades as runs on the bus (novox/hq ADR 0508 §1): every full upgrade of a node, and every rollback, is a run +// in the state `runs` at `.`, written at each of its steps and at no other time, and each step is also +// an event. The keeper's loop calls what is here; everything it reaches is behind an interface (a state, the +// events, the root acts) or the Machine's injected runner and files, so it is tested without a bus or a machine. + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Bucket is one of the module's states as a writer sees it: a process's state, or a tools bundle's +// (stdio.KeptState). A write that lost a race is stdio.ErrChanged. +type Bucket interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) + Put(key string, value any) (uint64, error) + Delete(key string) error + Keys() ([]string, error) +} + +// EventSink emits the keeper's events: the process's connection (process.Conn). +type EventSink interface { + Emit(eventType string, body any, opts ...stdio.EmitOptions) error +} + +// RootActs are the acts that need root, carried out by the module's root helper (ADR 0510): the keeper never +// runs them itself. +type RootActs interface { + // Transaction starts the run's transaction as a unit of the service manager, named TransactionUnit(kind, + // run), leaving out the packages ignore names, and answers once it started. + Transaction(kind, run string, ignore []string) error + // Rollback starts a rollback's transaction as a unit of the service manager, named TransactionUnit(RunRollback, + // run), carrying out the plan in the request (ADR 0510: the plan travels with the verb), and answers once it + // started. + Rollback(run string, plan []RollbackItem) error + // Journal is the last n lines a transaction unit logged. + Journal(unit string, n int) ([]string, error) + // Maps is every process still mapping a replaced file, by its unit or container and its executable. + Maps() ([]Mapped, error) +} + +// TransactionUnit is the unit a run's transaction runs as: named by the run, so a keeper started again finds it. +func TransactionUnit(kind, run string) string { + return unitPrefix + kind + "-" + strings.ToLower(run) +} + +// Upgrades is one node's upgrades: what the keeper's loop calls for its own node. +type Upgrades struct { + Node string + Config KeeperConfig + Machine *Machine + Root RootActs + Events EventSink + // The module's states. + Installed, Updates, Runs, Records, Rounds, Asks Bucket + // AskRecord reads the operator channel's record of one of this module's asks (process.Conn.Ask on + // MessengerAsksBucket): how a pass is answered. + AskRecord func(id string) (*stdio.StateEntry, error) + // Check is the after-upgrade check (ADR 0508 §4); nil is AfterCheck. + Check func(run Run, record *Record) Check +} + +// openRun is a run being written, with the revision of its key. +type openRun struct { + Run + rev uint64 +} + +// step writes the run with one more step: created at its first step, then only ever updated from the revision +// this writer wrote, so a second writer is refused rather than overwritten. +func (u *Upgrades) step(r *openRun, s Step) error { + r.Steps = append(r.Steps, s) + key := NodeKey(r.Node, r.Run.Run) + var err error + if r.rev == 0 { + r.rev, err = u.Runs.Create(key, r.Run) + } else { + r.rev, err = u.Runs.Update(key, r.Run, r.rev) + } + if err != nil { + r.Steps = r.Steps[:len(r.Steps)-1] + return fmt.Errorf("writing step %s of run %s: %w", s.Step, key, err) + } + return nil +} + +// emit says a step as its event; an event that is not sent is said in the error, the step stays written. +func (u *Upgrades) emit(event string, body any) error { + if u.Events == nil { + return nil + } + if err := u.Events.Emit(event, body); err != nil { + return fmt.Errorf("emitting %s: %w", event, err) + } + return nil +} + +// The package manager's log, read as blocks: each begins at a `Running ''` line (or a full upgrade +// starting with none before it) and holds the transaction it ran and the packages it changed. +var ( + runningLine = regexp.MustCompile(`^\[([^\]]+)\] \[PACMAN\] Running '(.*)'$`) + transactionLine = regexp.MustCompile(`^\[([^\]]+)\] \[ALPM\] transaction (started|completed|failed|interrupted)$`) + pacnewLine = regexp.MustCompile(`^\[[^\]]+\] \[ALPM\] warning: \S+ installed as (\S+\.pacnew)$`) + pacsaveLine = regexp.MustCompile(`^\[[^\]]+\] \[ALPM\] warning: \S+ saved as (\S+\.pacsave)$`) +) + +// logBlock is one command pacman ran, as its log tells it. +type logBlock struct { + Start time.Time + Command string + Full bool // it started a full system upgrade + FullAt time.Time // when + Began bool // a transaction started + End string // completed, failed or interrupted; empty when the log holds no end + EndAt time.Time + Changes Changes + // The .pacnew and .pacsave files the transaction left, by path, in the order logged. + Pacnew, Pacsave []string +} + +// logBlocks reads pacman's log into the commands it ran, oldest first. +func logBlocks(text string) []logBlock { + var blocks []logBlock + var cur *logBlock + open := func(at time.Time, command string) { + blocks = append(blocks, logBlock{Start: at, Command: command}) + cur = &blocks[len(blocks)-1] + } + for _, l := range strings.Split(text, "\n") { + l = strings.TrimRight(l, "\r") + if m := runningLine.FindStringSubmatch(l); m != nil { + if t, ok := logTime(m[1]); ok { + open(t, m[2]) + } + continue + } + if m := fullUpdate.FindStringSubmatch(l); m != nil { + t, ok := logTime(m[1]) + if !ok { + continue + } + if cur == nil || cur.Full || cur.Began { + open(t, "") + } + cur.Full, cur.FullAt = true, t + continue + } + if cur == nil { + continue + } + if m := transactionLine.FindStringSubmatch(l); m != nil { + t, _ := logTime(m[1]) + if m[2] == "started" { + cur.Began = true + } else if cur.Began && cur.End == "" { + cur.End, cur.EndAt = m[2], t + } + continue + } + if cur.Began && cur.End == "" { + if m := logLine.FindStringSubmatch(l); m != nil { + addChange(&cur.Changes, m[2], m[3], m[4]) + } else if m := pacnewLine.FindStringSubmatch(l); m != nil { + cur.Pacnew = append(cur.Pacnew, m[1]) + } else if m := pacsaveLine.FindStringSubmatch(l); m != nil { + cur.Pacsave = append(cur.Pacsave, m[1]) + } + } + } + for i := range blocks { + sortChanges(&blocks[i].Changes) + } + return blocks +} + +func addChange(c *Changes, action, name, version string) { + switch action { + case "installed": + c.Added = append(c.Added, PackageVersion{Name: name, Version: version}) + case "removed": + c.Removed = append(c.Removed, PackageVersion{Name: name, Version: version}) + case "upgraded", "downgraded": + from, to, _ := strings.Cut(version, " -> ") + c.Changed = append(c.Changed, VersionChange{Name: name, From: from, To: to}) + } +} + +func sortChanges(c *Changes) { + if c.Added == nil { + c.Added = []PackageVersion{} + } + if c.Removed == nil { + c.Removed = []PackageVersion{} + } + if c.Changed == nil { + c.Changed = []VersionChange{} + } + sort.Slice(c.Added, func(i, j int) bool { return c.Added[i].Name < c.Added[j].Name }) + sort.Slice(c.Removed, func(i, j int) bool { return c.Removed[i].Name < c.Removed[j].Name }) + sort.Slice(c.Changed, func(i, j int) bool { return c.Changed[i].Name < c.Changed[j].Name }) +} + +// noTransaction is why a full upgrade the log tells of has no transaction: the log does not say which. +const noTransaction = "the log holds no transaction for this upgrade: there was nothing to do, or it ended before changing anything" + +// PastUpgrades is every full upgrade pacman's log tells of, as runs read from the log (ADR 0508 §1): started +// when the upgrade started, finished or failed when its transaction ended, with what it changed. One whose +// transaction has no end in the log failed, so said. +func PastUpgrades(node, text string) []Run { + var runs []Run + for _, b := range logBlocks(text) { + if !b.Full { + continue + } + r := Run{Node: node, Run: RunID(b.FullAt), Kind: RunUpgrade, From: FromLog, + Steps: []Step{{Step: StepStarted, At: b.FullAt}}} + changes := b.Changes + switch { + case !b.Began: + r.Steps = append(r.Steps, Step{Step: StepFinished, At: b.FullAt, Why: noTransaction, Changes: &changes}) + case b.End == "completed": + r.Steps = append(r.Steps, Step{Step: StepFinished, At: b.EndAt, Changes: &changes}) + case b.End == "": + r.Steps = append(r.Steps, Step{Step: StepFailed, At: b.FullAt, Error: "the log holds no end of this transaction", Changes: &changes}) + default: + r.Steps = append(r.Steps, Step{Step: StepFailed, At: b.EndAt, Error: "transaction " + b.End, Changes: &changes}) + } + runs = append(runs, r) + } + return runs +} + +// ImportLog reads the node's past full upgrades from pacman's log into `runs` once: when the node has no run +// yet. It answers how many it wrote. A run read from the log is history and emits nothing. +func (u *Upgrades) ImportLog() (int, error) { + keys, err := u.Runs.Keys() + if err != nil { + return 0, fmt.Errorf("listing runs: %w", err) + } + for _, k := range keys { + if strings.HasPrefix(k, u.Node+".") { + return 0, nil + } + } + text, err := u.Machine.ReadFile(PacmanLog) + if err != nil { + return 0, fmt.Errorf("reading %s: %w", PacmanLog, err) + } + n := 0 + for _, r := range PastUpgrades(u.Node, string(text)) { + if _, err := u.Runs.Create(NodeKey(u.Node, r.Run), r); err != nil { + if errors.Is(err, stdio.ErrChanged) { + continue // two upgrades started in the same second: the first is kept + } + return n, fmt.Errorf("writing run %s: %w", r.Run, err) + } + n++ + } + return n, nil +} + +// runTime is when a run began, from its id. +func runTime(id string) (time.Time, bool) { + t, err := time.Parse("20060102T150405Z", id) + return t, err == nil +} + +// RunSummary is one run as pacman_upgrades lists it. +type RunSummary struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Began time.Time `json:"began"` + Outcome string `json:"outcome"` + Ended *time.Time `json:"ended,omitempty"` + Error string `json:"error,omitempty"` + Held int `json:"held_kernels,omitempty"` + Checked string `json:"check,omitempty"` +} + +// summary is where a run stands: its outcome is its last step that ends it (refused, finished, failed), else +// running. +func summary(r Run) RunSummary { + s := RunSummary{Node: r.Node, Run: r.Run, Kind: r.Kind, From: r.From, Round: r.Round, Outcome: "running"} + if t, ok := runTime(r.Run); ok { + s.Began = t + } + for _, st := range r.Steps { + switch st.Step { + case StepRefused, StepFinished, StepFailed: + at := st.At + s.Outcome, s.Ended = st.Step, &at + s.Error = st.Error + if st.Step == StepRefused { + s.Error = st.Why + } + case StepCheckedBefore: + s.Held = len(st.Held) + case StepCheckedAfter: + s.Checked = "failed" + if st.Check != nil && st.Check.Passed { + s.Checked = "passed" + } + } + } + return s +} + +// ListUpgrades answers pacman_upgrades from `runs`: one run whole, or the runs of every node or one, between +// two days, newest first, at most limit of them. +func ListUpgrades(runs Bucket, node, from, until, run string, limit int) (map[string]any, error) { + if run != "" { + if node == "" { + return nil, fmt.Errorf("a run is named with its node") + } + e, err := runs.Get(NodeKey(node, run)) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("no run %s of %s", run, node) + } + var r Run + if err := jsonValue(e.Value, &r); err != nil { + return nil, err + } + return map[string]any{"run": r, "summary": summary(r)}, nil + } + day := func(s string, end bool) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + t, err := time.ParseInLocation("2006-01-02", s, time.Local) + if err != nil { + return t, fmt.Errorf("%q is not a day as YYYY-MM-DD", s) + } + if end { + t = t.AddDate(0, 0, 1) + } + return t, nil + } + since, err := day(from, false) + if err != nil { + return nil, err + } + before, err := day(until, true) + if err != nil { + return nil, err + } + keys, err := runs.Keys() + if err != nil { + return nil, err + } + var listed []RunSummary + for _, k := range keys { + n, id, ok := strings.Cut(k, ".") + if !ok || node != "" && n != node { + continue + } + t, ok := runTime(id) + if !ok || t.Before(since) || !before.IsZero() && !t.Before(before) { + continue + } + e, err := runs.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var r Run + if err := jsonValue(e.Value, &r); err != nil { + return nil, fmt.Errorf("run %s: %w", k, err) + } + listed = append(listed, summary(r)) + } + sort.Slice(listed, func(i, j int) bool { + if !listed[i].Began.Equal(listed[j].Began) { + return listed[i].Began.After(listed[j].Began) + } + return listed[i].Node < listed[j].Node + }) + truncated := len(listed) > limit + if truncated { + listed = listed[:limit] + } + if listed == nil { + listed = []RunSummary{} + } + return map[string]any{"count": len(listed), "runs": listed, "truncated": truncated}, nil +} + +// jsonValue reads a state's value into its shape. +func jsonValue(raw []byte, v any) error { return json.Unmarshal(raw, v) } + +// upgradeState is a state of this module as the tools bundle reads it; a test replaces it. +var upgradeState = func(name string) Bucket { return stdio.State(name) } diff --git a/modules/pacman/cmd/pacman-tools/view.go b/modules/pacman/cmd/pacman-tools/view.go new file mode 100644 index 00000000..1d7918b5 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/view.go @@ -0,0 +1,377 @@ +package main + +// The package view (novox/hq ADR 0507 §6, §7; acceptance 1840/AC6): verbs that read the module's states and the +// controller's `declared-packages`, so any instance answers for every node and no machine is asked. A node that is +// off keeps its last values, shown with the time they describe; a node whose packages are over the bus's cap is +// marked not current. A foreign package is listed apart in every answer. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// DeclaredPackagesState is the controller's state of the packages each node's modules declare (ADR 0507 §5), +// read through the manifest's `reads`. +const DeclaredPackagesState = "mesh-controller.declared-packages" + +// declaredValue is a node's value of `declared-packages`, as the controller writes it. +type declaredValue struct { + Node string `json:"node"` + Packages []struct { + Package string `json:"package"` + Modules []string `json:"modules"` + Absent bool `json:"absent"` + } `json:"packages"` +} + +// nodeInstalled is one node's key of `installed` with its not-current mark. +type nodeInstalled struct { + Node string + Value InstalledValue + NotCurrent *NotCurrent +} + +func readValue(r StateReader, key string, v any) (bool, error) { + e, err := r.Get(key) + if err != nil || e == nil { + return false, err + } + if err := json.Unmarshal(e.Value, v); err != nil { + return false, fmt.Errorf("%s is not readable: %w", key, err) + } + return true, nil +} + +// nodeKeys are the node keys of a state (``), sorted, and the marks (`.not-current`) apart. +func nodeKeys(r StateReader) (nodes []string, marks map[string]bool, err error) { + keys, err := r.Keys() + if err != nil { + return nil, nil, err + } + marks = map[string]bool{} + for _, k := range keys { + if node, ok := strings.CutSuffix(k, ".not-current"); ok { + marks[node] = true + continue + } + if !strings.Contains(k, ".") { + nodes = append(nodes, k) + } + } + sort.Strings(nodes) + return nodes, marks, nil +} + +func (m *Machine) states() error { + if m.States == nil { + return fmt.Errorf("the module's states cannot be read here") + } + return nil +} + +// installedOn is the installed packages of one node, or of every node when node is empty. +func (m *Machine) installedOn(node string) ([]nodeInstalled, error) { + if err := m.states(); err != nil { + return nil, err + } + r := m.States(StateInstalled) + nodes, marks, err := nodeKeys(r) + if err != nil { + return nil, err + } + if node != "" { + nodes = []string{node} + } + var out []nodeInstalled + for _, n := range nodes { + ni := nodeInstalled{Node: n} + found, err := readValue(r, n, &ni.Value) + if err != nil { + return nil, err + } + if !found { + if node != "" { + return nil, fmt.Errorf("no packages of %s are on the bus: its keeper has not written them", node) + } + continue + } + if marks[n] || node != "" { + var nc NotCurrent + if ok, err := readValue(r, NotCurrentKey(n), &nc); err != nil { + return nil, err + } else if ok { + ni.NotCurrent = &nc + } + } + out = append(out, ni) + } + return out, nil +} + +// declaredOn is what each node's modules declare present, by node and package, with the declaring modules; a +// node the controller holds nothing for is missing. +func (m *Machine) declaredOn(nodes []string) (map[string]map[string][]string, error) { + r := m.States(DeclaredPackagesState) + out := map[string]map[string][]string{} + for _, n := range nodes { + var v declaredValue + found, err := readValue(r, n, &v) + if err != nil { + return nil, err + } + if !found { + continue + } + out[n] = map[string][]string{} + for _, p := range v.Packages { + if !p.Absent { + out[n][p.Package] = p.Modules + } + } + } + return out, nil +} + +// NodeView is one node in the view. +type NodeView struct { + Node string `json:"node"` + // At is the time of the transaction that left the node's packages as shown. + At time.Time `json:"at"` + Packages int `json:"packages"` + Explicit int `json:"explicit"` + // Current is false while the node's packages are over the bus's cap: what is shown is older. + Current bool `json:"current"` + NotCurrent *NotCurrent `json:"not_current,omitempty"` + // Undeclared is the packages installed explicitly from a repository that no module assigned to the node + // declares; null when the controller holds no declaration of the node. + Undeclared []PackageVersion `json:"undeclared"` + Foreign []PackageVersion `json:"foreign"` +} + +// Spread is one package across the nodes: its version on each node that has it. +type Spread struct { + Name string `json:"name"` + Versions map[string]string `json:"versions"` +} + +// PackageOnNode is one package on one node. +type PackageOnNode struct { + Node string `json:"node"` + Installed bool `json:"installed"` + Version string `json:"version,omitempty"` + Explicit bool `json:"explicit,omitempty"` + Repository string `json:"repository,omitempty"` + Foreign bool `json:"foreign,omitempty"` + DeclaredBy []string `json:"declared_by"` + Current bool `json:"current"` +} + +// Packages is pacman_packages: one package on every node, or the whole view: per node its counts, its explicit +// undeclared packages and its foreign ones; the packages whose versions differ between nodes, and the packages +// on some nodes only, among the packages from a repository (each node's foreign ones are in its own list). +func (m *Machine) Packages(name string, limit int) (map[string]any, error) { + all, err := m.installedOn("") + if err != nil { + return nil, err + } + nodes := make([]string, 0, len(all)) + for _, ni := range all { + nodes = append(nodes, ni.Node) + } + declared, err := m.declaredOn(nodes) + if err != nil { + return nil, err + } + if name != "" { + on := []PackageOnNode{} + for _, ni := range all { + p := PackageOnNode{Node: ni.Node, Current: ni.NotCurrent == nil, DeclaredBy: declared[ni.Node][name]} + if p.DeclaredBy == nil { + p.DeclaredBy = []string{} + } + for _, ip := range ni.Value.Packages { + if ip.Name == name { + p.Installed, p.Version, p.Explicit = true, ip.Version, ip.Explicit + p.Foreign = ip.Repo == RepoForeign + if !p.Foreign { + p.Repository = ip.Repo + } + } + } + on = append(on, p) + } + return map[string]any{"package": name, "nodes": on}, nil + } + views := []NodeView{} + versions := map[string]map[string]string{} + for _, ni := range all { + v := NodeView{Node: ni.Node, At: ni.Value.At, Packages: len(ni.Value.Packages), Current: ni.NotCurrent == nil, + NotCurrent: ni.NotCurrent, Foreign: []PackageVersion{}} + d, known := declared[ni.Node] + if known { + v.Undeclared = []PackageVersion{} + } + for _, p := range ni.Value.Packages { + if p.Explicit { + v.Explicit++ + } + if p.Repo == RepoForeign { + v.Foreign = append(v.Foreign, PackageVersion{p.Name, p.Version}) + continue + } + if versions[p.Name] == nil { + versions[p.Name] = map[string]string{} + } + versions[p.Name][ni.Node] = p.Version + if known && p.Explicit && d[p.Name] == nil { + v.Undeclared = append(v.Undeclared, PackageVersion{p.Name, p.Version}) + } + } + views = append(views, v) + } + names := make([]string, 0, len(versions)) + for n := range versions { + names = append(names, n) + } + sort.Strings(names) + differing, some := []Spread{}, []Spread{} + for _, n := range names { + vs := versions[n] + if len(vs) < len(all) { + some = append(some, Spread{n, vs}) + } + seen := map[string]bool{} + for _, ver := range vs { + seen[ver] = true + } + if len(seen) > 1 { + differing = append(differing, Spread{n, vs}) + } + } + out := map[string]any{"nodes": views} + for key, list := range map[string][]Spread{"differing": differing, "some_nodes": some} { + b := bound(key, list, limit) + out[key] = b[key] + out[key+"_count"] = b["count"] + out[key+"_truncated"] = b["truncated"] + } + return out, nil +} + +// UpdatesView is one node's `updates` with the time its key last changed. +type UpdatesView struct { + Node string `json:"node"` + Reachable bool `json:"reachable"` + Count int `json:"count"` + Packages []VersionChange `json:"packages"` + Changed *time.Time `json:"changed,omitempty"` +} + +// UpdatesFor is pacman_updates: what a full upgrade would change, on one node or on every node, each with when its +// key last changed (the bus's time of the write; the value holds no time of a read, ADR 0289 §4). +func (m *Machine) UpdatesFor(node string) (map[string]any, error) { + if err := m.states(); err != nil { + return nil, err + } + r := m.States(StateUpdates) + nodes := []string{node} + if node == "" { + var err error + if nodes, _, err = nodeKeys(r); err != nil { + return nil, err + } + } + views := []UpdatesView{} + for _, n := range nodes { + var v UpdatesValue + found, err := readValue(r, n, &v) + if err != nil { + return nil, err + } + if !found { + if node != "" { + return nil, fmt.Errorf("no updates of %s are on the bus: its keeper has not written them", node) + } + continue + } + uv := UpdatesView{Node: n, Reachable: v.Reachable, Count: len(v.Packages), Packages: v.Packages} + if uv.Packages == nil { + uv.Packages = []VersionChange{} + } + if h, err := r.History(n); err == nil && len(h) > 0 { + at := h[len(h)-1].At + uv.Changed = &at + } + views = append(views, uv) + } + return map[string]any{"nodes": views}, nil +} + +// InstalledFrom is pacman_installed from the bus: a node's installed packages with version, reason, repository +// and whether foreign, narrowed by name, reason or foreign, with totals, the time they describe and whether +// they are current. +func (m *Machine) InstalledFrom(node, match, reason string, foreignOnly bool, limit int) (map[string]any, error) { + ni, err := m.oneNode(node) + if err != nil { + return nil, err + } + pkgs := []Package{} + counts := map[string]int{"explicit": 0, "dependency": 0, "foreign": 0} + for _, ip := range ni.Value.Packages { + p := Package{Name: ip.Name, Version: ip.Version, Reason: "dependency", Foreign: ip.Repo == RepoForeign} + if ip.Explicit { + p.Reason = "explicit" + } + if !p.Foreign { + p.Repository = ip.Repo + } + counts[p.Reason]++ + if p.Foreign { + counts["foreign"]++ + } + if match != "" && !strings.Contains(p.Name, match) || reason != "" && p.Reason != reason || foreignOnly && !p.Foreign { + continue + } + pkgs = append(pkgs, p) + } + out := bound("packages", pkgs, limit) + out["totals"] = counts + out["node"], out["at"], out["current"] = ni.Node, ni.Value.At, ni.NotCurrent == nil + if ni.NotCurrent != nil { + out["not_current"] = ni.NotCurrent + } + return out, nil +} + +// ForeignFrom is pacman_foreign from the bus: a node's packages from outside the repositories. +func (m *Machine) ForeignFrom(node string) (map[string]any, error) { + ni, err := m.oneNode(node) + if err != nil { + return nil, err + } + pkgs := []PackageVersion{} + for _, ip := range ni.Value.Packages { + if ip.Repo == RepoForeign { + pkgs = append(pkgs, PackageVersion{ip.Name, ip.Version}) + } + } + return map[string]any{"node": ni.Node, "count": len(pkgs), "packages": pkgs, "at": ni.Value.At, "current": ni.NotCurrent == nil}, nil +} + +// oneNode is the named node's packages, this node's when none is named. +func (m *Machine) oneNode(node string) (nodeInstalled, error) { + if node == "" { + node = m.Node + } + if node == "" { + return nodeInstalled{}, fmt.Errorf("node is required: this bundle does not know its own") + } + all, err := m.installedOn(node) + if err != nil { + return nodeInstalled{}, err + } + return all[0], nil +} diff --git a/modules/pacman/cmd/pacman-tools/view_test.go b/modules/pacman/cmd/pacman-tools/view_test.go new file mode 100644 index 00000000..8a823c55 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/view_test.go @@ -0,0 +1,160 @@ +package main + +// The package view over the bus (novox/hq ADR 0507 §6, §7; acceptance 1840/AC6): two nodes' `installed` keys and +// their `declared-packages`, answered without asking any machine. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +// viewStates is two nodes' packages on the bus: node-a current and declared, node-b over the cap since a while, +// with a foreign package, and no declaration the controller holds. +func viewStates(t *testing.T) (*fakeStates, *Machine) { + t.Helper() + states := &fakeStates{} + inst := states.store(StateInstalled) + inst.Put("node-a", InstalledValue{At: t0, Packages: []InstalledPackage{ + {"glibc", "2.42-1", false, "core"}, {"tmux", "3.5-1", true, "extra"}, {"vim", "9.1-1", true, "extra"}, + {"zsh", "5.9-2", true, "extra"}, + }}) + inst.Put("node-b", InstalledValue{At: t0.Add(-time.Hour), Packages: []InstalledPackage{ + {"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", false, "extra"}, + }}) + inst.Put("node-b.not-current", NotCurrent{Bytes: 300000, Since: t0.Add(-30 * time.Minute)}) + decl := states.store(DeclaredPackagesState) + var d declaredValue + json.Unmarshal([]byte(`{"node":"node-a","packages":[{"package":"vim","modules":["editor"],"absent":false},`+ + `{"package":"tmux","modules":["shell"],"absent":true},{"package":"zsh","modules":["login","shell"],"absent":false}]}`), &d) + decl.Put("node-a", d) + m := machine(func(_ context.Context, name string, args ...string) Ran { + t.Errorf("a machine was asked: %s %v", name, args) + return Ran{Status: 99} + }, 1000) + m.Node, m.States = "node-a", readerOf(states) + return states, m +} + +func TestOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { + _, m := viewStates(t) + r, err := m.Packages("zsh", 500) + if err != nil { + t.Fatal(err) + } + on := r["nodes"].([]PackageOnNode) + if len(on) != 2 { + t.Fatalf("%+v", on) + } + a, b := on[0], on[1] + if a.Node != "node-a" || !a.Installed || a.Version != "5.9-2" || !a.Explicit || a.Repository != "extra" || + strings.Join(a.DeclaredBy, ",") != "login,shell" || !a.Current { + t.Errorf("node-a: %+v", a) + } + if b.Node != "node-b" || b.Version != "5.9-1" || b.Explicit || len(b.DeclaredBy) != 0 || b.Current { + t.Errorf("node-b: %+v", b) + } + r, _ = m.Packages("yay", 500) + on = r["nodes"].([]PackageOnNode) + if on[0].Installed || !on[1].Installed || !on[1].Foreign || on[1].Repository != "" { + t.Errorf("yay: %+v", on) + } +} + +func TestTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart(t *testing.T) { + _, m := viewStates(t) + r, err := m.Packages("", 500) + if err != nil { + t.Fatal(err) + } + nodes := r["nodes"].([]NodeView) + if len(nodes) != 2 { + t.Fatalf("%+v", nodes) + } + a, b := nodes[0], nodes[1] + if a.Node != "node-a" || a.Packages != 4 || a.Explicit != 3 || !a.Current || fmt.Sprint(a.Undeclared) != "[{tmux 3.5-1}]" || + len(a.Foreign) != 0 || !a.At.Equal(t0) { + t.Errorf("node-a: %+v", a) + } + if b.Current || b.NotCurrent == nil || b.NotCurrent.Bytes != 300000 || b.Undeclared != nil || + fmt.Sprint(b.Foreign) != "[{yay 12.0-1}]" || !b.At.Equal(t0.Add(-time.Hour)) { + t.Errorf("node-b (not current, no declaration, a foreign package): %+v", b) + } + if d := r["differing"].([]Spread); len(d) != 1 || d[0].Name != "zsh" || d[0].Versions["node-a"] != "5.9-2" || d[0].Versions["node-b"] != "5.9-1" { + t.Errorf("differing: %+v", d) + } + some := r["some_nodes"].([]Spread) + var names []string + for _, s := range some { + names = append(names, s.Name) + } + if strings.Join(names, ",") != "tmux,vim" { + t.Errorf("on some nodes only, the foreign apart: %v", names) + } + r, _ = m.Packages("", 1) + if r["some_nodes_count"] != 2 || r["some_nodes_truncated"] != true || len(r["some_nodes"].([]Spread)) != 1 { + t.Errorf("bounded: %v", r) + } +} + +func TestInstalledAndForeignAnswerFromTheBus(t *testing.T) { + _, m := viewStates(t) + r, err := m.InstalledFrom("", "", "", false, 10) + if err != nil || r["node"] != "node-a" || r["count"] != 4 || r["current"] != true { + t.Fatalf("this node's: %v %v", r, err) + } + pk := r["packages"].([]Package) + if pk[0].Reason != "dependency" || pk[0].Repository != "core" || pk[1].Reason != "explicit" { + t.Errorf("%+v", pk) + } + r, err = m.InstalledFrom("node-b", "", "", true, 10) + if err != nil || r["count"] != 1 || r["current"] != false || r["not_current"] == nil { + t.Fatalf("node-b's foreign: %v %v", r, err) + } + if tot := r["totals"].(map[string]int); tot["explicit"] != 1 || tot["dependency"] != 2 || tot["foreign"] != 1 { + t.Errorf("totals: %v", tot) + } + f, err := m.ForeignFrom("node-b") + if err != nil || f["count"] != 1 || fmt.Sprint(f["packages"]) != "[{yay 12.0-1}]" { + t.Errorf("%v %v", f, err) + } + if _, err := m.InstalledFrom("node-c", "", "", false, 10); err == nil || !strings.Contains(err.Error(), "node-c") { + t.Errorf("a node with no packages on the bus: %v", err) + } +} + +func TestUpdatesAnswerEveryNodeAndWhenItsListLastChanged(t *testing.T) { + states, m := viewStates(t) + up := states.store(StateUpdates) + up.now = func() time.Time { return t0.Add(-2 * time.Hour) } + up.Put("node-a", UpdatesValue{Reachable: true, Packages: []VersionChange{{"zsh", "5.9-2", "5.9-3"}}}) + up.now = func() time.Time { return t0.Add(-3 * time.Hour) } + up.Put("node-b", UpdatesValue{Reachable: false, Packages: []VersionChange{}}) + r, err := m.UpdatesFor("") + if err != nil { + t.Fatal(err) + } + v := r["nodes"].([]UpdatesView) + if len(v) != 2 || v[0].Count != 1 || !v[0].Reachable || v[0].Changed == nil || !v[0].Changed.Equal(t0.Add(-2*time.Hour)) || + v[1].Reachable || !v[1].Changed.Equal(t0.Add(-3*time.Hour)) { + t.Fatalf("%+v", v) + } + r, _ = m.UpdatesFor("node-b") + if v := r["nodes"].([]UpdatesView); len(v) != 1 || v[0].Node != "node-b" { + t.Errorf("one node: %+v", v) + } +} + +func TestANodeArgumentIsOneKeyToken(t *testing.T) { + for _, bad := range []string{"node-a.not-current", "node-a.*", "a b"} { + if _, err := nodeOf(map[string]any{"node": bad}); err == nil { + t.Errorf("%q was taken as a node", bad) + } + } + if n, err := nodeOf(map[string]any{"node": "node-a"}); err != nil || n != "node-a" { + t.Errorf("%q %v", n, err) + } +} diff --git a/modules/pacman/cmd/pacman-tools/watch.go b/modules/pacman/cmd/pacman-tools/watch.go new file mode 100644 index 00000000..e66f69ae --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/watch.go @@ -0,0 +1,129 @@ +package main + +// The machine's watch (novox/hq ADR 0507 §2, §4): inotify on the directory of pacman's database lock, for the +// lock file deleted (a transaction ended), and on the mirror list's directory, for the list written or moved +// into place (reflector rewrote it). A directory is watched, not the file: the lock comes and goes, and a list +// replaced by a rename is a new file. Nothing is polled. + +import ( + "errors" + "fmt" + "os" + "sync" + "unsafe" + + "golang.org/x/sys/unix" +) + +type inotify struct { + f *os.File + lockWD int + mirWD int + lock string + mirrors string + out chan Signal + // pending holds a signal of each kind until the keeper takes it, so a burst of one kind is one signal and + // never crowds out the other. + mu sync.Mutex + pending map[Signal]bool + notify chan struct{} +} + +// WatchMachine watches lockDir for lockName deleted and mirrorDir for mirrorName written or moved in. +func WatchMachine(lockDir, lockName, mirrorDir, mirrorName string) (Watcher, error) { + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if err != nil { + return nil, fmt.Errorf("inotify: %w", err) + } + w := &inotify{lock: lockName, mirrors: mirrorName, out: make(chan Signal), pending: map[Signal]bool{}, + notify: make(chan struct{}, 1)} + if w.lockWD, err = unix.InotifyAddWatch(fd, lockDir, unix.IN_DELETE|unix.IN_MOVED_FROM); err != nil { + unix.Close(fd) + return nil, fmt.Errorf("watching %s: %w", lockDir, err) + } + if w.mirWD, err = unix.InotifyAddWatch(fd, mirrorDir, unix.IN_CLOSE_WRITE|unix.IN_MOVED_TO); err != nil { + unix.Close(fd) + return nil, fmt.Errorf("watching %s: %w", mirrorDir, err) + } + // a non-blocking descriptor as an os.File is read through the runtime's poller, so Close ends a read. + w.f = os.NewFile(uintptr(fd), "inotify") + go w.read() + go w.forward() + return w, nil +} + +func (w *inotify) Signals() <-chan Signal { return w.out } + +func (w *inotify) Close() error { return w.f.Close() } + +func (w *inotify) read() { + defer close(w.notify) + buf := make([]byte, 64*(unix.SizeofInotifyEvent+unix.NAME_MAX+1)) + for { + n, err := w.f.Read(buf) + if err != nil { + if !errors.Is(err, os.ErrClosed) { + fmt.Fprintf(os.Stderr, "[pacman] keeper: the machine's watch failed: %v\n", err) + } + return + } + for off := 0; off+unix.SizeofInotifyEvent <= n; { + ev := (*unix.InotifyEvent)(unsafe.Pointer(&buf[off])) + nameAt := off + unix.SizeofInotifyEvent + end := nameAt + int(ev.Len) + if end > n { + break + } + name := cstring(buf[nameAt:end]) + off = end + if ev.Mask&unix.IN_Q_OVERFLOW != 0 { + // events were lost: anything may have happened, so both are read again. + w.send(LockReleased) + w.send(MirrorsRewritten) + continue + } + switch { + case int(ev.Wd) == w.lockWD && name == w.lock: + w.send(LockReleased) + case int(ev.Wd) == w.mirWD && name == w.mirrors: + w.send(MirrorsRewritten) + } + } + } +} + +// send marks a signal pending; one of the same kind already waiting covers it. +func (w *inotify) send(s Signal) { + w.mu.Lock() + w.pending[s] = true + w.mu.Unlock() + select { + case w.notify <- struct{}{}: + default: + } +} + +// forward hands the pending signals to the keeper, in a fixed order, until the watch ends. +func (w *inotify) forward() { + defer close(w.out) + for range w.notify { + for _, s := range []Signal{LockReleased, MirrorsRewritten} { + w.mu.Lock() + due := w.pending[s] + delete(w.pending, s) + w.mu.Unlock() + if due { + w.out <- s + } + } + } +} + +func cstring(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} diff --git a/modules/pacman/files/keeper.json b/modules/pacman/files/keeper.json new file mode 100644 index 00000000..8e340a94 --- /dev/null +++ b/modules/pacman/files/keeper.json @@ -0,0 +1,5 @@ +{ + "upgrade-order": [], + "record-days": 180, + "round-start-minutes": 120 +} diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/modules/pacman/files/mesh-pacman-weekly-upgrade.service deleted file mode 100644 index 69741dd7..00000000 --- a/modules/pacman/files/mesh-pacman-weekly-upgrade.service +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. -[Unit] -Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) -Wants=network-online.target -After=network-online.target - -[Service] -# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health -# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. -Type=oneshot -Nice=10 -ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer deleted file mode 100644 index a8a9b4d9..00000000 --- a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer +++ /dev/null @@ -1,11 +0,0 @@ -# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. -[Unit] -Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) - -[Timer] -# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. -OnCalendar=Sat *-*-* 04:00:00 -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/modules/pacman/go.mod b/modules/pacman/go.mod index a88b95d8..9b1d193f 100644 --- a/modules/pacman/go.mod +++ b/modules/pacman/go.mod @@ -1,5 +1,25 @@ module pacman -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.6 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800 + github.com/nats-io/nats-server/v2 v2.11.17 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + golang.org/x/time v0.15.0 // indirect +) + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 +) diff --git a/modules/pacman/go.sum b/modules/pacman/go.sum index 0dd60617..243e264a 100644 --- a/modules/pacman/go.sum +++ b/modules/pacman/go.sum @@ -1,2 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800 h1:Q29osS06HHcXH1WXURDyITbOLnJG7XJYgCzmZD++Nws= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161324-8892a1176800/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/pacman/module.json b/modules/pacman/module.json index ca1a81a3..69d08ff7 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -11,15 +11,73 @@ "scope": "node" } ], + "reads": [ + "mesh-controller.declared-packages" + ], + "uses": [ + "operator-channel" + ], + "config": { + "upgrade-order": { + "type": "list", + "description": "The nodes in the order they upgrade, each by its name: the first opens each week's round on Saturday at 04:00 local time, and each node upgrades after the one before it passed its after-upgrade check (novox/hq ADR 0508 §5). Empty: no round starts, and every node's health says why. Falls back to the mesh-level key of the same name.", + "trusted": true, + "predefined": "upgrade-order", + "default": [] + }, + "record-days": { + "type": "number", + "description": "How many days a package record taken before an upgrade is kept on the bus before it is deleted (novox/hq ADR 0508 §2); a rollback can return only to a record still kept.", + "minimum": 1, + "trusted": false, + "default": 180 + }, + "round-start-minutes": { + "type": "number", + "description": "How many minutes a node whose turn in the round came may take to start its upgrade before the round is said to stand still, in that node's and the waiting nodes' health (novox/hq ADR 0508 §6).", + "minimum": 1, + "trusted": false, + "default": 120 + } + }, + "own-secrets": { + "broker-keeper": { + "path": "${dir:state}/broker-keeper.secret", + "owner": "pacman-keeper" + } + }, + "state": [ + "installed", + "updates", + "runs", + "records", + "rounds", + "asks" + ], + "emits": [ + "package.changed", + "upgrade.recorded", + "upgrade.kernel-held", + "upgrade.started", + "upgrade.finished", + "upgrade.failed", + "upgrade.checked", + "rollback.started", + "rollback.finished" + ], "tools": [ "pacman_search", "pacman_info", "pacman_installed", + "pacman_packages", "pacman_owns", "pacman_files", "pacman_updates", "pacman_upgrade", "pacman_upgrade_health", + "pacman_upgrades", + "pacman_rollback", + "pacman_round", "pacman_orphans", "pacman_remove_orphans", "pacman_cache", @@ -80,29 +138,89 @@ } }, { - "id": "upgrade-service", - "type": "file", - "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service", - "mode": "0644", - "template": "files/mesh-pacman-weekly-upgrade.service" + "id": "keeper-account", + "type": "user", + "name": "pacman-keeper", + "shell": "/usr/bin/nologin" }, { - "id": "upgrade-timer", - "type": "file", - "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer", - "mode": "0644", - "template": "files/mesh-pacman-weekly-upgrade.timer" + "id": "state", + "type": "directory", + "mode": "0711", + "place": ".", + "unknown": "program" }, { - "id": "weekly-upgrade", - "type": "service", - "unit": "mesh-pacman-weekly-upgrade.timer", - "state": "running", - "boot": "enabled", - "restart-on": [ - "upgrade-service", - "upgrade-timer" + "id": "keeper-config", + "type": "file", + "path": "${dir:state}/keeper.json", + "mode": "0644", + "takes-config": [ + "upgrade-order", + "record-days", + "round-start-minutes" ], + "merge": "json", + "template": "files/keeper.json", + "trusted": true + }, + { + "id": "root-helper", + "type": "process", + "name": "pacman-root", + "artifact": "tools", + "run": [ + "./pacman-tools", + "root" + ], + "env": { + "PACMAN_ROOT_SOCKET": "${dir:state}/root/helper.sock", + "PACMAN_ROOT_PEER": "pacman-keeper" + }, + "health": { + "kind": "unit" + } + }, + { + "id": "keeper", + "type": "process", + "name": "pacman-keeper", + "artifact": "tools", + "run": [ + "./pacman-tools", + "keeper" + ], + "user": "pacman-keeper", + "env": { + "PACMAN_KEEPER_BROKER": "${dir:state}/broker-keeper.secret", + "PACMAN_KEEPER_CONFIG": "${dir:state}/keeper.json", + "PACMAN_ROOT_SOCKET": "${dir:state}/root/helper.sock" + }, + "restart-on": [ + "keeper-config" + ], + "bus": { + "state": [ + "installed", + "updates", + "runs", + "records", + "rounds", + "asks" + ], + "emits": [ + "package.changed", + "upgrade.recorded", + "upgrade.kernel-held", + "upgrade.started", + "upgrade.finished", + "upgrade.failed", + "upgrade.checked", + "rollback.started", + "rollback.finished" + ], + "asks": true + }, "health": { "kind": "tool", "tool": "pacman_upgrade_health", @@ -124,7 +242,10 @@ "binary": "pacman-tools", "loads": [ "pacman-tools" - ] + ], + "env": { + "PACMAN_KEEPER_CONFIG": "${dir:state}/keeper.json" + } }, { "name": "files", diff --git a/predefined.json b/predefined.json index 9e536767..56ba37d0 100644 --- a/predefined.json +++ b/predefined.json @@ -9,5 +9,10 @@ "description": "The size a desktop program shows its text in, in points.", "minimum": 6, "maximum": 72 + }, + "upgrade-order": { + "type": "list", + "description": "The nodes in the order they upgrade, each by its name: the first opens each week's round of upgrades, and each node upgrades only after the one before it passed its after-upgrade check. Empty or unset: no round starts.", + "trusted": true } }