From 0bb6584437e0b0b5711cbd3ffa8c0b462ed28ee3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 14:30:34 +0200 Subject: [PATCH] Take the node's package record before a run and read it back, refusing a run whose packages are not current, so a rollback always has a true record to return to (issue 1841/S3) --- modules/pacman/cmd/pacman-tools/record.go | 183 ++++++++++++++++++ .../pacman/cmd/pacman-tools/record_test.go | 152 +++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 modules/pacman/cmd/pacman-tools/record.go create mode 100644 modules/pacman/cmd/pacman-tools/record_test.go diff --git a/modules/pacman/cmd/pacman-tools/record.go b/modules/pacman/cmd/pacman-tools/record.go new file mode 100644 index 00000000..25570861 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record.go @@ -0,0 +1,183 @@ +package main + +// The package record taken before every upgrade (novox/hq ADR 0508 §2): a copy of the node's `installed` key, +// the running kernel, the units failed at that moment and the installed DKMS modules, written to `records` at +// `.` and read back. A run does not start, and says why, when the node's packages are marked not +// current or the record cannot be written and read back equal. Records older than `record-days` are deleted. + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// OSRelease is the running kernel's release. +const OSRelease = "/proc/sys/kernel/osrelease" + +// defaultRecordDays is record-days' default in the manifest, for a config file that holds none. +const defaultRecordDays = 180 + +// FailedUnits are the units of the service manager failed now, by name. +func (m *Machine) FailedUnits() ([]string, error) { + out, err := m.Out("systemctl", "list-units", "--state=failed", "--plain", "--no-legend", "--no-pager") + if err != nil { + return nil, err + } + units := []string{} + for _, l := range lines(out) { + if f := strings.Fields(l); len(f) > 0 { + units = append(units, f[0]) + } + } + sort.Strings(units) + return units, nil +} + +// ParseDKMSStatus reads `dkms status`: each module at its version and the kernels it is installed for. A module +// only added (not built for any kernel) is listed with none. Both forms dkms has written are read: +// `name/version, kernel, arch: state` and the older `name, version, kernel, arch: state`. +func ParseDKMSStatus(out string) []DKMSModule { + byModule := map[string]*DKMSModule{} + var order []string + for _, l := range lines(out) { + head, state, ok := strings.Cut(l, ": ") + if !ok { + continue + } + parts := strings.Split(head, ", ") + var name, version, kernel string + if n, v, slash := strings.Cut(parts[0], "/"); slash { + name, version = n, v + if len(parts) > 1 { + kernel = parts[1] + } + } else if len(parts) >= 2 { + name, version = parts[0], parts[1] + if len(parts) > 2 { + kernel = parts[2] + } + } else { + continue + } + key := name + "/" + version + m := byModule[key] + if m == nil { + m = &DKMSModule{Module: name, Version: version, Kernels: []string{}} + byModule[key] = m + order = append(order, key) + } + if kernel != "" && strings.HasPrefix(strings.TrimSpace(state), "installed") { + m.Kernels = append(m.Kernels, kernel) + } + } + sort.Strings(order) + modules := []DKMSModule{} + for _, k := range order { + sort.Strings(byModule[k].Kernels) + modules = append(modules, *byModule[k]) + } + return modules +} + +// DKMSModules are the DKMS modules installed on the machine; none when dkms is not installed. +func (m *Machine) DKMSModules() ([]DKMSModule, error) { + r := m.Run(bg(), "dkms", "status") + if r.Err == "ENOENT" { + return []DKMSModule{}, nil + } + if r.Status != 0 || r.Err != "" { + return nil, failure("dkms", "dkms", r) + } + return ParseDKMSStatus(r.Stdout), nil +} + +// TakeRecord takes the node's record for a run, writes it and reads it back. It answers the record, or why +// it could not be taken: a run that has a why does not start. +func (u *Upgrades) TakeRecord(run string) (*Record, string) { + nc, err := u.Installed.Get(NotCurrentKey(u.Node)) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if nc != nil { + var mark NotCurrent + _ = json.Unmarshal(nc.Value, &mark) + return nil, fmt.Sprintf("the node's packages on the bus are not current since %s (%d bytes, over the bus's cap on a value): a record of them would not be what is installed", + mark.Since.Format(time.RFC3339), mark.Bytes) + } + e, err := u.Installed.Get(u.Node) + if err != nil { + return nil, "the node's packages could not be read from the bus: " + err.Error() + } + if e == nil { + return nil, "the node's packages are not on the bus yet: there is nothing to record" + } + var installed InstalledValue + if err := json.Unmarshal(e.Value, &installed); err != nil { + return nil, "the node's packages on the bus are not readable: " + err.Error() + } + kernel, err := u.Machine.ReadFile(OSRelease) + if err != nil { + return nil, "the running kernel could not be read: " + err.Error() + } + failed, err := u.Machine.FailedUnits() + if err != nil { + return nil, "the failed units could not be read: " + err.Error() + } + dkms, err := u.Machine.DKMSModules() + if err != nil { + return nil, "the DKMS modules could not be read: " + err.Error() + } + rec := Record{Node: u.Node, Run: run, At: u.Machine.Now().UTC(), Installed: installed, + Kernel: strings.TrimSpace(string(kernel)), FailedUnits: failed, DKMS: dkms} + key := NodeKey(u.Node, run) + if _, err := u.Records.Put(key, rec); err != nil { + return nil, "the record could not be written: " + err.Error() + } + back, err := u.Records.Get(key) + if err != nil { + return nil, "the record could not be read back: " + err.Error() + } + if back == nil { + return nil, "the record was written and is not there when read back" + } + var read Record + want, _ := json.Marshal(rec) + if err := json.Unmarshal(back.Value, &read); err != nil { + return nil, "the record read back is not readable: " + err.Error() + } + if got, _ := json.Marshal(read); !bytes.Equal(got, want) { + return nil, "the record read back differs from the one written" + } + return &rec, "" +} + +// PruneRecords deletes the node's records older than record-days, judged by the time of their run, and +// answers how many it deleted. +func (u *Upgrades) PruneRecords() (int, error) { + days := u.Config.RecordDays + if days < 1 { + days = defaultRecordDays + } + cutoff := u.Machine.Now().AddDate(0, 0, -days) + keys, err := u.Records.Keys() + if err != nil { + return 0, fmt.Errorf("listing records: %w", err) + } + n := 0 + for _, k := range keys { + node, id, ok := strings.Cut(k, ".") + if !ok || node != u.Node { + continue + } + if t, ok := runTime(id); ok && t.Before(cutoff) { + if err := u.Records.Delete(k); err != nil { + return n, fmt.Errorf("deleting record %s: %w", k, err) + } + n++ + } + } + return n, nil +} diff --git a/modules/pacman/cmd/pacman-tools/record_test.go b/modules/pacman/cmd/pacman-tools/record_test.go new file mode 100644 index 00000000..ac1dca95 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/record_test.go @@ -0,0 +1,152 @@ +package main + +// The record before every upgrade (novox/hq ADR 0508 §2; acceptance 1841/AC3). + +import ( + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func capturedDKMSStatus(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("testdata/dkms-status-installed-and-added/status") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// recordMachine is a machine running one kernel, with one unit failed and the captured DKMS modules. +func recordMachine(t *testing.T) *Machine { + m := machine(byLine(map[string]Ran{ + "systemctl list-units --state=failed --plain --no-legend --no-pager": {Stdout: "backup.service loaded failed failed The backup\n"}, + "dkms status": {Stdout: capturedDKMSStatus(t)}, + }, nil), 1000) + m.ReadFile = func(path string) ([]byte, error) { + if path == OSRelease { + return []byte("7.2.8-arch1-2\n"), nil + } + return nil, errNoFile + } + return m +} + +var nodeAPackages = InstalledValue{At: time.Date(2026, 10, 3, 9, 0, 0, 0, time.UTC), Packages: []InstalledPackage{ + {Name: "linux", Version: "7.2.8.arch1-2", Explicit: true, Repo: "core"}, + {Name: "zfs-dkms", Version: "2.4.4-1", Explicit: true, Repo: RepoForeign}, +}} + +func recordUpgrades(t *testing.T) (*Upgrades, *memBucket, *memBucket) { + installed, records := newBucket(), newBucket() + if _, err := installed.Put("node-a", nodeAPackages); err != nil { + t.Fatal(err) + } + return &Upgrades{Node: "node-a", Machine: recordMachine(t), Installed: installed, Records: records}, installed, records +} + +func TestDKMSStatusInBothFormsIsEachModuleWithTheKernelsItIsBuiltFor(t *testing.T) { + mods := ParseDKMSStatus(capturedDKMSStatus(t) + "zfs, 2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs, 2.4.4, 6.18.54-2-lts, x86_64: installed\n") + var got []string + for _, m := range mods { + got = append(got, m.Module+"/"+m.Version+"="+strings.Join(m.Kernels, "+")) + } + want := "openrazer-driver/3.12.4=7.2.8-arch1-2 vboxhost/7.2.20_OSE=7.2.8-arch1-2 xone/0.5.8=7.2.8-arch1-2 xpad/0.4= zfs/2.4.4=6.18.54-2-lts+7.2.8-arch1-2" + if strings.Join(got, " ") != want { + t.Errorf("%v", got) + } +} + +func TestARecordIsTheNodesPackagesKernelFailedUnitsAndDKMSWrittenAndReadBack(t *testing.T) { + u, _, records := recordUpgrades(t) + rec, why := u.TakeRecord("20261004T120000Z") + if why != "" || rec == nil { + t.Fatalf("refused: %s", why) + } + var kept Record + if !records.value("node-a.20261004T120000Z", &kept) { + t.Fatal("no record at .") + } + a, _ := json.Marshal(kept.Installed) + b, _ := json.Marshal(nodeAPackages) + if string(a) != string(b) { + t.Errorf("the record copies installed.: %s", a) + } + if kept.Kernel != "7.2.8-arch1-2" || strings.Join(kept.FailedUnits, " ") != "backup.service" || len(kept.DKMS) != 4 || + kept.Node != "node-a" || kept.Run != "20261004T120000Z" { + t.Errorf("record: %+v", kept) + } + if w := strings.Join(records.writes, " "); w != "put node-a.20261004T120000Z" { + t.Errorf("writes: %s", w) + } +} + +func TestNoRecordAndNoRunWhenThePackagesAreNotCurrentOrTheRecordIsNotReadBack(t *testing.T) { + u, installed, records := recordUpgrades(t) + if _, err := installed.Put(NotCurrentKey("node-a"), NotCurrent{Bytes: 300000, Since: time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)}); err != nil { + t.Fatal(err) + } + if rec, why := u.TakeRecord("r1"); rec != nil || !strings.Contains(why, "not current") || !strings.Contains(why, "300000") { + t.Errorf("not current: %v %q", rec, why) + } + if len(records.writes) != 0 { + t.Errorf("a record of packages not current was written: %v", records.writes) + } + + u, installed, _ = recordUpgrades(t) + _ = installed.Delete("node-a") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "not on the bus") { + t.Errorf("no installed key: %q", why) + } + + u, _, records = recordUpgrades(t) + records.fail["put node-a.r1"] = errors.New("value over the bucket's limit") + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "could not be written") || !strings.Contains(why, "over the bucket's limit") { + t.Errorf("not written: %q", why) + } + + u, _, _ = recordUpgrades(t) + u.Records = &mangling{newBucket()} + if _, why := u.TakeRecord("r1"); !strings.Contains(why, "differs") { + t.Errorf("read back unequal: %q", why) + } +} + +// mangling is a state that reads back other than what was written. +type mangling struct{ *memBucket } + +func (m *mangling) Get(key string) (*stdio.StateEntry, error) { + e, err := m.memBucket.Get(key) + if e != nil { + e.Value = []byte(strings.Replace(string(e.Value), "7.2.8-arch1-2", "7.2.7-arch1-1", 1)) + } + return e, err +} + +func TestRecordsOlderThanRecordDaysAreDeletedAndYoungerOnesKept(t *testing.T) { + u, _, records := recordUpgrades(t) + u.Config.RecordDays = 180 + now := u.Machine.Now() // 2026-10-04 + for _, k := range []string{ + NodeKey("node-a", RunID(now.AddDate(0, 0, -181))), + NodeKey("node-a", RunID(now.AddDate(0, 0, -179))), + NodeKey("node-b", RunID(now.AddDate(0, 0, -400))), + } { + if _, err := records.Put(k, Record{}); err != nil { + t.Fatal(err) + } + } + n, err := u.PruneRecords() + if err != nil || n != 1 { + t.Fatalf("%d deleted, %v", n, err) + } + keys, _ := records.Keys() + if strings.Join(keys, " ") != NodeKey("node-a", RunID(now.AddDate(0, 0, -179)))+" "+NodeKey("node-b", RunID(now.AddDate(0, 0, -400))) { + t.Errorf("kept: %v (the younger record, and another node's, which its own keeper prunes)", keys) + } +}