Take the node's package record before a run and read it back, refusing a run whose packages are not current, so a rollback always has a true record to return to (issue 1841/S3)

2026-10-22 14:30:34 +02:00
parent c0bc0d4826
commit 0bb6584437
2 changed files with 335 additions and 0 deletions
@@ -0,0 +1,183 @@
+package main
+
+// The package record taken before every upgrade (novox/hq ADR 0508 §2): a copy of the node's `installed` key,
+// the running kernel, the units failed at that moment and the installed DKMS modules, written to `records` at
+// `<node>.<run>` and read back. A run does not start, and says why, when the node's packages are marked not
+// current or the record cannot be written and read back equal. Records older than `record-days` are deleted.
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// OSRelease is the running kernel's release.
+const OSRelease = "/proc/sys/kernel/osrelease"
+
+// defaultRecordDays is record-days' default in the manifest, for a config file that holds none.
+const defaultRecordDays = 180
+
+// FailedUnits are the units of the service manager failed now, by name.
+func (m *Machine) FailedUnits() ([]string, error) {
+	out, err := m.Out("systemctl", "list-units", "--state=failed", "--plain", "--no-legend", "--no-pager")
+	if err != nil {
+		return nil, err
+	}
+	units := []string{}
+	for _, l := range lines(out) {
+		if f := strings.Fields(l); len(f) > 0 {
+			units = append(units, f[0])
+		}
+	}
+	sort.Strings(units)
+	return units, nil
+}
+
+// ParseDKMSStatus reads `dkms status`: each module at its version and the kernels it is installed for. A module
+// only added (not built for any kernel) is listed with none. Both forms dkms has written are read:
+// `name/version, kernel, arch: state` and the older `name, version, kernel, arch: state`.
+func ParseDKMSStatus(out string) []DKMSModule {
+	byModule := map[string]*DKMSModule{}
+	var order []string
+	for _, l := range lines(out) {
+		head, state, ok := strings.Cut(l, ": ")
+		if !ok {
+			continue
+		}
+		parts := strings.Split(head, ", ")
+		var name, version, kernel string
+		if n, v, slash := strings.Cut(parts[0], "/"); slash {
+			name, version = n, v
+			if len(parts) > 1 {
+				kernel = parts[1]
+			}
+		} else if len(parts) >= 2 {
+			name, version = parts[0], parts[1]
+			if len(parts) > 2 {
+				kernel = parts[2]
+			}
+		} else {
+			continue
+		}
+		key := name + "/" + version
+		m := byModule[key]
+		if m == nil {
+			m = &DKMSModule{Module: name, Version: version, Kernels: []string{}}
+			byModule[key] = m
+			order = append(order, key)
+		}
+		if kernel != "" && strings.HasPrefix(strings.TrimSpace(state), "installed") {
+			m.Kernels = append(m.Kernels, kernel)
+		}
+	}
+	sort.Strings(order)
+	modules := []DKMSModule{}
+	for _, k := range order {
+		sort.Strings(byModule[k].Kernels)
+		modules = append(modules, *byModule[k])
+	}
+	return modules
+}
+
+// DKMSModules are the DKMS modules installed on the machine; none when dkms is not installed.
+func (m *Machine) DKMSModules() ([]DKMSModule, error) {
+	r := m.Run(bg(), "dkms", "status")
+	if r.Err == "ENOENT" {
+		return []DKMSModule{}, nil
+	}
+	if r.Status != 0 || r.Err != "" {
+		return nil, failure("dkms", "dkms", r)
+	}
+	return ParseDKMSStatus(r.Stdout), nil
+}
+
+// TakeRecord takes the node's record for a run, writes it and reads it back. It answers the record, or why
+// it could not be taken: a run that has a why does not start.
+func (u *Upgrades) TakeRecord(run string) (*Record, string) {
+	nc, err := u.Installed.Get(NotCurrentKey(u.Node))
+	if err != nil {
+		return nil, "the node's packages could not be read from the bus: " + err.Error()
+	}
+	if nc != nil {
+		var mark NotCurrent
+		_ = json.Unmarshal(nc.Value, &mark)
+		return nil, fmt.Sprintf("the node's packages on the bus are not current since %s (%d bytes, over the bus's cap on a value): a record of them would not be what is installed",
+			mark.Since.Format(time.RFC3339), mark.Bytes)
+	}
+	e, err := u.Installed.Get(u.Node)
+	if err != nil {
+		return nil, "the node's packages could not be read from the bus: " + err.Error()
+	}
+	if e == nil {
+		return nil, "the node's packages are not on the bus yet: there is nothing to record"
+	}
+	var installed InstalledValue
+	if err := json.Unmarshal(e.Value, &installed); err != nil {
+		return nil, "the node's packages on the bus are not readable: " + err.Error()
+	}
+	kernel, err := u.Machine.ReadFile(OSRelease)
+	if err != nil {
+		return nil, "the running kernel could not be read: " + err.Error()
+	}
+	failed, err := u.Machine.FailedUnits()
+	if err != nil {
+		return nil, "the failed units could not be read: " + err.Error()
+	}
+	dkms, err := u.Machine.DKMSModules()
+	if err != nil {
+		return nil, "the DKMS modules could not be read: " + err.Error()
+	}
+	rec := Record{Node: u.Node, Run: run, At: u.Machine.Now().UTC(), Installed: installed,
+		Kernel: strings.TrimSpace(string(kernel)), FailedUnits: failed, DKMS: dkms}
+	key := NodeKey(u.Node, run)
+	if _, err := u.Records.Put(key, rec); err != nil {
+		return nil, "the record could not be written: " + err.Error()
+	}
+	back, err := u.Records.Get(key)
+	if err != nil {
+		return nil, "the record could not be read back: " + err.Error()
+	}
+	if back == nil {
+		return nil, "the record was written and is not there when read back"
+	}
+	var read Record
+	want, _ := json.Marshal(rec)
+	if err := json.Unmarshal(back.Value, &read); err != nil {
+		return nil, "the record read back is not readable: " + err.Error()
+	}
+	if got, _ := json.Marshal(read); !bytes.Equal(got, want) {
+		return nil, "the record read back differs from the one written"
+	}
+	return &rec, ""
+}
+
+// PruneRecords deletes the node's records older than record-days, judged by the time of their run, and
+// answers how many it deleted.
+func (u *Upgrades) PruneRecords() (int, error) {
+	days := u.Config.RecordDays
+	if days < 1 {
+		days = defaultRecordDays
+	}
+	cutoff := u.Machine.Now().AddDate(0, 0, -days)
+	keys, err := u.Records.Keys()
+	if err != nil {
+		return 0, fmt.Errorf("listing records: %w", err)
+	}
+	n := 0
+	for _, k := range keys {
+		node, id, ok := strings.Cut(k, ".")
+		if !ok || node != u.Node {
+			continue
+		}
+		if t, ok := runTime(id); ok && t.Before(cutoff) {
+			if err := u.Records.Delete(k); err != nil {
+				return n, fmt.Errorf("deleting record %s: %w", k, err)
+			}
+			n++
+		}
+	}
+	return n, nil
+}
@@ -0,0 +1,152 @@
+package main
+
+// The record before every upgrade (novox/hq ADR 0508 §2; acceptance 1841/AC3).
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+)
+
+func capturedDKMSStatus(t *testing.T) string {
+	t.Helper()
+	raw, err := os.ReadFile("testdata/dkms-status-installed-and-added/status")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
+// recordMachine is a machine running one kernel, with one unit failed and the captured DKMS modules.
+func recordMachine(t *testing.T) *Machine {
+	m := machine(byLine(map[string]Ran{
+		"systemctl list-units --state=failed --plain --no-legend --no-pager": {Stdout: "backup.service loaded failed failed The backup\n"},
+		"dkms status": {Stdout: capturedDKMSStatus(t)},
+	}, nil), 1000)
+	m.ReadFile = func(path string) ([]byte, error) {
+		if path == OSRelease {
+			return []byte("7.2.8-arch1-2\n"), nil
+		}
+		return nil, errNoFile
+	}
+	return m
+}
+
+var nodeAPackages = InstalledValue{At: time.Date(2026, 10, 3, 9, 0, 0, 0, time.UTC), Packages: []InstalledPackage{
+	{Name: "linux", Version: "7.2.8.arch1-2", Explicit: true, Repo: "core"},
+	{Name: "zfs-dkms", Version: "2.4.4-1", Explicit: true, Repo: RepoForeign},
+}}
+
+func recordUpgrades(t *testing.T) (*Upgrades, *memBucket, *memBucket) {
+	installed, records := newBucket(), newBucket()
+	if _, err := installed.Put("node-a", nodeAPackages); err != nil {
+		t.Fatal(err)
+	}
+	return &Upgrades{Node: "node-a", Machine: recordMachine(t), Installed: installed, Records: records}, installed, records
+}
+
+func TestDKMSStatusInBothFormsIsEachModuleWithTheKernelsItIsBuiltFor(t *testing.T) {
+	mods := ParseDKMSStatus(capturedDKMSStatus(t) + "zfs, 2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs, 2.4.4, 6.18.54-2-lts, x86_64: installed\n")
+	var got []string
+	for _, m := range mods {
+		got = append(got, m.Module+"/"+m.Version+"="+strings.Join(m.Kernels, "+"))
+	}
+	want := "openrazer-driver/3.12.4=7.2.8-arch1-2 vboxhost/7.2.20_OSE=7.2.8-arch1-2 xone/0.5.8=7.2.8-arch1-2 xpad/0.4= zfs/2.4.4=6.18.54-2-lts+7.2.8-arch1-2"
+	if strings.Join(got, " ") != want {
+		t.Errorf("%v", got)
+	}
+}
+
+func TestARecordIsTheNodesPackagesKernelFailedUnitsAndDKMSWrittenAndReadBack(t *testing.T) {
+	u, _, records := recordUpgrades(t)
+	rec, why := u.TakeRecord("20261004T120000Z")
+	if why != "" || rec == nil {
+		t.Fatalf("refused: %s", why)
+	}
+	var kept Record
+	if !records.value("node-a.20261004T120000Z", &kept) {
+		t.Fatal("no record at <node>.<run>")
+	}
+	a, _ := json.Marshal(kept.Installed)
+	b, _ := json.Marshal(nodeAPackages)
+	if string(a) != string(b) {
+		t.Errorf("the record copies installed.<node>: %s", a)
+	}
+	if kept.Kernel != "7.2.8-arch1-2" || strings.Join(kept.FailedUnits, " ") != "backup.service" || len(kept.DKMS) != 4 ||
+		kept.Node != "node-a" || kept.Run != "20261004T120000Z" {
+		t.Errorf("record: %+v", kept)
+	}
+	if w := strings.Join(records.writes, " "); w != "put node-a.20261004T120000Z" {
+		t.Errorf("writes: %s", w)
+	}
+}
+
+func TestNoRecordAndNoRunWhenThePackagesAreNotCurrentOrTheRecordIsNotReadBack(t *testing.T) {
+	u, installed, records := recordUpgrades(t)
+	if _, err := installed.Put(NotCurrentKey("node-a"), NotCurrent{Bytes: 300000, Since: time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)}); err != nil {
+		t.Fatal(err)
+	}
+	if rec, why := u.TakeRecord("r1"); rec != nil || !strings.Contains(why, "not current") || !strings.Contains(why, "300000") {
+		t.Errorf("not current: %v %q", rec, why)
+	}
+	if len(records.writes) != 0 {
+		t.Errorf("a record of packages not current was written: %v", records.writes)
+	}
+
+	u, installed, _ = recordUpgrades(t)
+	_ = installed.Delete("node-a")
+	if _, why := u.TakeRecord("r1"); !strings.Contains(why, "not on the bus") {
+		t.Errorf("no installed key: %q", why)
+	}
+
+	u, _, records = recordUpgrades(t)
+	records.fail["put node-a.r1"] = errors.New("value over the bucket's limit")
+	if _, why := u.TakeRecord("r1"); !strings.Contains(why, "could not be written") || !strings.Contains(why, "over the bucket's limit") {
+		t.Errorf("not written: %q", why)
+	}
+
+	u, _, _ = recordUpgrades(t)
+	u.Records = &mangling{newBucket()}
+	if _, why := u.TakeRecord("r1"); !strings.Contains(why, "differs") {
+		t.Errorf("read back unequal: %q", why)
+	}
+}
+
+// mangling is a state that reads back other than what was written.
+type mangling struct{ *memBucket }
+
+func (m *mangling) Get(key string) (*stdio.StateEntry, error) {
+	e, err := m.memBucket.Get(key)
+	if e != nil {
+		e.Value = []byte(strings.Replace(string(e.Value), "7.2.8-arch1-2", "7.2.7-arch1-1", 1))
+	}
+	return e, err
+}
+
+func TestRecordsOlderThanRecordDaysAreDeletedAndYoungerOnesKept(t *testing.T) {
+	u, _, records := recordUpgrades(t)
+	u.Config.RecordDays = 180
+	now := u.Machine.Now() // 2026-10-04
+	for _, k := range []string{
+		NodeKey("node-a", RunID(now.AddDate(0, 0, -181))),
+		NodeKey("node-a", RunID(now.AddDate(0, 0, -179))),
+		NodeKey("node-b", RunID(now.AddDate(0, 0, -400))),
+	} {
+		if _, err := records.Put(k, Record{}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	n, err := u.PruneRecords()
+	if err != nil || n != 1 {
+		t.Fatalf("%d deleted, %v", n, err)
+	}
+	keys, _ := records.Keys()
+	if strings.Join(keys, " ") != NodeKey("node-a", RunID(now.AddDate(0, 0, -179)))+" "+NodeKey("node-b", RunID(now.AddDate(0, 0, -400))) {
+		t.Errorf("kept: %v (the younger record, and another node's, which its own keeper prunes)", keys)
+	}
+}