diff --git a/modules/pacman/cmd/pacman-tools/acts.go b/modules/pacman/cmd/pacman-tools/acts.go index 146d6d5e..c53b68b9 100644 --- a/modules/pacman/cmd/pacman-tools/acts.go +++ b/modules/pacman/cmd/pacman-tools/acts.go @@ -56,8 +56,8 @@ func (m *Machine) journal(unit string, n int) []string { } // UpgradeArgs is the full system upgrade, in one place: pacman_upgrade starts pacman with these arguments as -// a transient unit, and the weekly upgrade's service (files/mesh-pacman-weekly-upgrade.service) runs -// pacman with the same ones; a test holds the two together (issue 1574, ADR 0477 clause 5). +// a transient unit (issue 1574). The weekly timer that ran the same is gone: a round of runs replaces it +// (ADR 0508 §5). var UpgradeArgs = []string{"-Syu", "--noconfirm"} // Upgrade starts a full system upgrade as a transient unit and answers at once, with the diff --git a/modules/pacman/cmd/pacman-tools/busshapes_test.go b/modules/pacman/cmd/pacman-tools/busshapes_test.go new file mode 100644 index 00000000..a6e9ec17 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/busshapes_test.go @@ -0,0 +1,203 @@ +package main + +// The keeper's bus shapes carry only their named fields (novox/hq ADR 0507 §1, AC7; ADR 0508): every key's value +// and every event's body, filled whole, marshals to exactly the fields named here, and no shape holds a map or a +// free-form value through which anything else could reach the bus. + +import ( + "encoding/json" + "reflect" + "regexp" + "sort" + "strings" + "testing" + "time" +) + +// busFields is every shape's JSON fields, by Go type: the interface the keeper and the verbs share. +var busFields = map[string][]string{ + "InstalledValue": {"at", "packages"}, + "InstalledPackage": {"name", "version", "explicit", "repo"}, + "NotCurrent": {"bytes", "since"}, + "UpdatesValue": {"reachable", "packages"}, + "VersionChange": {"name", "from", "to"}, + "PackageVersion": {"name", "version"}, + "Run": {"node", "run", "kind", "from", "round", "ask", "of", "steps"}, + "Step": {"step", "at", "why", "held", "unit", "ignored", "plan", "exit", "error", "changes", "check"}, + "HeldKernel": {"package", "version", "module", "bound"}, + "Changes": {"added", "removed", "changed"}, + "Check": {"passed", "pacnew", "pacsave", "news", "mapped", "newly_failed", "restart_needed", "running_kernel", + "installed_kernel", "unbuilt"}, + "NewsTitle": {"title", "link", "published"}, + "Mapped": {"unit", "container", "executable"}, + "DKMSBuild": {"module", "version", "kernel"}, + "RollbackItem": {"name", "from", "to", "source", "location"}, + "Record": {"node", "run", "at", "installed", "kernel", "failed_units", "dkms"}, + "DKMSModule": {"module", "version", "kernels"}, + "Round": {"round", "opened", "opened_by", "order"}, + "RoundNode": {"round", "node", "run", "pass"}, + "Pass": {"reason", "ask", "asked", "answer", "answered"}, + "Ask": {"id", "node", "act", "at", "of", "remove_added", "round", "pass", "reason"}, + "PackageChanged": {"node", "at", "added", "removed", "changed"}, + "UpgradeRecorded": {"node", "run", "packages", "kernel"}, + "KernelHeld": {"node", "run", "held"}, + "UpgradeStarted": {"node", "run", "from", "round", "unit", "ignored"}, + "UpgradeFinished": {"node", "run", "changes"}, + "UpgradeFailed": {"node", "run", "step", "exit", "error"}, + "UpgradeChecked": {"node", "run", "kind", "passed"}, + "RollbackStarted": {"node", "run", "of", "unit"}, + "RollbackFinished": {"node", "run", "of", "exit", "error"}, +} + +// The value of each state's keys. +var stateValues = []any{InstalledValue{}, NotCurrent{}, UpdatesValue{}, Run{}, Record{}, Round{}, RoundNode{}, Ask{}} + +var stamp = time.Date(2026, 10, 24, 4, 0, 12, 0, time.UTC) + +// fill sets every field of v, recursively, to a value that is not empty: one element in each list, every pointer +// set, so that every field marshals, omitempty or not. +func fill(v reflect.Value) { + switch v.Kind() { + case reflect.String: + v.SetString("x") + case reflect.Bool: + v.SetBool(true) + case reflect.Int, reflect.Int64: + v.SetInt(1) + case reflect.Pointer: + v.Set(reflect.New(v.Type().Elem())) + fill(v.Elem()) + case reflect.Slice: + s := reflect.MakeSlice(v.Type(), 1, 1) + fill(s.Index(0)) + v.Set(s) + case reflect.Struct: + if v.Type() == reflect.TypeOf(time.Time{}) { + v.Set(reflect.ValueOf(stamp)) + return + } + for i := 0; i < v.NumField(); i++ { + fill(v.Field(i)) + } + } +} + +// fieldsOf walks a filled value against its marshalled JSON: each struct's object holds exactly its named +// fields. It answers the struct types it met. +func fieldsOf(t *testing.T, v reflect.Value, raw json.RawMessage, met map[string]bool) { + t.Helper() + switch v.Kind() { + case reflect.Map, reflect.Interface: + t.Errorf("%s holds a free-form value: a bus shape names its fields", v.Type()) + case reflect.Pointer: + fieldsOf(t, v.Elem(), raw, met) + case reflect.Slice: + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil || len(items) != v.Len() { + t.Errorf("%s: %s is not its list", v.Type(), raw) + return + } + for i := range items { + fieldsOf(t, v.Index(i), items[i], met) + } + case reflect.Struct: + if v.Type() == reflect.TypeOf(time.Time{}) { + return + } + name := v.Type().Name() + met[name] = true + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + t.Errorf("%s: %s is not an object", name, raw) + return + } + var got []string + for k := range obj { + got = append(got, k) + } + want, named := busFields[name] + if !named { + t.Errorf("%s is on the bus and its fields are not named in busFields", name) + return + } + want = append([]string(nil), want...) + sort.Strings(got) + sort.Strings(want) + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("%s marshals to %v, its named fields are %v", name, got, want) + } + for i := 0; i < v.NumField(); i++ { + tag, _, _ := strings.Cut(v.Type().Field(i).Tag.Get("json"), ",") + if r, ok := obj[tag]; ok { + fieldsOf(t, v.Field(i), r, met) + } + } + } +} + +func TestEveryStateValueAndEventBodyCarriesOnlyItsNamedFields(t *testing.T) { + roots := append([]any(nil), stateValues...) + for _, e := range Events { + roots = append(roots, e.Body) + } + met := map[string]bool{} + for _, root := range roots { + v := reflect.New(reflect.TypeOf(root)).Elem() + fill(v) + raw, err := json.Marshal(v.Interface()) + if err != nil { + t.Fatal(err) + } + fieldsOf(t, v, raw, met) + } + for name := range busFields { + if !met[name] { + t.Errorf("busFields names %s, which no state value or event body holds", name) + } + } +} + +func TestEveryEventIsNamedOnceWithItsBody(t *testing.T) { + seen := map[string]bool{} + for _, e := range Events { + if seen[e.Name] { + t.Errorf("%s twice", e.Name) + } + seen[e.Name] = true + if e.Body == nil || reflect.TypeOf(e.Body).Kind() != reflect.Struct { + t.Errorf("%s has no body type", e.Name) + } + } + if len(Events) != 9 { + t.Errorf("package.changed and the eight upgrade and rollback events, not %d", len(Events)) + } +} + +// A key is node and id tokens joined by dots; each token is one NATS key token. +func TestKeysAreTokensJoinedByDots(t *testing.T) { + token := regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + run := RunID(time.Date(2026, 10, 24, 6, 0, 12, 0, time.FixedZone("CEST", 2*3600))) + if run != "20261024T040012Z" { + t.Errorf("RunID: %s", run) + } + round := RoundID(time.Date(2026, 10, 24, 4, 0, 0, 0, time.Local)) + if round != "2026-W43" { + t.Errorf("RoundID: %s", round) + } + if RoundID(time.Date(2027, 1, 2, 4, 0, 0, 0, time.UTC)) != "2026-W53" { + t.Errorf("the ISO week of 2 January 2027 is 2026's 53rd: %s", RoundID(time.Date(2027, 1, 2, 4, 0, 0, 0, time.UTC))) + } + for key, parts := range map[string]int{ + NodeKey("node-a", run): 2, NotCurrentKey("node-a"): 2, RoundNodeKey(round, "node-b"): 2, + } { + ts := strings.Split(key, ".") + if len(ts) != parts { + t.Errorf("%s: %d tokens", key, len(ts)) + } + for _, x := range ts { + if !token.MatchString(x) { + t.Errorf("%s: token %q", key, x) + } + } + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go new file mode 100644 index 00000000..caaff26f --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go @@ -0,0 +1,201 @@ +package main + +// The manifest of the keeper (novox/hq ADR 0507 §1-§3, §5; ADR 0508 §1, §2, §5, §6, §8; acceptance 1841/AC12): +// the module's background work is a process of its own under an account the module makes, with a bus account of +// its own, the module's six states and the nine events it emits; the module's config values land in the +// keeper's config file, which restarts it; the weekly timer and its service are gone (a dropped file or service +// is removed by the node-engine); `upgrade-order` falls back to the mesh-level key of that name. + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// rawManifest is the manifest as written, every field kept. +func rawManifest(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func strs(v any) []string { + var out []string + l, _ := v.([]any) + for _, x := range l { + s, _ := x.(string) + out = append(out, s) + } + return out +} + +func TestTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { + m := manifest(t) + for _, r := range m.Resources { + id, _ := r["id"].(string) + path, _ := r["path"].(string) + unit, _ := r["unit"].(string) + if id == "weekly-upgrade" || id == "upgrade-timer" || id == "upgrade-service" || + strings.Contains(path, "weekly-upgrade") || strings.Contains(unit, "weekly-upgrade") { + t.Errorf("the weekly upgrade is replaced by the round (ADR 0508 §5), and %v still declares it", id) + } + } + for _, name := range []string{"mesh-pacman-weekly-upgrade.timer", "mesh-pacman-weekly-upgrade.service"} { + if _, err := os.Stat(filepath.Join(moduleDir, "files", name)); err == nil { + t.Errorf("files/%s is still shipped", name) + } + } +} + +func TestTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn(t *testing.T) { + m := manifest(t) + user := m.resource(t, "keeper-account") + if user["type"] != "user" || user["name"] != "pacman-keeper" || user["shell"] != "/usr/bin/nologin" { + t.Fatalf("the keeper's account: %v", user) + } + k := m.resource(t, "keeper") + if k["type"] != "process" || k["name"] != "pacman-keeper" || k["artifact"] != "tools" || + strings.Join(strs(k["run"]), " ") != "./"+binaryName+" keeper" { + t.Fatalf("the keeper: %v", k) + } + if k["user"] != "pacman-keeper" { + t.Errorf("the keeper runs as its own account, never root (ADR 0508 §8): user %v", k["user"]) + } + env, _ := k["env"].(map[string]any) + if len(env) != 2 || env[EnvBroker] != "${dir:state}/broker-keeper.secret" || env[EnvConfig] != "${dir:state}/keeper.json" { + t.Errorf("the keeper's environment names its account file and its config file: %v", env) + } + secrets, _ := rawManifest(t)["own-secrets"].(map[string]any) + broker, _ := secrets["broker-keeper"].(map[string]any) + if broker["path"] != env[EnvBroker] || broker["owner"] != "pacman-keeper" || len(secrets) != 1 { + t.Errorf("the bus account is the keeper's own secret, owned by its account: %v", secrets) + } + state := m.resource(t, "state") + if state["type"] != "directory" || state["place"] != "." || state["unknown"] == nil { + t.Errorf("the state directory: %v", state) + } + order := map[string]int{} + for i, r := range m.Resources { + order[r["id"].(string)] = i + } + if order["keeper-account"] > order["keeper"] || order["state"] > order["keeper-config"] || order["keeper-config"] > order["keeper"] { + t.Errorf("the account, the directory and the config file come before the process: %v", order) + } +} + +func TestTheKeeperDeclaresTheModulesStatesAndEvents(t *testing.T) { + raw := rawManifest(t) + k := manifest(t).resource(t, "keeper") + bus, _ := k["bus"].(map[string]any) + var events []string + for _, e := range Events { + events = append(events, e.Name) + } + if got := strs(bus["state"]); !reflect.DeepEqual(got, States) { + t.Errorf("the keeper's states: %v, want %v", got, States) + } + if got := strs(bus["emits"]); !reflect.DeepEqual(got, events) { + t.Errorf("the keeper's events: %v, want %v", got, events) + } + for key := range bus { + if key != "state" && key != "emits" { + t.Errorf("the keeper's bus declares %s: it keeps its own states and emits its events, nothing else yet", key) + } + } + if got := strs(raw["state"]); !reflect.DeepEqual(got, States) { + t.Errorf("the module's states: %v, want %v", got, States) + } + if got := strs(raw["emits"]); !reflect.DeepEqual(got, events) { + t.Errorf("the module's events: %v, want %v", got, events) + } + // The verbs read which packages each node's modules declare; the process reads no other module's state + // (ADR 0507 §5, ADR 0436 §1). + if got := strs(raw["reads"]); !reflect.DeepEqual(got, []string{"mesh-controller.declared-packages"}) { + t.Errorf("reads: %v", got) + } +} + +// configValue is the module's config value of that key. +func configValue(t *testing.T, key string) map[string]any { + t.Helper() + config, _ := rawManifest(t)["config"].(map[string]any) + v, _ := config[key].(map[string]any) + if v == nil { + t.Fatalf("no config value %s", key) + } + if d, _ := v["description"].(string); len(d) < 40 { + t.Errorf("%s says what it means: %q", key, d) + } + return v +} + +func TestTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt(t *testing.T) { + order := configValue(t, "upgrade-order") + if order["type"] != "list" || order["trusted"] != true || order["predefined"] != "upgrade-order" || + !reflect.DeepEqual(order["default"], []any{}) || order["required"] != nil { + t.Errorf("upgrade-order is a trusted list falling back to the mesh-level key, empty by default: %v", order) + } + for key, def := range map[string]float64{"record-days": 180, "round-start-minutes": 120} { + v := configValue(t, key) + if v["type"] != "number" || v["default"] != def || v["minimum"] != float64(1) { + t.Errorf("%s is a number of at least 1, %v by default: %v", key, def, v) + } + } + config, _ := rawManifest(t)["config"].(map[string]any) + if len(config) != 3 { + t.Errorf("config: %v", config) + } + m := manifest(t) + f := m.resource(t, "keeper-config") + if f["type"] != "file" || f["path"] != "${dir:state}/keeper.json" || f["merge"] != "json" || f["trusted"] != true || f["mode"] != "0644" { + t.Fatalf("the keeper's config file: %v", f) + } + if got := strs(f["takes-config"]); strings.Join(got, " ") != "upgrade-order record-days round-start-minutes" { + t.Errorf("the file takes every config value: %v", got) + } + if got := strs(m.resource(t, "keeper")["restart-on"]); strings.Join(got, " ") != "keeper-config" { + t.Errorf("a changed config file restarts the keeper, which reads it at its start: %v", got) + } + // The template is the file a node starts with: the defaults, read as the keeper reads them. + dec := json.NewDecoder(bytes.NewReader([]byte(templateText(t, f)))) + dec.DisallowUnknownFields() + var c KeeperConfig + if err := dec.Decode(&c); err != nil { + t.Fatalf("the keeper cannot read its config file: %v", err) + } + if c.UpgradeOrder == nil || len(c.UpgradeOrder) != 0 || c.RecordDays != 180 || c.RoundStartMinutes != 120 { + t.Errorf("the template holds the defaults: %+v", c) + } +} + +func TestUpgradeOrderIsATrustedListOfTheMesh(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "..", "..", "predefined.json")) + if err != nil { + t.Fatal(err) + } + var keys map[string]map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + k := keys["upgrade-order"] + if k["type"] != "list" || k["trusted"] != true { + t.Fatalf("predefined upgrade-order: %v", k) + } + if d, _ := k["description"].(string); !strings.Contains(d, "order") || !strings.Contains(d, "round") { + t.Errorf("its description says the nodes' order and the round: %q", d) + } + if k["type"] != configValue(t, "upgrade-order")["type"] { + t.Error("a value falls back only to a key of its own type (ADR 0348 §4)") + } +} diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index da7fb4ad..b55f6908 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -101,53 +101,3 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { t.Fatalf("a package's file and timer come after the package: %v", order) } } - -// The weekly upgrade (issue 1574, ADR 0477 clause 5): a timer on Saturday 04:00 local time, persistent so a -// node that was off upgrades at its next start, the same moment on every node (no random delay), starting a -// oneshot service that runs the very upgrade pacman_upgrade runs. -func TestTheWeeklyUpgradeIsATimerOnSaturdayFourPersistentRunningTheToolsUpgrade(t *testing.T) { - m := manifest(t) - timer := m.resource(t, "upgrade-timer") - service := m.resource(t, "upgrade-service") - if timer["path"] != "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer" || service["path"] != "/etc/systemd/system/"+WeeklyUnit { - t.Fatalf("paths: %v %v", timer["path"], service["path"]) - } - tt, st := templateText(t, timer), templateText(t, service) - for _, want := range []string{"\nOnCalendar=Sat *-*-* 04:00:00\n", "\nPersistent=true\n", "\nWantedBy=timers.target\n"} { - if !strings.Contains(tt, want) { - t.Errorf("the timer lacks %q:\n%s", strings.TrimSpace(want), tt) - } - } - for _, not := range []string{"RandomizedDelaySec", "OnBootSec", "OnUnitActiveSec", "UTC"} { - if strings.Contains(tt, not) { - t.Errorf("the timer says %s: the same local moment on every node, and no other moment", not) - } - } - if strings.Count(tt, "OnCalendar=") != 1 { - t.Errorf("one moment only:\n%s", tt) - } - if !strings.Contains(st, "\nType=oneshot\n") { - t.Errorf("the service is a oneshot:\n%s", st) - } - if want := "\nExecStart=/usr/bin/pacman " + strings.Join(UpgradeArgs, " ") + "\n"; !strings.Contains(st, want) { - t.Errorf("the service does not run the upgrade pacman_upgrade runs (%q):\n%s", want, st) - } - if strings.Contains(st, "reboot") || strings.Contains(tt, "reboot") || strings.Contains(st, "systemctl") { - t.Error("the mesh restarts no machine for an upgrade") - } - r := m.resource(t, "weekly-upgrade") - if r["type"] != "service" || r["unit"] != "mesh-pacman-weekly-upgrade.timer" || r["state"] != "running" || r["boot"] != "enabled" { - t.Fatalf("the timer runs and is enabled: %v", r) - } - h, _ := r["health"].(map[string]any) - if h["kind"] != "tool" || h["tool"] != "pacman_upgrade_health" { - t.Errorf("the upgrade's outcome is the resource's health: %v", r["health"]) - } - order := map[string]int{} - for i, x := range m.Resources { - order[x["id"].(string)] = i - } - if order["upgrade-service"] > order["weekly-upgrade"] || order["upgrade-timer"] > order["weekly-upgrade"] || order["package"] > order["upgrade-service"] { - t.Errorf("the units are placed before the timer is started: %v", order) - } -} diff --git a/modules/pacman/cmd/pacman-tools/shapes.go b/modules/pacman/cmd/pacman-tools/shapes.go new file mode 100644 index 00000000..ac07c003 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/shapes.go @@ -0,0 +1,468 @@ +package main + +// The keeper's bus shapes (novox/hq ADR 0507, ADR 0508): the value of every key of the module's states and the +// body of every event its process emits, in one place, so the process that writes them and the verbs that read +// them agree by type. Each is a struct of named fields only: no map and no free-form value, so nothing but what +// is named here goes on the bus (ADR 0507 §1: package names, versions, unit and container names, file paths and +// exit statuses, never a process's command line or environment). A time in a value is the time of the thing it +// describes (a transaction, a step, an opening), never the time it was read: a key is written only when what it +// describes changed (ADR 0289 §1), and no field changes on a read alone (ADR 0289 §4). When a key last changed +// is the bus's to say (its revision), not a field's. +// +// A key is `` or `.`: a node name and an id are each one key token ([A-Za-z0-9_-]), so the +// dot only ever separates them. + +import ( + "fmt" + "time" +) + +// The module's states (manifest `state` and the keeper's `bus` `state`). +const ( + // StateInstalled holds each node's installed packages: key `` an InstalledValue, and key + // `.not-current` a NotCurrent while the node's packages are over the bus's cap (ADR 0507 §2). + StateInstalled = "installed" + // StateUpdates holds what a full upgrade would change: key `` an UpdatesValue (ADR 0507 §4). + StateUpdates = "updates" + // StateRuns holds every upgrade and rollback: key `.` a Run, kept (ADR 0508 §1). + StateRuns = "runs" + // StateRecords holds the package record taken before a run: key `.` a Record, deleted after + // the config value `record-days` (ADR 0508 §2). + StateRecords = "records" + // StateRounds holds the weekly rounds: key `` a Round, created once by the first node of the order, + // and key `.` that node's RoundNode (ADR 0508 §5, §6). + StateRounds = "rounds" + // StateAsks holds the acts asked of a node: key `.` an Ask, taken by that node's keeper alone + // (ADR 0508 §8). + StateAsks = "asks" +) + +// States are the module's states, in the manifest's order. +var States = []string{StateInstalled, StateUpdates, StateRuns, StateRecords, StateRounds, StateAsks} + +// The events the keeper emits (the keeper's `bus` `emits`, and the manifest's `emits`). +const ( + EventPackageChanged = "package.changed" // PackageChanged (ADR 0507 §3) + EventRecorded = "upgrade.recorded" // UpgradeRecorded (ADR 0508 §1, §2) + EventKernelHeld = "upgrade.kernel-held" // KernelHeld (ADR 0508 §3) + EventStarted = "upgrade.started" // UpgradeStarted + EventFinished = "upgrade.finished" // UpgradeFinished + EventFailed = "upgrade.failed" // UpgradeFailed + EventChecked = "upgrade.checked" // UpgradeChecked (ADR 0508 §4) + EventRollbackStarted = "rollback.started" // RollbackStarted (ADR 0508 §7) + EventRollbackFinished = "rollback.finished" // RollbackFinished +) + +// Events are the events the keeper emits, each with the type of its body. +var Events = []struct { + Name string + Body any +}{ + {EventPackageChanged, PackageChanged{}}, + {EventRecorded, UpgradeRecorded{}}, + {EventKernelHeld, KernelHeld{}}, + {EventStarted, UpgradeStarted{}}, + {EventFinished, UpgradeFinished{}}, + {EventFailed, UpgradeFailed{}}, + {EventChecked, UpgradeChecked{}}, + {EventRollbackStarted, RollbackStarted{}}, + {EventRollbackFinished, RollbackFinished{}}, +} + +// Key names. + +// NotCurrentKey is the key in `installed` that marks a node's packages as not current. +func NotCurrentKey(node string) string { return node + ".not-current" } + +// NodeKey is a key `.`: a run in `runs` or `records`, an ask in `asks`. +func NodeKey(node, id string) string { return node + "." + id } + +// RoundNodeKey is a node's key in a round: `.` in `rounds`. +func RoundNodeKey(round, node string) string { return round + "." + node } + +// RunID names a run by the moment it began, in UTC: `20261024T040012Z`. A run read from the package manager's +// log is named by the log's time of its start the same way. +func RunID(began time.Time) string { return began.UTC().Format("20060102T150405Z") } + +// RoundID names the round of the ISO week that holds the given moment, in its own time zone: `2026-W43`. The +// round opens on Saturday 04:00 local time, so the local date decides the week. +func RoundID(at time.Time) string { + year, week := at.ISOWeek() + return fmt.Sprintf("%04d-W%02d", year, week) +} + +// installed + +// InstalledValue is `installed.`: every installed package, sorted by name, and the time of the transaction +// that left the database so (ADR 0507 §2). Written compactly: one short entry per package. +type InstalledValue struct { + At time.Time `json:"at"` + Packages []InstalledPackage `json:"packages"` +} + +// InstalledPackage is one installed package: its version, whether it was installed explicitly (else as a +// dependency), and its repository, or RepoForeign when no repository holds it (ADR 0507 §7). +type InstalledPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Explicit bool `json:"explicit"` + Repo string `json:"repo"` +} + +// RepoForeign marks a package from outside the repositories. +const RepoForeign = "foreign" + +// NotCurrent is `installed..not-current`: the node's packages came to more than the bus's cap on a value, +// so `installed.` keeps the last value written. Written once, when it first happens, and deleted at the +// next successful write of `installed.`. +type NotCurrent struct { + Bytes int `json:"bytes"` + Since time.Time `json:"since"` +} + +// updates + +// UpdatesValue is `updates.`: every package a full upgrade would change, sorted by name, and whether the +// node reached its mirrors (ADR 0507 §4). A failed read writes Reachable false and keeps the last list, never an +// empty list in its place. +type UpdatesValue struct { + Reachable bool `json:"reachable"` + Packages []VersionChange `json:"packages"` +} + +// VersionChange is one package going from one version to another. +type VersionChange struct { + Name string `json:"name"` + From string `json:"from"` + To string `json:"to"` +} + +// PackageVersion is one package at one version. +type PackageVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// runs + +// The kinds of run. +const ( + RunUpgrade = "upgrade" + RunRollback = "rollback" +) + +// What started a run. +const ( + FromRound = "round" // its turn in the week's round + FromAsk = "ask" // an ask: pacman_upgrade, or pacman_rollback with apply + FromLog = "log" // read from the package manager's log at the keeper's first start (ADR 0508 §1) +) + +// The steps of a run, in the order they are written. A run that does not start ends at StepRefused; a +// transaction ends at StepFinished or StepFailed, and the after-upgrade check follows either. +const ( + StepRecorded = "recorded" // the record is written and read back (upgrade only) + StepCheckedBefore = "checked-before" // the DKMS gate ran; Held names what it held + StepRefused = "refused" // the run did not start; Why says why + StepStarted = "started" // the transaction's unit started + StepFinished = "finished" // the transaction exited 0 + StepFailed = "failed" // the transaction failed + StepCheckedAfter = "checked-after" // the after-upgrade check +) + +// Run is `runs..`: one upgrade or rollback of a node, written at each step and at no other time +// (ADR 0508 §1). Round names the round it ran in; Ask the ask that started it (the keeper deletes the ask once +// the run names it); Of, for a rollback, the run whose record it returns to. +type Run struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Ask string `json:"ask,omitempty"` + Of string `json:"of,omitempty"` + Steps []Step `json:"steps"` +} + +// Step is one step of a run with its time and outcome; only the fields of its kind are set. +type Step struct { + Step string `json:"step"` + At time.Time `json:"at"` + // StepRefused: why the run did not start. + Why string `json:"why,omitempty"` + // StepCheckedBefore: the kernels held from this run. + Held []HeldKernel `json:"held,omitempty"` + // StepStarted: the transaction's unit, the packages left out of it (held kernels and their headers), and + // for a rollback its plan. + Unit string `json:"unit,omitempty"` + Ignored []string `json:"ignored,omitempty"` + Plan []RollbackItem `json:"plan,omitempty"` + // StepFinished and StepFailed: the exit status, the last error line, and what the transaction changed, + // read from its lines in the package manager's log. + Exit *int `json:"exit,omitempty"` + Error string `json:"error,omitempty"` + Changes *Changes `json:"changes,omitempty"` + // StepCheckedAfter: the after-upgrade check. + Check *Check `json:"check,omitempty"` +} + +// HeldKernel is a kernel package left out of a run because an installed DKMS module's own bound excludes its +// new version (ADR 0508 §3): `Linux-Maximum` in the module's META, or `BUILD_EXCLUSIVE_KERNEL` in its dkms.conf. +type HeldKernel struct { + Package string `json:"package"` + Version string `json:"version"` + Module string `json:"module"` + Bound string `json:"bound"` +} + +// Changes is what a transaction changed, each list sorted by name. +type Changes struct { + Added []PackageVersion `json:"added"` + Removed []PackageVersion `json:"removed"` + Changed []VersionChange `json:"changed"` +} + +// Check is the after-upgrade check (ADR 0508 §4). Passed when the transaction succeeded, no unit newly failed +// and every DKMS module is built for every installed kernel; the rest is said and does not stop the round. +type Check struct { + Passed bool `json:"passed"` + // The .pacnew and .pacsave files the transaction left, by path. + Pacnew []string `json:"pacnew"` + Pacsave []string `json:"pacsave"` + // The distribution's news since the node's previous upgrade. + News []NewsTitle `json:"news"` + // The units and containers whose processes still map a file the upgrade replaced. + Mapped []Mapped `json:"mapped"` + // The units failed now that were not failed in the record. + NewlyFailed []string `json:"newly_failed"` + // Whether a restart is needed: the running kernel is not the installed version of its package. + RestartNeeded bool `json:"restart_needed"` + RunningKernel string `json:"running_kernel"` + InstalledKernel string `json:"installed_kernel"` + // The DKMS modules not built for an installed kernel. + Unbuilt []DKMSBuild `json:"unbuilt"` +} + +// NewsTitle is one item of the distribution's news, without its text. +type NewsTitle struct { + Title string `json:"title"` + Link string `json:"link"` + Published string `json:"published"` +} + +// Mapped is a process still mapping a replaced file, named by its unit or its container and its executable's +// path; never its command line or environment. +type Mapped struct { + Unit string `json:"unit,omitempty"` + Container string `json:"container,omitempty"` + Executable string `json:"executable"` +} + +// DKMSBuild is one DKMS module at one version for one kernel. +type DKMSBuild struct { + Module string `json:"module"` + Version string `json:"version"` + Kernel string `json:"kernel"` +} + +// RollbackItem is one package a rollback brings back (ADR 0508 §7): from its version now to the record's, and +// where that version comes from. +type RollbackItem struct { + Name string `json:"name"` + From string `json:"from"` + To string `json:"to"` + Source string `json:"source"` + Location string `json:"location"` +} + +// Where a rollback takes a package's version from, in the order it is looked for. +const ( + SourceCache = "cache" // the node's package cache: a file path + SourceArchive = "archive" // the Arch Linux Archive's copy of that exact version: a URL + SourceRepository = "repository" // the archive's repository as it stood on the record's date: a URL +) + +// records + +// Record is `records..`: the node's packages taken before a run and read back (ADR 0508 §2): a copy of +// `installed.`, the running kernel (/proc/sys/kernel/osrelease), the units failed at that moment, and the +// installed DKMS modules. +type Record struct { + Node string `json:"node"` + Run string `json:"run"` + At time.Time `json:"at"` + Installed InstalledValue `json:"installed"` + Kernel string `json:"kernel"` + FailedUnits []string `json:"failed_units"` + DKMS []DKMSModule `json:"dkms"` +} + +// DKMSModule is one installed DKMS module and the kernels it is built for. +type DKMSModule struct { + Module string `json:"module"` + Version string `json:"version"` + Kernels []string `json:"kernels"` +} + +// rounds + +// Round is `rounds.`: the week's round, created once (a create, so one round per week) by the first node +// of the order when it opens it, with the order as it stood then (ADR 0508 §5). +type Round struct { + Round string `json:"round"` + Opened time.Time `json:"opened"` + OpenedBy string `json:"opened_by"` + Order []string `json:"order"` +} + +// RoundNode is `rounds..`: the node's place in the round. Run is the run it started in its turn, +// written by the node itself; Pass, the operator's pass of a node that stopped the round, written by the keeper +// of the node waiting on it (ADR 0508 §6). Both write through a compare-and-set. +type RoundNode struct { + Round string `json:"round"` + Node string `json:"node"` + Run string `json:"run,omitempty"` + Pass *Pass `json:"pass,omitempty"` +} + +// Pass is a pass of a node asked of the operator through the operator's channel (ADR 0259): the reason, the +// channel's ask, and its answer once given (Answer empty while it waits). +type Pass struct { + Reason string `json:"reason"` + Ask string `json:"ask"` + Asked time.Time `json:"asked"` + Answer string `json:"answer,omitempty"` + Answered *time.Time `json:"answered,omitempty"` +} + +// The operator's answers to a pass. +const ( + PassApproved = "approved" + PassDeclined = "declined" +) + +// asks + +// The acts an ask asks. +const ( + ActUpgrade = "upgrade" // pacman_upgrade + ActRollback = "rollback" // pacman_rollback with apply + ActPass = "pass" // pacman_round with pass +) + +// Ask is `asks..`: an act asked of a node, taken by that node's keeper alone, which answers in `runs` +// (an upgrade, a rollback) or in `rounds` (a pass) and then deletes the ask (ADR 0508 §8). A rollback names the +// run whose record it returns to and whether the packages the upgrade added are removed; a pass names the +// round, the node passed and the reason, and is asked of the node waiting on it. +type Ask struct { + ID string `json:"id"` + Node string `json:"node"` + Act string `json:"act"` + At time.Time `json:"at"` + Of string `json:"of,omitempty"` + RemoveAdded bool `json:"remove_added,omitempty"` + Round string `json:"round,omitempty"` + Pass string `json:"pass,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// events + +// PackageChanged is `package.changed`: a write of `installed.` that changed it (ADR 0507 §3). +type PackageChanged struct { + Node string `json:"node"` + At time.Time `json:"at"` + Added []PackageVersion `json:"added"` + Removed []PackageVersion `json:"removed"` + Changed []VersionChange `json:"changed"` +} + +// UpgradeRecorded is `upgrade.recorded`: the run's record is written and read back. +type UpgradeRecorded struct { + Node string `json:"node"` + Run string `json:"run"` + Packages int `json:"packages"` + Kernel string `json:"kernel"` +} + +// KernelHeld is `upgrade.kernel-held`: kernels left out of a run, each with the module and bound that hold it. +type KernelHeld struct { + Node string `json:"node"` + Run string `json:"run"` + Held []HeldKernel `json:"held"` +} + +// UpgradeStarted is `upgrade.started`: a run's transaction started. +type UpgradeStarted struct { + Node string `json:"node"` + Run string `json:"run"` + From string `json:"from"` + Round string `json:"round,omitempty"` + Unit string `json:"unit"` + Ignored []string `json:"ignored"` +} + +// UpgradeFinished is `upgrade.finished`: a run's transaction exited 0, with what it changed. +type UpgradeFinished struct { + Node string `json:"node"` + Run string `json:"run"` + Changes Changes `json:"changes"` +} + +// UpgradeFailed is `upgrade.failed`: a run that did not start (Step StepRefused, with why in Error) or whose +// transaction failed (Step StepFailed, with its exit status and last error line). +type UpgradeFailed struct { + Node string `json:"node"` + Run string `json:"run"` + Step string `json:"step"` + Exit *int `json:"exit,omitempty"` + Error string `json:"error"` +} + +// UpgradeChecked is `upgrade.checked`: the after-upgrade check of a run, an upgrade's or a rollback's; the +// check whole is in the run. +type UpgradeChecked struct { + Node string `json:"node"` + Run string `json:"run"` + Kind string `json:"kind"` + Passed bool `json:"passed"` +} + +// RollbackStarted is `rollback.started`: a rollback's transaction started, returning to the record of run Of. +type RollbackStarted struct { + Node string `json:"node"` + Run string `json:"run"` + Of string `json:"of"` + Unit string `json:"unit"` +} + +// RollbackFinished is `rollback.finished`: a rollback's transaction ended, with its exit status and, when it +// failed, its last error line. +type RollbackFinished struct { + Node string `json:"node"` + Run string `json:"run"` + Of string `json:"of"` + Exit int `json:"exit"` + Error string `json:"error,omitempty"` +} + +// The keeper's config file + +// The keeper's environment, named by the manifest's process resource. +const ( + EnvBroker = "PACMAN_KEEPER_BROKER" // the keeper's bus account file + EnvConfig = "PACMAN_KEEPER_CONFIG" // the keeper's config file, a KeeperConfig +) + +// KeeperConfig is the keeper's config file: the module's config values the mesh merges into it (ADR 0348 §5). +// The process reads it at its start; the mesh restarts the process when it changes (the manifest's +// `restart-on`). +type KeeperConfig struct { + // UpgradeOrder is the nodes in the order they upgrade; the first opens each week's round. Empty: no round. + UpgradeOrder []string `json:"upgrade-order"` + // RecordDays is how long a record is kept, in days. + RecordDays int `json:"record-days"` + // RoundStartMinutes is how long a node whose turn came may wait before starting, in minutes, before the + // round is said to stand still. + RoundStartMinutes int `json:"round-start-minutes"` +} diff --git a/modules/pacman/files/keeper.json b/modules/pacman/files/keeper.json new file mode 100644 index 00000000..8e340a94 --- /dev/null +++ b/modules/pacman/files/keeper.json @@ -0,0 +1,5 @@ +{ + "upgrade-order": [], + "record-days": 180, + "round-start-minutes": 120 +} diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/modules/pacman/files/mesh-pacman-weekly-upgrade.service deleted file mode 100644 index 69741dd7..00000000 --- a/modules/pacman/files/mesh-pacman-weekly-upgrade.service +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. -[Unit] -Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) -Wants=network-online.target -After=network-online.target - -[Service] -# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health -# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. -Type=oneshot -Nice=10 -ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer deleted file mode 100644 index a8a9b4d9..00000000 --- a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer +++ /dev/null @@ -1,11 +0,0 @@ -# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. -[Unit] -Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) - -[Timer] -# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. -OnCalendar=Sat *-*-* 04:00:00 -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/modules/pacman/module.json b/modules/pacman/module.json index ca1a81a3..4e3d0084 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -11,6 +11,57 @@ "scope": "node" } ], + "reads": [ + "mesh-controller.declared-packages" + ], + "config": { + "upgrade-order": { + "type": "list", + "description": "The nodes in the order they upgrade, each by its name: the first opens each week's round on Saturday at 04:00 local time, and each node upgrades after the one before it passed its after-upgrade check (novox/hq ADR 0508 §5). Empty: no round starts, and every node's health says why. Falls back to the mesh-level key of the same name.", + "trusted": true, + "predefined": "upgrade-order", + "default": [] + }, + "record-days": { + "type": "number", + "description": "How many days a package record taken before an upgrade is kept on the bus before it is deleted (novox/hq ADR 0508 §2); a rollback can return only to a record still kept.", + "minimum": 1, + "trusted": false, + "default": 180 + }, + "round-start-minutes": { + "type": "number", + "description": "How many minutes a node whose turn in the round came may take to start its upgrade before the round is said to stand still, in that node's and the waiting nodes' health (novox/hq ADR 0508 §6).", + "minimum": 1, + "trusted": false, + "default": 120 + } + }, + "own-secrets": { + "broker-keeper": { + "path": "${dir:state}/broker-keeper.secret", + "owner": "pacman-keeper" + } + }, + "state": [ + "installed", + "updates", + "runs", + "records", + "rounds", + "asks" + ], + "emits": [ + "package.changed", + "upgrade.recorded", + "upgrade.kernel-held", + "upgrade.started", + "upgrade.finished", + "upgrade.failed", + "upgrade.checked", + "rollback.started", + "rollback.finished" + ], "tools": [ "pacman_search", "pacman_info", @@ -80,29 +131,70 @@ } }, { - "id": "upgrade-service", - "type": "file", - "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service", - "mode": "0644", - "template": "files/mesh-pacman-weekly-upgrade.service" + "id": "keeper-account", + "type": "user", + "name": "pacman-keeper", + "shell": "/usr/bin/nologin" }, { - "id": "upgrade-timer", - "type": "file", - "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer", - "mode": "0644", - "template": "files/mesh-pacman-weekly-upgrade.timer" + "id": "state", + "type": "directory", + "mode": "0711", + "place": ".", + "unknown": "program" }, { - "id": "weekly-upgrade", - "type": "service", - "unit": "mesh-pacman-weekly-upgrade.timer", - "state": "running", - "boot": "enabled", - "restart-on": [ - "upgrade-service", - "upgrade-timer" + "id": "keeper-config", + "type": "file", + "path": "${dir:state}/keeper.json", + "mode": "0644", + "takes-config": [ + "upgrade-order", + "record-days", + "round-start-minutes" ], + "merge": "json", + "template": "files/keeper.json", + "trusted": true + }, + { + "id": "keeper", + "type": "process", + "name": "pacman-keeper", + "artifact": "tools", + "run": [ + "./pacman-tools", + "keeper" + ], + "user": "pacman-keeper", + "env": { + "PACMAN_KEEPER_BROKER": "${dir:state}/broker-keeper.secret", + "PACMAN_KEEPER_CONFIG": "${dir:state}/keeper.json" + }, + "restart-on": [ + "keeper-config" + ], + "bus": { + "state": [ + "installed", + "updates", + "runs", + "records", + "rounds", + "asks" + ], + "emits": [ + "package.changed", + "upgrade.recorded", + "upgrade.kernel-held", + "upgrade.started", + "upgrade.finished", + "upgrade.failed", + "upgrade.checked", + "rollback.started", + "rollback.finished" + ] + }, "health": { "kind": "tool", "tool": "pacman_upgrade_health", diff --git a/predefined.json b/predefined.json index 9e536767..56ba37d0 100644 --- a/predefined.json +++ b/predefined.json @@ -9,5 +9,10 @@ "description": "The size a desktop program shows its text in, in points.", "minimum": 6, "maximum": 72 + }, + "upgrade-order": { + "type": "list", + "description": "The nodes in the order they upgrade, each by its name: the first opens each week's round of upgrades, and each node upgrades only after the one before it passed its after-upgrade check. Empty or unset: no round starts.", + "trusted": true } }