jochen
0d1bb035cb
Under a module's own record (its manifest's, or its irreplaceable data's), every move waited for a person's push, even one that only changed a tools bundle: the database server stayed behind on ace on 2026-10-15 although its walk said it recreated none of its containers. ADR 0377 lets such a move roll out. A move is judged per machine against the build that machine was last sent: it goes when no container, mounted file, data path, data declaration, provider or (for a module with data) long-running process changes. A person's record, the bus and any move that cannot be judged still wait. The walk keeps per machine which way each recorded module went and why, says it in its note and log, and status names a machine behind with `push <node>` only for a move that waits.
362 lines
14 KiB
Go
362 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR
|
|
// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it
|
|
// "recreates none of its containers", and ace stayed behind until a person pushed.
|
|
|
|
// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its
|
|
// mailboxes as data, mounts its environment file, and serves its tools from a bundle.
|
|
func aMailu(policy bool) catalogue.Manifest {
|
|
raw := `{"module":"mailu","version":"1","tools":["mailu_status"],
|
|
"data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]},
|
|
"bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}],
|
|
"resources":[
|
|
{"id":"state","type":"directory"},
|
|
{"id":"mesh-state","type":"directory"},
|
|
{"id":"data-mail","type":"directory"},
|
|
{"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"},
|
|
{"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"},
|
|
{"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1",
|
|
"volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}`
|
|
var m catalogue.Manifest
|
|
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
|
panic(err)
|
|
}
|
|
if policy {
|
|
m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func resourceOf(m *catalogue.Manifest, id string) map[string]any {
|
|
for _, r := range m.Resources {
|
|
if r["id"] == id {
|
|
return r
|
|
}
|
|
}
|
|
panic("no resource " + id)
|
|
}
|
|
|
|
// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent.
|
|
type recordedMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
}
|
|
|
|
func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh {
|
|
t.Helper()
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
rm := &recordedMesh{open: open}
|
|
rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour))
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, c1.Module); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return rm
|
|
}
|
|
|
|
// register keeps a build of a module and makes it the mesh's.
|
|
func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
raw, _ := json.Marshal(m)
|
|
if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module,
|
|
Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps.
|
|
func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
under, err := sendKeeps(ctx, rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kept, err := recordedKept(under, rm.open, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return kept
|
|
}
|
|
|
|
// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send
|
|
// composes it — a recorded module kept where its move waits — and applied at once.
|
|
func (rm *recordedMesh) walk(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
inv := rm.open.inventory
|
|
wasSend := sendRollout
|
|
n := 0
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
rm.sent = append(rm.sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
kept := rm.keptOn(t, node)
|
|
builds := map[string]string{"mailu": current["mailu"].Commit}
|
|
if was, held := kept["mailu"]; held {
|
|
builds["mailu"] = was
|
|
}
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}},
|
|
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 8; i++ {
|
|
advancePlans(ctx, rm.open)
|
|
}
|
|
p, err := inv.PlanByID(ctx, "plan-mailu")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (rm *recordedMesh) sentBuild(t *testing.T, node string) string {
|
|
t.Helper()
|
|
sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sent["mailu"]
|
|
}
|
|
|
|
// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6).
|
|
func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string {
|
|
t.Helper()
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return held
|
|
}
|
|
|
|
// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out
|
|
// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy
|
|
// still reads record; and status names no machine behind for it.
|
|
func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) {
|
|
c1 := aMailu(true)
|
|
rm := aRecordedMesh(t, c1)
|
|
c2 := aMailu(true)
|
|
c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb"
|
|
c2.Tools = append(c2.Tools, "mailu_queue")
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, kept := rm.keptOn(t, n)["mailu"]; kept {
|
|
t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n)
|
|
}
|
|
}
|
|
p := rm.walk(t)
|
|
if p.State != inventory.PlanDone {
|
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
|
}
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if got := rm.sentBuild(t, n); got != "c2" {
|
|
t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy {
|
|
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, n); len(held) != 0 {
|
|
t.Fatalf("status names %s behind for %v", n, held)
|
|
}
|
|
}
|
|
if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") {
|
|
t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note)
|
|
}
|
|
if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord {
|
|
t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err)
|
|
}
|
|
}
|
|
|
|
// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push;
|
|
// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with
|
|
// `push <node>` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine
|
|
// was last sent.
|
|
func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) {
|
|
for name, change := range map[string]func(m *catalogue.Manifest){
|
|
"image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" },
|
|
"spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
change(&c2)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
p := rm.walk(t)
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if got := rm.sentBuild(t, n); got != "c1" {
|
|
t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") {
|
|
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") {
|
|
t.Fatalf("status names %s behind for %v", n, held)
|
|
}
|
|
}
|
|
// §2: the next build changes only the bundle, and the machines were last sent c1: still held.
|
|
c3 := c2
|
|
c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}}
|
|
rm.register(t, c3, "c3", time.Now())
|
|
if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" {
|
|
t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a
|
|
// provider waits.
|
|
func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) {
|
|
for name, c := range map[string]struct {
|
|
change func(m *catalogue.Manifest)
|
|
says string
|
|
}{
|
|
"a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"},
|
|
"a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"},
|
|
"the data declaration": {func(m *catalogue.Manifest) {
|
|
m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}}
|
|
}, "data declaration"},
|
|
"a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
c.change(&c2)
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
kept := rm.keptOn(t, "anchor")
|
|
if kept["mailu"] != "c1" {
|
|
t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept)
|
|
}
|
|
held := rm.recordedWaitsOn(t, "anchor")
|
|
if len(held) != 1 || !strings.Contains(held[0], c.says) {
|
|
t.Fatalf("status says %v; want it to name %q", held, c.says)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §5: a module a person set to record, and the bus, wait on a bundle-only move.
|
|
func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) {
|
|
t.Run("a person sets record", func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(false))
|
|
c2 := aMailu(false)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := rm.walk(t)
|
|
if got := rm.sentBuild(t, "anchor"); got != "c1" {
|
|
t.Fatalf("a person's record was walked past: anchor was sent %s", got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") {
|
|
t.Fatalf("the walk says %q", said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 {
|
|
t.Fatalf("status names anchor behind for %v", held)
|
|
}
|
|
})
|
|
t.Run("the bus", func(t *testing.T) {
|
|
bus := aMailu(false)
|
|
bus.Module = "nats"
|
|
bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
bus.Data = nil
|
|
rm := aRecordedMesh(t, bus)
|
|
c2 := bus
|
|
c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}}
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" {
|
|
t.Fatalf("the bus moved by an ordinary send: %v", kept)
|
|
}
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") {
|
|
t.Fatalf("the bus's move: %+v", v)
|
|
}
|
|
})
|
|
}
|
|
|
|
// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so.
|
|
func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
// anchor was last sent a build the records do not hold.
|
|
if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0",
|
|
map[string]string{"mailu": "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") ||
|
|
!strings.Contains(v.said, "not in the build records") {
|
|
t.Fatalf("an unjudgeable move: %+v", v)
|
|
}
|
|
if _, _, err := planFor(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") {
|
|
t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err)
|
|
}
|
|
// laptop, judged from c1, goes; anchor waits.
|
|
p := rm.walk(t)
|
|
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") {
|
|
t.Fatalf("the walk says of anchor %q", said)
|
|
}
|
|
if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" {
|
|
t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded)
|
|
}
|
|
}
|