364 lines
14 KiB
Go
364 lines
14 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR
|
|
// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it
|
|
// "recreates none of its containers", and ace stayed behind until a person pushed.
|
|
|
|
// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its
|
|
// mailboxes as data, mounts its environment file, and serves its tools from a bundle.
|
|
func aMailu(policy bool) catalogue.Manifest {
|
|
raw := `{"module":"mailu","version":"1","tools":["mailu_status"],
|
|
"data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]},
|
|
"bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}],
|
|
"resources":[
|
|
{"id":"state","type":"directory"},
|
|
{"id":"mesh-state","type":"directory"},
|
|
{"id":"data-mail","type":"directory"},
|
|
{"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"},
|
|
{"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"},
|
|
{"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1",
|
|
"volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}`
|
|
var m catalogue.Manifest
|
|
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
|
panic(err)
|
|
}
|
|
if policy {
|
|
m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func resourceOf(m *catalogue.Manifest, id string) map[string]any {
|
|
for _, r := range m.Resources {
|
|
if r["id"] == id {
|
|
return r
|
|
}
|
|
}
|
|
panic("no resource " + id)
|
|
}
|
|
|
|
// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent.
|
|
type recordedMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
}
|
|
|
|
func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh {
|
|
t.Helper()
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
rm := &recordedMesh{open: open}
|
|
rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour))
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, c1.Module); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return rm
|
|
}
|
|
|
|
// register keeps a build of a module and makes it the mesh's.
|
|
func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
raw, _ := json.Marshal(m)
|
|
if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module,
|
|
Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps.
|
|
func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
under, err := sendKeeps(ctx, rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kept, err := recordedKept(under, rm.open, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return kept
|
|
}
|
|
|
|
// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send
|
|
// composes it — a recorded module kept where its move waits — and applied at once.
|
|
func (rm *recordedMesh) walk(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
inv := rm.open.inventory
|
|
wasSend := sendRollout
|
|
n := 0
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
rm.sent = append(rm.sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
kept := rm.keptOn(t, node)
|
|
builds := map[string]string{"mailu": current["mailu"].Commit}
|
|
if was, held := kept["mailu"]; held {
|
|
builds["mailu"] = was
|
|
}
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}},
|
|
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 8; i++ {
|
|
advancePlans(ctx, rm.open)
|
|
}
|
|
p, err := inv.PlanByID(ctx, "plan-mailu")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (rm *recordedMesh) sentBuild(t *testing.T, node string) string {
|
|
t.Helper()
|
|
sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sent["mailu"]
|
|
}
|
|
|
|
// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6).
|
|
func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string {
|
|
t.Helper()
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return held
|
|
}
|
|
|
|
// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out
|
|
// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy
|
|
// still reads record; and status names no machine behind for it.
|
|
func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) {
|
|
c1 := aMailu(true)
|
|
rm := aRecordedMesh(t, c1)
|
|
c2 := aMailu(true)
|
|
c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb"
|
|
c2.Tools = append(c2.Tools, "mailu_queue")
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, kept := rm.keptOn(t, n)["mailu"]; kept {
|
|
t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n)
|
|
}
|
|
}
|
|
p := rm.walk(t)
|
|
if p.State != inventory.PlanDone {
|
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
|
}
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if got := rm.sentBuild(t, n); got != "c2" {
|
|
t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy {
|
|
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, n); len(held) != 0 {
|
|
t.Fatalf("status names %s behind for %v", n, held)
|
|
}
|
|
}
|
|
if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") {
|
|
t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note)
|
|
}
|
|
if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord {
|
|
t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err)
|
|
}
|
|
}
|
|
|
|
// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push;
|
|
// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with
|
|
// `push <node>` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine
|
|
// was last sent.
|
|
func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) {
|
|
for name, change := range map[string]func(m *catalogue.Manifest){
|
|
"image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" },
|
|
"spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
change(&c2)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
p := rm.walk(t)
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if got := rm.sentBuild(t, n); got != "c1" {
|
|
t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") {
|
|
t.Fatalf("the walk says of mailu on %s %q", n, said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") {
|
|
t.Fatalf("status names %s behind for %v", n, held)
|
|
}
|
|
}
|
|
// §2: the next build changes only the bundle, and the machines were last sent c1: still held.
|
|
c3 := c2
|
|
c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}}
|
|
rm.register(t, c3, "c3", time.Now())
|
|
if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" {
|
|
t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a
|
|
// provider waits.
|
|
func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) {
|
|
for name, c := range map[string]struct {
|
|
change func(m *catalogue.Manifest)
|
|
says string
|
|
}{
|
|
"a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"},
|
|
"a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"},
|
|
"the data declaration": {func(m *catalogue.Manifest) {
|
|
m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}}
|
|
}, "data declaration"},
|
|
"a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
c.change(&c2)
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
kept := rm.keptOn(t, "anchor")
|
|
if kept["mailu"] != "c1" {
|
|
t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept)
|
|
}
|
|
held := rm.recordedWaitsOn(t, "anchor")
|
|
if len(held) != 1 || !strings.Contains(held[0], c.says) {
|
|
t.Fatalf("status says %v; want it to name %q", held, c.says)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §5: a module a person set to record, and the bus, wait on a bundle-only move.
|
|
func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) {
|
|
t.Run("a person sets record", func(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(false))
|
|
c2 := aMailu(false)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := rm.walk(t)
|
|
if got := rm.sentBuild(t, "anchor"); got != "c1" {
|
|
t.Fatalf("a person's record was walked past: anchor was sent %s", got)
|
|
}
|
|
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") {
|
|
t.Fatalf("the walk says %q", said)
|
|
}
|
|
if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 {
|
|
t.Fatalf("status names anchor behind for %v", held)
|
|
}
|
|
})
|
|
t.Run("the bus", func(t *testing.T) {
|
|
bus := aMailu(false)
|
|
bus.Module = "nats"
|
|
bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
bus.Data = nil
|
|
rm := aRecordedMesh(t, bus)
|
|
c2 := bus
|
|
c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}}
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" {
|
|
t.Fatalf("the bus moved by an ordinary send: %v", kept)
|
|
}
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") {
|
|
t.Fatalf("the bus's move: %+v", v)
|
|
}
|
|
})
|
|
}
|
|
|
|
// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so.
|
|
func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) {
|
|
rm := aRecordedMesh(t, aMailu(true))
|
|
c2 := aMailu(true)
|
|
c2.Bundles[0].Digest = "sha256:bbb"
|
|
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
|
|
// anchor was last sent a build the records do not hold.
|
|
if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0",
|
|
map[string]string{"mailu": "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f, err := readMoveFacts(t.Context(), rm.open.inventory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") ||
|
|
!strings.Contains(v.said, "not in the build records") {
|
|
t.Fatalf("an unjudgeable move: %+v", v)
|
|
}
|
|
if _, _, err := planFor(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") {
|
|
t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err)
|
|
}
|
|
// laptop, judged from c1, goes; anchor waits.
|
|
p := rm.walk(t)
|
|
if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") {
|
|
t.Fatalf("the walk says of anchor %q", said)
|
|
}
|
|
if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" {
|
|
t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded)
|
|
}
|
|
}
|