mesh-sdk/go/delivery/placement_test.go

package delivery

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
)

// ADR 0549 §1, §2: the placement key and file, the value's refusals, its reading as the composition reads a node, what
// a change says, and the catalogue's check of every file.
func TestPlacement(t *testing.T) {
	t.Run("key and file", func(t *testing.T) {
		// Research 072 row 37: one node key and one key per module, a legacy key read beside them, any other refused.
		for key, want := range map[string]struct {
			node, module string
			kind         PlacementKeyKind
		}{"g14.node": {"g14", "", PlacementNodeKind}, "g14.app": {"g14", "app", PlacementModuleKind},
			"placement.g14": {"g14", "", PlacementLegacyKind}} {
			if n, m, k, err := ParsePlacementKey(key); err != nil || n != want.node || m != want.module || k != want.kind {
				t.Errorf("ParsePlacementKey(%q) = %q, %q, %d, %v", key, n, m, k, err)
			}
		}
		for _, bad := range []string{"want.g14.x", "placement.", "placement.a.b", "g14", "g14.a.b", ".node"} {
			if _, _, _, err := ParsePlacementKey(bad); err == nil {
				t.Errorf("ParsePlacementKey(%q) took it", bad)
			}
		}
		if k, err := PlacementNodeKey("g14"); err != nil || k != "g14.node" {
			t.Errorf("PlacementNodeKey = %q, %v", k, err)
		}
		if k, err := PlacementModuleKey("g14", "app"); err != nil || k != "g14.app" {
			t.Errorf("PlacementModuleKey = %q, %v", k, err)
		}
		if k, err := PlacementNodeKeys("g14"); err != nil || k != "g14.>" {
			t.Errorf("PlacementNodeKeys = %q, %v", k, err)
		}
		if k, err := LegacyPlacementKey("g14"); err != nil || k != "placement.g14" {
			t.Errorf("LegacyPlacementKey = %q, %v", k, err)
		}
		for name, err := range map[string]error{"a dotted node": func() error { _, err := PlacementNodeKey("a.b"); return err }(),
			"a node named placement": func() error { _, err := PlacementNodeKey("placement"); return err }(),
			"a module named node":    func() error { _, err := PlacementModuleKey("g14", "node"); return err }()} {
			if err == nil {
				t.Errorf("%s was taken", name)
			}
		}
		// ADR 0567 §1: the seat's three states are named after the holding module, and every reader takes the names here.
		for got, want := range map[string]string{PlacementBucket: "mesh-delivery-assigner_placement",
			PlacementProposalsBucket: "mesh-delivery-assigner_proposals", PlacementCandidatesBucket: "mesh-delivery-assigner_candidates",
			PlacementRead: "mesh-delivery-assigner.placement", PlacementCandidatesRead: "mesh-delivery-assigner.candidates",
			PlacementProposalsRead: "mesh-delivery-assigner.proposals"} {
			if got != want {
				t.Errorf("a bucket is named %q, want %q", got, want)
			}
		}
		if PlacementHistory != 64 || PlacementStreamDays != 30 {
			t.Errorf("history %d, stream %d days", PlacementHistory, PlacementStreamDays)
		}
	})

	g14 := Placement{Node: "g14", Revision: "abc",
		Decided: NodeDecided{Endpoint: "g14.example:51820", Site: "home", Address: "10.77.0.2", Account: "op", Taken: []string{"zsh"}},
		Modules: []PlacedModule{
			{Module: "zsh"},
			{Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "novox", Module: "postgres"}}},
		},
		Pins:  map[string]Provider{"shell": {Node: "g14", Module: "zsh"}},
		Holds: []Holding{{Claim: "node-shell", Scope: "node", Module: "zsh"}},
	}
	novox := Placement{Node: "novox", Decided: NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []PlacedModule{{Module: "postgres", Ports: map[string]int{"5432": 5432}}},
		Holds: []Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}}

	t.Run("value", func(t *testing.T) {
		raw, err := EncodePlacement(g14)
		if err != nil {
			t.Fatal(err)
		}
		back, err := DecodePlacement(raw)
		if err != nil || back.V != Version || !SamePlacement(back, g14) {
			t.Fatalf("round trip: %+v, %v", back, err)
		}
		moved := g14
		moved.Revision = "def"
		if !SamePlacement(g14, moved) {
			t.Error("a new revision alone counts as a change")
		}
		if _, err := DecodePlacement([]byte(`{"v":9,"node":"g14","modules":[]}`)); !errors.Is(err, ErrUnknownVersion) {
			t.Errorf("version 9: %v", err)
		}
		for name, bad := range map[string]string{
			"module twice":     `{"v":1,"node":"g14","modules":[{"module":"a"},{"module":"a"}]}`,
			"port not number":  `{"v":1,"node":"g14","modules":[{"module":"a","ports":{"http":80}}]}`,
			"host port zero":   `{"v":1,"node":"g14","modules":[{"module":"a","ports":{"80":0}}]}`,
			"binding no node":  `{"v":1,"node":"g14","modules":[{"module":"a","bindings":{"pg":{"module":"pg"}}}]}`,
			"pin no module":    `{"v":1,"node":"g14","modules":[],"pins":{"pg":{"node":"x"}}}`,
			"hold not placed":  `{"v":1,"node":"g14","modules":[],"holds":[{"claim":"s","module":"a"}]}`,
			"node not a name":  `{"v":1,"node":"g.14","modules":[]}`,
			"value not shaped": `{"v":1,"node":"g14","modules":{}}`,
		} {
			if _, err := DecodePlacement([]byte(bad)); err == nil {
				t.Errorf("%s: taken", name)
			}
		}
	})

	t.Run("as the composition reads it", func(t *testing.T) {
		if got := g14.Assigned(); !reflect.DeepEqual(got, []string{"zsh", "app"}) {
			t.Errorf("Assigned = %v, want the file's order", got)
		}
		if got := g14.ChosenPins(); !reflect.DeepEqual(got, map[string]catalogue.Chosen{"shell": {Node: "g14", Module: "zsh"}}) {
			t.Errorf("ChosenPins = %v", got)
		}
		if got := g14.Bound(); !reflect.DeepEqual(got, map[string]map[string]catalogue.Chosen{"app": {"postgres": {Node: "novox", Module: "postgres"}}}) {
			t.Errorf("Bound = %v", got)
		}
		if got := g14.HostPorts(); !reflect.DeepEqual(got, map[string]map[int]int{"app": {8080: 18080}}) {
			t.Errorf("HostPorts = %v", got)
		}
		if got := g14.Held(); !reflect.DeepEqual(got, []catalogue.Held{{Claim: "node-shell", Scope: "node", Node: "g14", Module: "zsh"}}) {
			t.Errorf("Held = %v", got)
		}
		layered := PlacedModule{Module: "app", HasMesh: true, Layer: map[string]any{"expose": map[string]any{"80": "mesh"}}}
		want := []catalogue.Layer{
			{From: catalogue.MeshWideLayer, Values: map[string]any{}},
			{From: "g14", Values: map[string]any{"mode": "x"}, Placement: map[string]any{"expose": map[string]any{"80": "mesh"}}},
		}
		if got := layered.Layers("g14", map[string]any{"mode": "x"}); !reflect.DeepEqual(got, want) {
			t.Errorf("Layers = %#v, want %#v", got, want)
		}
		if got := (PlacedModule{Module: "a"}).Layers("g14", map[string]any{"k": 1}); len(got) != 1 || got[0].From != "g14" {
			t.Errorf("values alone make the node's layer: %#v", got)
		}
		if got := (PlacedModule{Module: "a", HasLayer: true}).Layers("g14", nil); len(got) != 1 || got[0].Values == nil {
			t.Errorf("an empty layer of the node's own is kept, with empty values: %#v", got)
		}
		if got := (PlacedModule{Module: "a"}).Layers("g14", nil); got != nil {
			t.Errorf("no layer and no value is no layer: %#v", got)
		}
		empty := Placement{Node: "x"}
		if empty.ChosenPins() != nil || empty.Bound() != nil || empty.HostPorts() != nil || len(empty.Held()) != 0 {
			t.Error("an empty placement reads as something")
		}
	})

	t.Run("what a change says", func(t *testing.T) {
		after := g14
		after.Modules = []PlacedModule{{Module: "app", Ports: map[string]int{"8080": 18081}}, {Module: "new"}}
		got := ChangeOf(g14, after)
		want := PlacementChange{Node: "g14", Added: []string{"new"}, Removed: []string{"zsh"}, Changed: []string{"app"}}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("ChangeOf = %+v, want %+v", got, want)
		}
		pinned := g14
		pinned.Pins = nil
		if got := ChangeOf(g14, pinned); !reflect.DeepEqual(got.Changed, []string{"app", "zsh"}) {
			t.Errorf("a pin change changes every module on the node: %+v", got)
		}
		if got := ChangeOf(Placement{}, novox); !reflect.DeepEqual(got.Added, []string{"postgres"}) || got.Node != "novox" {
			t.Errorf("a new node: %+v", got)
		}
	})

	// Research 072 row 37: a node's placement as its node key and module keys, written so a reader never takes a part of
	// a write, read beside the form before, and converted from it once.
	t.Run("one key per module and node", func(t *testing.T) {
		values, err := PlacementValues(g14)
		if err != nil || len(values) != 4 || values["g14.node"] == nil || values["g14.zsh"] == nil || values["g14.app"] == nil ||
			values["placement.g14"] == nil {
			t.Fatalf("PlacementValues = %v, %v", values, err)
		}
		got, err := PlacementsOf(values)
		if err != nil || !SamePlacement(got["g14"], g14) || len(got) != 1 {
			t.Fatalf("PlacementsOf = %+v, %v", got, err)
		}
		if got := InsideAddresses(map[string]Placement{"g14": g14, "novox": novox, "x": {Node: "x"}, "y": {Node: "y", Decided: NodeDecided{Address: " 10.77.0.1"}}}); strings.Join(got, ",") != "10.77.0.1,10.77.0.2" {
			t.Errorf("InsideAddresses = %v", got)
		}
		legacy, _ := EncodePlacement(novox)
		stale, _ := EncodePlacement(Placement{Node: "g14"})
		both := map[string][]byte{"placement.novox": legacy, "placement.g14": stale}
		for k, v := range values {
			both[k] = v
		}
		if got, err := PlacementsOf(both); err != nil || !SamePlacement(got["g14"], g14) || !SamePlacement(got["novox"], novox) {
			t.Errorf("a node key wins over a legacy key, and a legacy key alone places its node: %+v, %v", got, err)
		}

		keys := NewPlacementKeys()
		rev := uint64(0)
		apply := func(writes []PlacementKeyWrite, upTo int) {
			t.Helper()
			for i, w := range writes {
				if i == upTo {
					return
				}
				rev++
				if _, err := keys.Put(w.Key, w.Value, rev, w.Value == nil); err != nil {
					t.Fatalf("write %d (%s): %v", i, w.Key, err)
				}
			}
		}
		// The form before converts once: its node and module keys, the legacy key left for the readers not yet delivered.
		rev++
		legacyG14, _ := EncodePlacement(g14)
		if _, err := keys.Put("placement.g14", legacyG14, rev, false); err != nil || !keys.Legacy("g14") {
			t.Fatalf("the legacy key: %v", err)
		}
		migrations, err := keys.Migrations()
		if err != nil || len(migrations["g14"]) != 3 || !migrations["g14"][2].Commit {
			// The legacy key holds the placement already: no mirror write.
			t.Fatalf("Migrations = %+v, %v", migrations, err)
		}
		apply(migrations["g14"], -1)
		if m, _ := keys.Migrations(); len(m) != 0 || keys.Legacy("g14") {
			t.Fatalf("a converted node converts again: %+v", m)
		}
		if p, r, ok := keys.Of("g14"); !ok || !SamePlacement(p, g14) || r != rev {
			t.Fatalf("Of after the conversion = %+v, %d, %v", p, r, ok)
		}

		// A write under way is never read half: app's new port is not read before the node key names it.
		after := clonePlacement(g14)
		after.Modules = []PlacedModule{{Module: "app", Ports: map[string]int{"8080": 18081}}, {Module: "new"}}
		after.Holds, after.Pins = nil, nil
		writes, err := keys.Writes("g14", &after)
		if err != nil {
			t.Fatal(err)
		}
		var kinds []string
		for _, w := range writes {
			kinds = append(kinds, fmt.Sprintf("%s:%v:%d", w.Key, w.Value != nil, w.Base))
		}
		if strings.Join(kinds, " ") != "g14.app:true:3 g14.new:true:0 g14.node:true:4 placement.g14:true:1 g14.zsh:false:2" ||
			!writes[3].Mirror {
			t.Fatalf("Writes = %v", kinds)
		}
		apply(writes, 2)
		if p, _, _ := keys.Of("g14"); !SamePlacement(p, g14) {
			t.Errorf("the module keys written before the node key are read: %+v", p)
		}
		apply(writes[2:], -1)
		if p, _, _ := keys.Of("g14"); !SamePlacement(p, after) {
			t.Errorf("the write done is not read whole: %+v", p)
		}
		if legacy, _ := PlacementsOf(map[string][]byte{"placement.g14": writes[3].Value}); !SamePlacement(legacy["g14"], after) {
			t.Errorf("the mirror is not the placement written: %+v", legacy)
		}
		// A module key that stays is not written again, nor a mirror that holds it.
		if again, _ := keys.Writes("g14", &after); len(again) != 1 || !again[0].Commit {
			t.Errorf("an unchanged placement writes %+v", again)
		}
		// Keys that do not agree keep what the node had, and say so.
		stray, _ := EncodePlacementModule(PlacementModule{Node: "g14", Placed: PlacedModule{Module: "app"}})
		if _, err := keys.Put("g14.app", stray, 99, false); err == nil {
			t.Error("a module key the node key does not name the value of was taken silently")
		}
		if p, _, _ := keys.Of("g14"); !SamePlacement(p, after) {
			t.Errorf("a stray module key moved the placement: %+v", p)
		}
		// The holder's next write puts the module key right again.
		repair, _ := keys.Writes("g14", &after)
		if len(repair) != 2 || repair[0].Key != "g14.app" || repair[0].Base != 99 || !repair[1].Commit {
			t.Fatalf("the repair writes %+v", repair)
		}
		apply(repair, -1)
		// No placement: the legacy key first, then the node key, then the module keys; none is read after.
		gone, _ := keys.Writes("g14", nil)
		kinds = nil
		for _, w := range gone {
			kinds = append(kinds, w.Key)
		}
		if strings.Join(kinds, " ") != "placement.g14 g14.node g14.app g14.new" {
			t.Errorf("Writes(nil) = %v", kinds)
		}
		apply(gone, 2)
		if _, _, ok := keys.Of("g14"); ok {
			t.Error("a node key deleted still places its node")
		}
		if _, err := DecodePlacementNode([]byte(`{"v":1,"node":"g14","modules":[{"module":"a","digest":"x"},{"module":"a","digest":"x"}]}`)); err == nil {
			t.Error("a node key naming a module twice reads")
		}
	})

	t.Run("the catalogue's check", func(t *testing.T) {
		held := func(m string) bool { return m != "ghost" }
		enrolled := func(n string) bool { return n != "stranger" }
		if r := CheckPlacements(map[string]Placement{"g14": g14, "novox": novox}, held, enrolled); len(r) != 0 {
			t.Fatalf("a sound placement is refused: %v", r)
		}
		// Issue 1126: the private network is the mesh's own module, held by no repository the indexer reads, and placed on
		// every node by the seed; the check takes it whatever the indexer holds.
		wired := Placement{Node: "novox", Modules: []PlacedModule{{Module: "mesh-wireguard"}}}
		if r := CheckPlacements(map[string]Placement{"novox": wired}, func(string) bool { return false }, enrolled); len(r) != 0 ||
			!ProvidedByMesh("mesh-wireguard") || ProvidedByMesh("postgres") {
			t.Errorf("the private network placed with no repository holding it is refused: %v", r)
		}
		if m, _, err := ProvidedManifest("mesh-wireguard"); err != nil || m.Module != "mesh-wireguard" {
			t.Errorf("the provided manifest = %+v, %v", m, err)
		}
		// A replicated mesh seat (ADR 0223) is held on two nodes; a seat that is not replicated is refused there.
		for seat, refused := range map[string]bool{"mesh-dns-resolver": false, "mesh-broker": true} {
			a := Placement{Node: "ace", Modules: []PlacedModule{{Module: "dns"}}, Holds: []Holding{{Claim: seat, Scope: "mesh", Module: "dns"}}}
			b := a
			b.Node = "novox"
			r := CheckPlacements(map[string]Placement{"ace": a, "novox": b}, held, enrolled)
			if (len(r) != 0) != refused {
				t.Errorf("%s on two nodes: refusals %v, want refused %v", seat, r, refused)
			}
		}
		clash := g14
		clash.Modules = append([]PlacedModule{{Module: "ghost", Ports: map[string]int{"80": 18080}}}, g14.Modules...)
		clash.Modules[2] = PlacedModule{Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "ace", Module: "postgres"}}}
		clash.Pins = map[string]Provider{"shell": {Node: "g14", Module: "bash"}}
		// ADR 0564 clause 6, Limits 5 (issue 1952 S8): mesh-watcher and mesh-self-check are never placed on one node.
		clash.Modules = append(clash.Modules, PlacedModule{Module: "mesh-watcher"}, PlacedModule{Module: "mesh-self-check"})
		twice := novox
		twice.Node = "stranger"
		twice.Decided.Address = "10.77.0.2"
		r := CheckPlacements(map[string]Placement{"g14": clash, "novox": novox, "stranger": twice}, held, enrolled)
		for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080",
			"bound to ace/postgres", "pin of shell names g14/bash", "stranger is a node with no record",
			"mesh-store, held once for the mesh, is held on novox and stranger",
			"g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it",
			"g14: mesh-watcher and mesh-self-check are never placed on one node (ADR 0227 §6, ADR 0564 clause 6)"} {
			found := false
			for _, line := range r {
				found = found || strings.Contains(line, want)
			}
			if !found {
				t.Errorf("no refusal says %q; got %v", want, r)
			}
		}
	})

	// ADR 0567 §3: a proposal's candidates, one key `<proposal>.<node>` per node it must hear from, and each node's answer
	// in its have.
	t.Run("candidates", func(t *testing.T) {
		const id = "p-20261028t183000-1a2b3c"
		key, err := PlacementCandidateKey(id, "g14")
		if err != nil || key != id+".g14" {
			t.Fatalf("PlacementCandidateKey = %q, %v", key, err)
		}
		if p, n, err := ParsePlacementCandidateKey(key); err != nil || p != id || n != "g14" {
			t.Fatalf("ParsePlacementCandidateKey = %q, %q, %v", p, n, err)
		}
		for _, bad := range []string{"g14", ".g14", "p-x.g.14", "p-x.", "P-X.g14", "feat/1.g14"} {
			if _, _, err := ParsePlacementCandidateKey(bad); err == nil {
				t.Errorf("ParsePlacementCandidateKey(%q) took it", bad)
			}
		}
		for name, c := range map[string]PlacementCandidate{
			"placed":  {Proposal: id, Placement: &g14},
			"removed": {Proposal: id},
			"reached": {Proposal: id, Reached: true, Placement: &g14},
		} {
			raw, err := EncodePlacementCandidate(c)
			if err != nil {
				t.Fatalf("%s: %v", name, err)
			}
			back, err := DecodePlacementCandidate(raw)
			if err != nil || back.Proposal != id || back.Reached != c.Reached || (back.Placement == nil) != (c.Placement == nil) ||
				(c.Placement != nil && !SamePlacement(*back.Placement, g14)) {
				t.Errorf("%s: round trip %+v, %v", name, back, err)
			}
		}
		for name, c := range map[string]PlacementCandidate{
			"no proposal": {},
			"a group":     {Proposal: "feat/x"},
			"bad place":   {Proposal: id, Placement: &Placement{Node: "a.b"}},
		} {
			if _, err := EncodePlacementCandidate(c); err == nil {
				t.Errorf("%s: taken", name)
			}
		}
		answers := map[string]CandidateAnswer{
			"same":    {Proposal: id, Revision: 3, Verdict: CandidateSame, Same: []string{"app", "node"}},
			"differs": {Proposal: id, Revision: 3, Verdict: CandidateDiffers, Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}},
			"refused": {Proposal: id, Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"},
			"added":   {Proposal: id, Revision: 1, Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}},
			"a value": {Proposal: id, Revision: 1, Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}},
			// A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126,
			// the seed's revert, mesh-catalog #526).
			"returns": {Proposal: id, Revision: 1, Verdict: CandidateSame, Returns: ReturnsToController, Same: []string{"app", "node"}},
			"returns, the controller's differs": {Proposal: id, Revision: 1, Verdict: CandidateDiffers, Returns: ReturnsToController, Same: []string{"node"},
				Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}, Removed: []string{"old"}},
		}
		n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers,
			Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"}
		raw, err := EncodeHaveNode(n)
		if err != nil {
			t.Fatal(err)
		}
		back, err := DecodeHaveNode(raw)
		if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers,
			Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) {
			t.Fatalf("a node's answers changed on the way: %+v, %v", back, err)
		}
		if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) {
			t.Error("a node's have without its answers counts as the same")
		}
		if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers,
			Composes: ComposesController, ComposesWhy: n.ComposesWhy}) {
			t.Error("a node's have composing otherwise counts as the same")
		}
		if _, err := EncodeHaveNode(HaveNode{Composes: "both"}); err == nil {
			t.Error("a node composing neither its own nor the controller's was taken")
		}
		// CompareComposed: the bus's user list is left out on both sides; anything else that differs is named.
		file := func(id, content string) json.RawMessage {
			return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`)
		}
		fp := "src2:" + strings.Repeat("a", 64)
		comp := func(rs ...json.RawMessage) Composed {
			return Composed{Node: "novox", Module: "nats", FP: fp, Resources: rs}
		}
		for name, tc := range map[string]struct {
			composed, running Composed
			resource, why     string
		}{
			"user list alone": {comp(file("conf", "a"), file("nats.bus-users", "stand-in")), comp(file("conf", "a"), file("nats.bus-users", "real")), "", ""},
			"a resource":      {comp(file("conf", "a")), comp(file("conf", "b")), "conf", "the resource differs"},
			"one side":        {comp(file("conf", "a"), file("more", "x")), comp(file("conf", "a")), "more", "composed, and not in what runs"},
			"order":           {comp(file("conf", "a"), file("more", "x")), comp(file("more", "x"), file("conf", "a")), "", "the same resources in another order"},
		} {
			c := CompareComposed(map[string]Composed{"novox.nats." + fp: tc.composed}, []Composed{tc.running})
			if tc.why == "" {
				if len(c.Differs) != 0 || len(c.Same) != 1 {
					t.Errorf("%s: %+v", name, c)
				}
			} else if len(c.Differs) != 1 || c.Differs[0].Resource != tc.resource || c.Differs[0].Why != tc.why ||
				c.Differs[0].Composed == "" || c.Differs[0].Running == "" || c.Differs[0].Composed == c.Differs[0].Running {
				t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why)
			}
		}
		// Issue 1126, the shadow shows its work: every differing resource with both bodies, each value a secret could be
		// in masked (never a secret in an answer), saying whether the two sides hold the same value.
		sealedFile := func(id, sealed, mode string) json.RawMessage {
			return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`)
		}
		// Issue 1126, C1: a sealed value is judged by the value it opens to at its own slot: b sealed again (box-2 and
		// box-3 open to one value) is no difference; d holds at its slot a box of another value, though that value is
		// the one b holds, and differs at its path `sealed`.
		sameBox := func(a, b string) (same, known bool) {
			return (a == "box-2" && b == "box-3") || a == b, true
		}
		shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"),
			file("c", "x"), sealedFile("d", "box-2", "0600"))},
			[]Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"), sealedFile("d", "box-4", "0600"))},
			sameBox)
		if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 {
			t.Fatalf("every differing resource is not shown, or b sealed again is shown: %+v", shown.Differs)
		}
		if c := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("b", "box-2", "0600"))},
			[]Composed{comp(sealedFile("b", "box-3", "0600"))}, sameBox); len(c.Same) != 1 {
			t.Errorf("a value sealed again at its own slot is not the same: %+v", c)
		}
		for _, row := range []struct {
			id, has, hasNot string
			paths           []string
		}{
			{"a", `"mode":"0600"`, "box-1", []string{"mode"}},
			{"c", `"content":"x"`, "", []string{"content"}},
			{"d", "<masked: another value>", "box-2", []string{"sealed"}},
		} {
			var body ResourceBodies
			for _, b := range shown.Differs[0].Bodies {
				if b.ID == row.id {
					body = b
				}
			}
			both := body.Composed + body.Running
			if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) ||
				!slices.Equal(body.Paths, row.paths) {
				t.Errorf("%s shown as %+v, want paths %v", row.id, body, row.paths)
			}
		}
		// Issue 1126, C1, the seed-7 probe: a file naming secrets carries each sealed value in its `secrets`, and each entry
		// is a slot of its own. The shape is intoFile's output, captured from catalogue for a file naming two secrets
		// (2026-10-28), with the module a composition names each resource of; only the boxes are replaced.
		withSecrets := func(db, key string) json.RawMessage {
			return json.RawMessage(`{"content":"[database]\npassword = ${secret:postgres-database}\n[security]\nsecret_key = ${secret:secret-key}\n",` +
				`"id":"grafana.ini","mode":"0600","module":"nats","path":"/etc/grafana/grafana.ini","secrets":{"postgres-database":"` + db +
				`","secret-key":"` + key + `"},"type":"file"}`)
		}
		if c := CompareComposedWith(map[string]Composed{"k": comp(withSecrets("box-2", "box-1"))},
			[]Composed{comp(withSecrets("box-3", "box-1"))}, sameBox); len(c.Same) != 1 {
			t.Errorf("a secret sealed again at its own entry is not the same: %+v", c)
		}
		if kept := KeepSeals(comp(withSecrets("box-2", "box-1")), SealsIn(comp(withSecrets("box-3", "box-1"))), sameBox); !bytes.Equal(kept.Resources[0], withSecrets("box-3", "box-1")) {
			t.Errorf("the box that runs is not kept byte for byte: %s", kept.Resources[0])
		}
		if s := SealsIn(comp(withSecrets("box-2", "box-1"))); s[SealSlot("grafana.ini", SecretSlotField("postgres-database"))] != "box-2" || len(s) != 2 {
			t.Errorf("the secrets' slots are %v", s)
		}
		// Another value at one entry differs there alone, each entry masked by what it opens to.
		// box-2 and box-4 open to one value, so the box that runs is kept there; box-5 and box-6 are two values.
		mixed := CompareComposedWith(map[string]Composed{"k": comp(withSecrets("box-2", "box-5"))},
			[]Composed{comp(withSecrets("box-4", "box-6"))}, func(a, b string) (bool, bool) {
				return (a == "box-2" && b == "box-4") || a == b, true
			})
		if len(mixed.Differs) != 1 || len(mixed.Differs[0].Bodies) != 1 {
			t.Fatalf("a secret of another value is not shown: %+v", mixed)
		}
		if body := mixed.Differs[0].Bodies[0]; !slices.Equal(body.Paths, []string{"secrets.secret-key"}) ||
			!strings.Contains(body.Composed, `"secret-key":"<masked: another value>"`) || strings.Contains(body.Composed+body.Running, "box-") ||
			!strings.Contains(body.Running, `"postgres-database":"<masked: the same on both sides>"`) {
			t.Errorf("a file's secrets shown as %+v", body)
		}
		if c, _ := MaskedPair(withSecrets("box-2", "box-5"), withSecrets("box-4", "box-6"), func(a, b string) (bool, bool) {
			return a == "box-2", true
		}); !strings.Contains(c, `"postgres-database":"<masked: another seal of the same value>"`) || !strings.Contains(c, `"secret-key":"<masked: another value>"`) {
			t.Errorf("each secret is not masked by what it opens to: %s", c)
		}
		// Only a file's secrets are split by entry: a vault box (an object under `vault-sealed`) is one masked word and one
		// path, judged whole by what it opens to (rev-seed-same-3, round 1).
		box := func(ct string) json.RawMessage {
			return json.RawMessage(`{"id":"core-box","type":"file","module":"nats","path":"/etc/core","mode":"0600","vault-sealed":{"v":1,"eph":"e-` +
				ct + `","ct":"` + ct + `"}}`)
		}
		vault := CompareComposedWith(map[string]Composed{"k": comp(box("c1"))}, []Composed{comp(box("c2"))},
			func(a, b string) (bool, bool) {
				return false, strings.Contains(a, `"ct":"c1"`) && strings.Contains(b, `"ct":"c2"`)
			})
		if len(vault.Differs) != 1 || len(vault.Differs[0].Bodies) != 1 {
			t.Fatalf("a box of another value is not shown: %+v", vault)
		}
		if body := vault.Differs[0].Bodies[0]; !slices.Equal(body.Paths, []string{"vault-sealed"}) ||
			!strings.Contains(body.Composed, `"vault-sealed":"<masked: another value>"`) || strings.Contains(body.Composed+body.Running, "c1") {
			t.Errorf("a vault box shown as %+v", body)
		}
		// Issue 1126, T1: the paths are found on the whole bodies, before they are cut; JSON in a string is walked into, a
		// key a secret could be under is named and never shown.
		long := strings.Repeat("x", MaxBody)
		conf := func(link, password string) json.RawMessage {
			inner, _ := json.Marshal(map[string]any{"pad": long, "links": []string{"a", link}, "password": password})
			outer, _ := json.Marshal(map[string]any{"id": "conf", "type": "file", "module": "nats", "path": "/etc/conf",
				"mode": "0600", "content": string(inner)})
			return outer
		}
		far := CompareComposedWith(map[string]Composed{"k": comp(conf("b", "hunter2"))}, []Composed{comp(conf("c", "hunter3"))}, nil)
		if len(far.Differs) != 1 || len(far.Differs[0].Bodies) != 1 ||
			!slices.Equal(far.Differs[0].Bodies[0].Paths, []string{"content.links.1", "content.password"}) ||
			!strings.Contains(far.Differs[0].Bodies[0].Composed, "cut at") || strings.Contains(fmt.Sprint(far), "hunter") {
			t.Errorf("a difference past the cut: %+v", far.Differs)
		}
		// And an answer too large for the node's key is fitted: the texts go first, the verdict and the paths stay.
		big := CandidateAnswer{Proposal: "p-x1", Revision: 1, Verdict: CandidateDiffers, Differs: far.Differs}
		if fitted := FitAnswer(big, 4<<10); len(fitted.Differs[0].Bodies[0].Composed) != 0 || len(fitted.Differs[0].Bodies[0].Paths) != 2 ||
			fitted.Verdict != CandidateDiffers || !strings.Contains(fitted.Note, "cut") || big.Differs[0].Bodies[0].Composed == "" {
			t.Errorf("fitted as %+v", fitted)
		}
		if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`),
			json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") ||
			strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "<masked: differs>") ||
			!strings.Contains(c, "<masked: the same on both sides>") {
			t.Errorf("masked as %s and %s", c, r)
		}
		// Issue 1126, cause C4: the own secrets whose two sides open to two values are read from the shown bodies, and
		// every body so masked is counted, own or not.
		// The ids are as a node runs them, its module's name first (captured from novox's shadow answer, 2026-10-28:
		// `only-office.needs-jwt`); another module's need, named here, is not nats's.
		drift := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("nats.needs-jwt", "box-1", "0600"),
			sealedFile("nats.needs-salt", "box-4", "0600"), sealedFile("nats.conf", "box-6", "0600"),
			sealedFile("other.needs-key", "box-8", "0600"))},
			[]Composed{comp(sealedFile("nats.needs-jwt", "box-2", "0600"), sealedFile("nats.needs-salt", "box-5", "0600"),
				sealedFile("nats.conf", "box-7", "0600"), sealedFile("other.needs-key", "box-9", "0600"))},
			func(a, b string) (same, known bool) { return a == "box-4", true })
		got := CandidateAnswer{Proposal: "p-x1", Revision: 1, Differs: drift.Differs}
		if own := OwnValuesDiffer(got); len(own) != 1 || own[0] != (OwnValueDiffers{Module: "nats", Name: "jwt"}) {
			t.Errorf("the own secrets with another value are %+v, want nats jwt alone", own)
		}
		if n := ValuesDiffer(got); n != 3 {
			t.Errorf("%d bodies hold another value, want 3 (jwt, conf and other's key)", n)
		}
		for id, want := range map[string]string{"nats.needs-jwt": "jwt", "needs-jwt": "jwt", "other.needs-jwt": "",
			"nats.conf": "", "nats.needs-": ""} {
			if name, own := OwnNeedName("nats", id); name != want && (own || want != "") {
				t.Errorf("%s names nats's own secret %q (%v), want %q", id, name, own, want)
			}
		}
		if w := WithValuesDiffer(map[string]CandidateAnswer{"g": got}); w["g"].ValuesDiffer != 3 {
			t.Errorf("the answer carries values_differ %d", w["g"].ValuesDiffer)
		}
		if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" {
			t.Errorf("the node's own composition run and not composed is removed: %+v", c)
		}
		if c := CompareComposed(map[string]Composed{"k": comp(file("conf", "a"))}, nil); len(c.Added) != 1 || c.Added[0].Module != "nats" || c.Added[0].Composed == "" {
			t.Errorf("a composition not run is added: %+v", c)
		}
		for name, a := range map[string]CandidateAnswer{
			"no head":             {Verdict: CandidateSame},
			"same with a diff":    {Proposal: "p-x1", Revision: 1, Verdict: CandidateSame, Differs: answers["differs"].Differs},
			"differs with none":   {Proposal: "p-x1", Revision: 1, Verdict: CandidateDiffers},
			"refused no reason":   {Proposal: "p-x1", Revision: 1, Verdict: CandidateRefused},
			"unknown verdict":     {Proposal: "p-x1", Revision: 1, Verdict: "maybe"},
			"diff with no module": {Proposal: "p-x1", Revision: 1, Verdict: CandidateDiffers, Differs: []ModuleDiffers{{Why: "x"}}},
			"same with an added":  {Proposal: "p-x1", Revision: 1, Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}},
			"differs refusing":    {Proposal: "p-x1", Revision: 1, Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}},
			"refused and modules": {Proposal: "p-x1", Revision: 1, Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}},
			"returns refusing":    {Proposal: "p-x1", Revision: 1, Verdict: CandidateRefused, Returns: ReturnsToController, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}},
			"returns refused":     {Proposal: "p-x1", Revision: 1, Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"},
		} {
			if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil {
				t.Errorf("%s: taken", name)
			}
		}
	})

	// ADR 0567 §2, §8: a proposal's changes laid over the current keys, what waits for the lease write, and the proposal's
	// encoding and refusals.
	t.Run("proposal", func(t *testing.T) {
		current := map[string]Placement{"g14": g14, "novox": novox}
		laid, findings := LayChanges(current, []PlacementProposalChange{
			{Op: OpPlace, Node: "g14", Module: "new"},
			{Op: OpTakeOff, Node: "g14", Module: "zsh"},
			{Op: OpUnpin, Node: "g14", Provision: "shell"},
			{Op: OpDecide, Node: "novox", Field: "site", Value: json.RawMessage(`"attic"`)},
		})
		if len(findings) != 0 || len(laid) != 2 || !reflect.DeepEqual(laid["g14"].Assigned(), []string{"app", "new"}) ||
			len(laid["g14"].Holds) != 0 || laid["g14"].Pins != nil || laid["novox"].Decided.Site != "attic" || !laid["novox"].Decided.Hub {
			t.Fatalf("laid %+v, %v", laid, findings)
		}
		if !reflect.DeepEqual(current["g14"].Assigned(), []string{"zsh", "app"}) {
			t.Error("laying changed the current placement")
		}
		if fresh, f := LayChanges(current, []PlacementProposalChange{{Op: OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" {
			t.Errorf("a node with no key: %+v, %v", fresh, f)
		}
		for name, c := range map[string]struct {
			change PlacementProposalChange
			want   string
		}{
			"not placed":      {PlacementProposalChange{Op: OpTakeOff, Node: "g14", Module: "nope"}, "not placed"},
			"no such field":   {PlacementProposalChange{Op: OpDecide, Node: "g14", Field: "colour", Value: json.RawMessage(`"x"`)}, "no decided field"},
			"wrong type":      {PlacementProposalChange{Op: OpDecide, Node: "g14", Field: "hub", Value: json.RawMessage(`"x"`)}, "does not take"},
			"unknown op":      {PlacementProposalChange{Op: "move", Node: "g14"}, "no change of placement"},
			"pin of nobody":   {PlacementProposalChange{Op: OpPin, Node: "g14", Provision: "x"}, "provider"},
			"not held":        {PlacementProposalChange{Op: OpUnhold, Node: "g14", Module: "app", Claim: "x"}, "holds no"},
			"bad port":        {PlacementProposalChange{Op: OpPlace, Node: "g14", Module: "x", Ports: map[string]int{"a": 1}}, "port"},
			"seed of another": {PlacementProposalChange{Op: OpSeed, Node: "g14", Whole: &novox}, "seed"},
		} {
			if _, f := LayChanges(current, []PlacementProposalChange{c.change}); len(f) != 1 || !strings.Contains(f[0], c.want) {
				t.Errorf("%s: %v, want a finding naming %q", name, f, c.want)
			}
		}

		const id = "p-20261028t183000-1a2b3c"
		if got := NewProposalID(time.Date(2026, 10, 28, 18, 30, 0, 0, time.UTC)); !strings.HasPrefix(got, "p-20261028t183000-") || CheckProposalID(got) != nil {
			t.Errorf("NewProposalID = %q", got)
		}
		p := PlacementProposal{ID: id, By: "operator@anchor", Why: "move the app", At: time.Unix(1, 0).UTC(),
			Changes: []PlacementProposalChange{{Op: OpTakeOff, Node: "g14", Module: "zsh"}}, Base: map[string]uint64{"g14": 4},
			State: ProposalAnswering}
		raw, err := EncodePlacementProposal(p)
		if err != nil {
			t.Fatal(err)
		}
		if back, err := DecodePlacementProposal(raw); err != nil || back.ID != id || back.Base["g14"] != 4 || !reflect.DeepEqual(back.Nodes(), []string{"g14"}) {
			t.Fatalf("round trip %+v, %v", back, err)
		}
		seed := p
		seed.Seed, seed.ExportDigest = true, ExportDigest(current)
		seed.Changes = []PlacementProposalChange{{Op: OpSeed, Node: "g14", Whole: &g14}}
		if _, err := EncodePlacementProposal(seed); err != nil {
			t.Errorf("the seed: %v", err)
		}
		if ExportDigest(current) != ExportDigest(map[string]Placement{"novox": novox, "g14": g14}) || ExportDigest(current) == ExportDigest(nil) {
			t.Error("the export digest depends on order or not on the placements")
		}
		back := p
		back.HandBack, back.Changes = true, nil
		if _, err := EncodePlacementProposal(back); err != nil {
			t.Errorf("a hand-back: %v", err)
		}
		if k, err := PulseAnswerKey(id, "g14"); err != nil || k != "answer."+id+".g14" {
			t.Errorf("PulseAnswerKey = %q, %v", k, err)
		}
		if k, err := PulseAskKey(id); err != nil || k != "ask."+id || PlacementPulsesBucket != "mesh-delivery-assigner_pulses" {
			t.Errorf("PulseAskKey = %q, %v", k, err)
		}
		if ChangesDigest(p) == ChangesDigest(seed) {
			t.Error("two proposals' changes digest alike")
		}
		for name, mut := range map[string]func(*PlacementProposal){
			"no id":                  func(p *PlacementProposal) { p.ID = "" },
			"no proposer":            func(p *PlacementProposal) { p.By = "" },
			"no why":                 func(p *PlacementProposal) { p.Why = " " },
			"no change":              func(p *PlacementProposal) { p.Changes = nil },
			"no base":                func(p *PlacementProposal) { p.Base = map[string]uint64{"novox": 1} },
			"no state":               func(p *PlacementProposal) { p.State = "" },
			"asked, no ask":          func(p *PlacementProposal) { p.State = ProposalAsked },
			"seed mixed":             func(p *PlacementProposal) { p.Seed, p.ExportDigest = true, "d" },
			"bad change":             func(p *PlacementProposal) { p.Changes[0].Op = "swap" },
			"seed, no digest":        func(p *PlacementProposal) { *p = seed; p.ExportDigest = "" },
			"hand-back with changes": func(p *PlacementProposal) { p.HandBack = true },
		} {
			q := p
			q.Changes = append([]PlacementProposalChange(nil), p.Changes...)
			mut(&q)
			if _, err := EncodePlacementProposal(q); err == nil {
				t.Errorf("%s: taken", name)
			}
		}
	})
}