Answer a placement candidate as ADR 0559 has it: added and removed modules, per-module refusals, both digests of a difference, and a node's read of every candidate key (issue 1126)

go/buscompose/state.go

@@ -312,12 +312,15 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
 const ResolvedBucket = "mesh-config_resolved"
 
 // NodeComposeReads is what a node's agent reads to compose its own modules (novox/hq ADR 0547 §2, ADR 0549 §3, issue
-// 1126): every key of the indexer's placement, modules and manifests and of the builder's builds, and of mesh-config's
-// resolved values only its own node's, `<node>.>`. Reads only; the node is checked as every subject token is by the
+// 1126): every key of the indexer's placement, placement candidates, modules and manifests and of the builder's builds,
+// and of mesh-config's resolved values only its own node's, `<node>.>`. Every candidate key, not only `*.<node>`: a node
+// answering its candidate composes with the whole group's placement (the seed places every node at once, and a binding
+// crosses nodes), and nothing in a placement is secret. Reads only; the node is checked as every subject token is by the
 // caller's principal.
 func NodeComposeReads(node string) []string {
 	var out []string
-	for _, bucket := range []string{delivery.PlacementBucket, delivery.ModulesBucket, delivery.ManifestsBucket, delivery.BuilderBuildsBucket} {
+	for _, bucket := range []string{delivery.PlacementBucket, delivery.PlacementCandidatesBucket, delivery.ModulesBucket,
+		delivery.ManifestsBucket, delivery.BuilderBuildsBucket} {
 		out = append(out, readGrants(bucket, "")...)
 	}
 	return append(out, readGrants(ResolvedBucket, node+".>")...)

go/compose/composed.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 
@@ -18,7 +19,8 @@ import (
 // (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not
 // signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest).
 func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) {
-	return compositions(in, node, fps, mode, false)
+	out, skipped, _, err = compositions(in, node, fps, mode, false)
+	return out, skipped, err
 }
 
 // ErrStoodIn is a composition that holds a stand-in for a value the input does not carry (an own secret, a grant's
@@ -34,10 +36,21 @@ const BusUsersStandIn = "the bus's user list"
 // ErrStoodIn, naming each, so a stand-in is never applied as if it were the value; but for the bus's user list
 // (BusUsersStandIn), which the bus node holds out of every apply of a composition it made itself.
 func CompositionsApplied(in Input, node string, fps map[string]string) (map[string]delivery.Composed, []string, error) {
-	return compositions(in, node, fps, Reading, true)
+	out, skipped, _, err := compositions(in, node, fps, Reading, true)
+	return out, skipped, err
 }
 
-func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]delivery.Composed, skipped []string, err error) {
+// CompositionsShadow is CompositionsApplied for a composition that is compared and never applied (a node's answer to a
+// placement candidate, novox/hq ADR 0559 §4): composed in the Foreseeing mode, so an own secret never issued (a module
+// the candidate adds) is a named stand-in rather than an error, and what it stands in for is answered (stood, sorted,
+// the bus's user list left out) rather than refused, so the caller judges each stand-in: a value not yet issued for a
+// module the candidate adds, or one missing for a module the node runs. A module whose every value is held composes
+// as CompositionsApplied composes it: the modes differ only in an own secret never issued.
+func CompositionsShadow(in Input, node string, fps map[string]string) (out map[string]delivery.Composed, skipped, stood []string, err error) {
+	return compositions(in, node, fps, Foreseeing, false)
+}
+
+func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]delivery.Composed, skipped, stood []string, err error) {
 	reading := in
 	reading.Nodes = append([]Node(nil), in.Nodes...)
 	found := false
@@ -48,36 +61,34 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 		}
 	}
 	if !found {
-		return nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
+		return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
 	}
 	d, problems, err := ComposeIn(reading, node, mode)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if len(problems) > 0 {
-		return nil, nil, fmt.Errorf("compose: %s's declaration is refused by the node-engine's validator: %v", node, problems)
+		return nil, nil, nil, fmt.Errorf("compose: %s's declaration is refused by the node-engine's validator: %v", node, problems)
 	}
-	if applied {
-		var stood []string
-		for _, x := range d.StoodIn {
-			// The bus's user list is the bus node's own to follow, written alone and never from a composition the node
-			// made itself (ADR 0547 §7, §8): its stand-in is held out of the apply, not applied.
-			if x != BusUsersStandIn {
-				stood = append(stood, x)
-			}
-		}
-		stood = append(stood, d.Foreseen...)
-		for _, p := range d.ForeseenPorts {
-			stood = append(stood, "port "+p)
-		}
-		if len(stood) > 0 {
-			sort.Strings(stood)
-			return nil, nil, fmt.Errorf("%w on %s: %s", ErrStoodIn, node, strings.Join(stood, ", "))
+	for _, x := range d.StoodIn {
+		// The bus's user list is the bus node's own to follow, written alone and never from a composition the node
+		// made itself (ADR 0547 §7, §8): its stand-in is held out of the apply, not applied.
+		if x != BusUsersStandIn {
+			stood = append(stood, x)
 		}
 	}
+	stood = append(stood, d.Foreseen...)
+	for _, p := range d.ForeseenPorts {
+		stood = append(stood, "port "+p)
+	}
+	sort.Strings(stood)
+	stood = slices.Compact(stood) // an own secret is both stood in for and foreseen
+	if applied && len(stood) > 0 {
+		return nil, nil, nil, fmt.Errorf("%w on %s: %s", ErrStoodIn, node, strings.Join(stood, ", "))
+	}
 	body, err := d.Body()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	var envelope struct {
 		Resources   []json.RawMessage     `json:"resources"`
@@ -86,7 +97,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 		Provisioner string                `json:"provisioner_account"`
 	}
 	if err := json.Unmarshal(body, &envelope); err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	var kept []json.RawMessage
 	keptPaths := map[string]bool{}
@@ -98,7 +109,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 			Path   string `json:"path"`
 		}
 		if err := json.Unmarshal(raw, &head); err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		if _, ok := fps[head.Module]; head.Module != "" && !ok {
 			left[head.Module] = true
@@ -118,7 +129,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 	out, err = delivery.SplitComposed(node, kept, fps, filled,
 		delivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	for key, c := range out {
 		if m, ok := in.Catalogue[c.Module]; ok && c.Module != "" {
@@ -130,5 +141,5 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
 		skipped = append(skipped, m)
 	}
 	sort.Strings(skipped)
-	return out, skipped, nil
+	return out, skipped, stood, nil
 }

go/compose/frombus/answer.go

@@ -0,0 +1,138 @@
+package frombus
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/compose"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// Answer is a node's answer to its group's placement candidates (novox/hq ADR 0559 §4): s.Node composed in shadow with
+// s.Placements laid over the trunk's placement (compose.CompositionsShadow, its own sealed values and boxes through r as
+// for its own composition), compared with what it runs (delivery.CompareComposed). It applies nothing and writes
+// nothing; head and revision are the candidate's, said back in the answer.
+//
+// fps gives the build each module is composed at: what the node runs, and for a module the candidate places that the
+// node does not run, the build the trunk holds; a placed module with none is added, with no build of it known. What the
+// composition stands in for is judged: a value of a module the group adds (on this node or another: its credential, own
+// secrets and boxes are made only once it is placed, ADR 0549 §5) is not yet issued, which is no refusal; any other
+// stand-in is a value missing for a module the node runs, a refusal naming it ("node" when it is the node's own).
+func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]string, running []delivery.Composed, head string, revision uint64) delivery.CandidateAnswer {
+	a := delivery.CandidateAnswer{Head: head, Revision: revision}
+	refuse := func(err error) delivery.CandidateAnswer {
+		a.Refused = err.Error()
+		a.Verdict = delivery.CandidateRefused
+		return a
+	}
+	trunk, err := r.Placements(ctx)
+	if err != nil {
+		return refuse(fmt.Errorf("frombus: placement: %w", err))
+	}
+	in, err := Input(ctx, r, mesh, s)
+	if err != nil {
+		return refuse(err)
+	}
+	asConsumer, asProvider, err := compose.CredentialsNeeded(in, s.Node)
+	if err != nil {
+		return refuse(err)
+	}
+	if len(asConsumer)+len(asProvider) > 0 {
+		if in.Credentials, in.Issued, err = r.Credentials(ctx, s.Node, asConsumer, asProvider); err != nil {
+			return refuse(fmt.Errorf("frombus: the credentials of %s: %w", s.Node, err))
+		}
+	}
+	composed, skipped, stood, err := compose.CompositionsShadow(in, s.Node, fps)
+	if err != nil {
+		return refuse(err)
+	}
+	cmp := delivery.CompareComposed(composed, running)
+	a.Same, a.Differs, a.Removed = cmp.Same, cmp.Differs, cmp.Removed
+
+	// What the group adds, by node and module: placed by a candidate and not by the trunk on that node.
+	added := map[string]bool{}
+	for node, p := range s.Placements {
+		if p == nil {
+			continue
+		}
+		was := map[string]bool{}
+		for _, m := range trunk[node].Modules {
+			was[m.Module] = true
+		}
+		for _, m := range p.Modules {
+			if !was[m.Module] {
+				added[node+"/"+m.Module] = true
+			}
+		}
+	}
+	unissued := map[string][]string{}
+	refusals := map[string][]string{}
+	for _, x := range stood {
+		node, module := standsInFor(x, s.Node)
+		switch {
+		case module != "" && added[node+"/"+module]:
+			if node == s.Node {
+				unissued[module] = append(unissued[module], x)
+			} // another node's addition: the module here that carries it differs, which the assigner judges
+		case module == "" || node != s.Node:
+			// The node's own, or a value this node holds for a module elsewhere (a grant it provides): said as the node's.
+			refusals[delivery.HaveNodeWord] = append(refusals[delivery.HaveNodeWord], x)
+		default:
+			refusals[module] = append(refusals[module], x)
+		}
+	}
+	addedHere := map[string]delivery.ModuleAdded{}
+	for _, m := range cmp.Added {
+		addedHere[m.Module] = m
+	}
+	for _, m := range skipped { // placed, and no build given to compose it at
+		addedHere[m] = delivery.ModuleAdded{Module: m, Unissued: []string{"no build of " + m + " is known to compose it at"}}
+	}
+	for m, xs := range unissued {
+		got := addedHere[m]
+		got.Module = m
+		got.Unissued = append(got.Unissued, xs...)
+		addedHere[m] = got
+	}
+	for _, m := range sortedKeys(addedHere) {
+		a.Added = append(a.Added, addedHere[m])
+	}
+	for _, m := range sortedKeys(refusals) {
+		if _, isAdded := addedHere[m]; isAdded {
+			continue // a module the candidate adds here: what it lacks is said under Added
+		}
+		a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")})
+	}
+	sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module })
+	a.Verdict = delivery.VerdictOf(a)
+	return a
+}
+
+var (
+	grantStandIn = regexp.MustCompile(`^the grant of \S+ for (\S+) on (\S+)$`)
+	needsStandIn = regexp.MustCompile(`^(\S+) on (\S+) needs `)
+)
+
+// standsInFor is the node and module a stand-in of compose's words is for: an own secret `module/name`, a port
+// `port module:port`, a grant `the grant of <provision> for <module> on <node>`, a need `<module> on <node> needs ...`;
+// the module empty when the words name none (the certificate, the authority).
+func standsInFor(x, node string) (string, string) {
+	if m := grantStandIn.FindStringSubmatch(x); m != nil {
+		return m[2], m[1]
+	}
+	if m := needsStandIn.FindStringSubmatch(x); m != nil {
+		return m[2], m[1]
+	}
+	if rest, ok := strings.CutPrefix(x, "port "); ok {
+		if module, _, ok := strings.Cut(rest, ":"); ok {
+			return node, module
+		}
+	}
+	if module, _, ok := strings.Cut(x, "/"); ok && !strings.Contains(x, " ") {
+		return node, module
+	}
+	return node, ""
+}

go/compose/frombus/frombus_test.go

@@ -191,8 +191,8 @@ func TestInput(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if same, differs := delivery.CompareComposed(asIs, running); len(differs) != 0 || len(same) != len(running) {
-		t.Errorf("the candidate equal to the placement: same %v, differs %+v", same, differs)
+	if c := delivery.CompareComposed(asIs, running); len(c.Differs)+len(c.Added)+len(c.Removed) != 0 || len(c.Same) != len(running) {
+		t.Errorf("the candidate equal to the placement: %+v", c)
 	}
 	moved := mk().placements["anchor"]
 	moved.Decided.Endpoint = "anchor.example:51821"
@@ -200,11 +200,76 @@ func TestInput(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, differs := delivery.CompareComposed(changed, running); len(differs) != 1 || differs[0].Module != overlay.Name || differs[0].Resource != overlay.Name+".overlay-config" {
-		t.Errorf("the hub's endpoint changed: differs %+v, want %s alone, naming its config", differs, overlay.Name)
+	if c := delivery.CompareComposed(changed, running); len(c.Differs) != 1 || c.Differs[0].Module != overlay.Name || c.Differs[0].Resource != overlay.Name+".overlay-config" {
+		t.Errorf("the hub's endpoint changed: %+v, want %s alone, naming its config", c, overlay.Name)
 	}
-	if _, differs := delivery.CompareComposed(changed, running[:0]); len(differs) != len(changed) {
-		t.Errorf("nothing runs: differs %+v", differs)
+	if c := delivery.CompareComposed(changed, running[:0]); len(c.Added) != len(changed) {
+		t.Errorf("nothing runs: %+v", c)
+	}
+
+	// ADR 0559 §4, the node's answer: the same; one module differing; a module the group adds, whose own secret is not
+	// yet issued (no refusal); a value missing for a module the node runs (a refusal); a module the candidate removes.
+	keeper := []byte(`{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},` +
+		`"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}`)
+	lemurs := []byte(`{"module":"lemurs","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`)
+	one := func(placed ...string) *bus {
+		p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}}
+		for _, m := range placed {
+			p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
+		}
+		return &bus{placements: map[string]delivery.Placement{"n1": p},
+			records:   map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: keys["anchor"], SealingKey: "sk", Capabilities: []string{"systemd"}}},
+			modules:   map[string]delivery.Module{"keeper": {Manifest: delivery.ManifestDigest(keeper)}, "lemurs": {Manifest: delivery.ManifestDigest(lemurs)}},
+			manifests: map[string][]byte{delivery.ManifestDigest(keeper): keeper, delivery.ManifestDigest(lemurs): lemurs},
+			own:       []compose.OwnSecretRow{{Node: "n1", Module: "keeper", Name: "password", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}}
+	}
+	oneFps := map[string]string{"keeper": "src2:" + strings.Repeat("d", 64), "lemurs": "src2:" + strings.Repeat("e", 64)}
+	runs, _, err := Compose(ctx, one("keeper", "lemurs"), Mesh{}, Scope{Node: "n1"}, oneFps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var runsNow []delivery.Composed
+	for _, c := range runs {
+		runsNow = append(runsNow, c)
+	}
+	placedAs := func(modules ...string) map[string]*delivery.Placement {
+		p := one(modules...).placements["n1"]
+		return map[string]*delivery.Placement{"n1": &p}
+	}
+	for name, c := range map[string]struct {
+		b       *bus
+		given   map[string]*delivery.Placement
+		running []delivery.Composed
+		verdict string
+		check   func(delivery.CandidateAnswer) bool
+	}{
+		"the same": {one("keeper", "lemurs"), placedAs("keeper", "lemurs"), runsNow, delivery.CandidateSame,
+			func(a delivery.CandidateAnswer) bool { return len(a.Same) == 3 && a.Head == "h1" && a.Revision == 7 }},
+		"one removed": {one("keeper", "lemurs"), placedAs("keeper"), runsNow, delivery.CandidateDiffers,
+			func(a delivery.CandidateAnswer) bool { return reflect.DeepEqual(a.Removed, []string{"lemurs"}) }},
+		"one added, its secret not yet issued": {func() *bus { b := one("lemurs"); b.own = nil; return b }(), placedAs("keeper", "lemurs"),
+			nil, delivery.CandidateDiffers, func(a delivery.CandidateAnswer) bool {
+				for _, m := range a.Added {
+					if m.Module == "keeper" {
+						return reflect.DeepEqual(m.Unissued, []string{"keeper/password"}) && len(a.Refusals) == 0
+					}
+				}
+				return false
+			}},
+		"a value missing for a module that runs": {func() *bus { b := one("keeper", "lemurs"); b.own = nil; return b }(), placedAs("keeper", "lemurs"),
+			runsNow, delivery.CandidateRefused, func(a delivery.CandidateAnswer) bool {
+				return len(a.Refusals) == 1 && a.Refusals[0].Module == "keeper" && strings.Contains(a.Refusals[0].Why, "keeper/password")
+			}},
+		"cannot compose": {one("keeper"), map[string]*delivery.Placement{"n1": {Node: "n2"}}, runsNow, delivery.CandidateRefused,
+			func(a delivery.CandidateAnswer) bool { return a.Refused != "" }},
+	} {
+		a := Answer(ctx, c.b, Mesh{}, Scope{Node: "n1", Placements: c.given}, oneFps, c.running, "h1", 7)
+		if a.Verdict != c.verdict || !c.check(a) {
+			t.Errorf("%s: %+v", name, a)
+		}
+		if raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil {
+			t.Errorf("%s: the answer is not one a have holds: %v (%s)", name, err, raw)
+		}
 	}
 	if got, err := Placed(want, map[string]*delivery.Placement{"laptop": &laptop}); err != nil || !reflect.DeepEqual(got, want) {
 		t.Errorf("Placed with the node's own placement changes the input: %v", err)

go/delivery/composed_compare.go

@@ -9,15 +9,25 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 )
 
+// Comparison is what CompareComposed finds, module by module: the same, differing, added by what was composed (not run
+// now) and removed by it (run now, not composed). Each list is sorted by module.
+type Comparison struct {
+	Same    []string
+	Differs []ModuleDiffers
+	Added   []ModuleAdded
+	Removed []string
+}
+
 // CompareComposed compares the compositions a node composed (by key, as compose.Compositions answers them) with the ones
-// it runs, as its agent would (issue 1126: a node's answer to a placement candidate). A module is the same when its
+// it runs, as its agent would (ADR 0559 §4: a node's answer to a placement candidate). A module is the same when its
 // composition's digest is the one it runs (ComposedDigestOf, the signature left out), so the agent would apply nothing
-// of it; it differs when the digests differ, when it is composed and not run, or run and not composed. The node's own
-// composition is named HaveNodeWord. The bus's user list (catalogue.BusUsersID) is left out on both sides: the bus node
-// follows it alone and never applies it from a composition it made (ADR 0547 §2, §8). A difference names the first
-// resource, by its id, whose content differs or that one side alone holds; when every resource is equal it says what
-// else differs (their order, the files filled, the data, the node's own parts, a file carried by digest).
-func CompareComposed(composed map[string]Composed, running []Composed) (same []string, differs []ModuleDiffers) {
+// of it; it differs when the digests differ (both named); it is added when composed and not run, removed when run and
+// not composed. The node's own composition is named HaveNodeWord. The bus's user list (catalogue.BusUsersID) is left out
+// on both sides: the bus node follows it alone and never applies it from a composition it made (ADR 0547 §2, §8). A
+// difference names the first resource, by its id, whose content differs or that one side alone holds; when every
+// resource is equal it says what else differs (their order, the files filled, the data, the node's own parts, a file
+// carried by digest).
+func CompareComposed(composed map[string]Composed, running []Composed) Comparison {
 	name := func(c Composed) string {
 		if c.Module == "" {
 			return HaveNodeWord
@@ -39,42 +49,47 @@ func CompareComposed(composed map[string]Composed, running []Composed) (same []s
 	for m := range theirs {
 		modules[m] = true
 	}
+	var out Comparison
 	for _, m := range sortedKeys(modules) {
 		a, composedHere := ours[m]
 		b, runs := theirs[m]
 		switch {
 		case !runs:
-			differs = append(differs, ModuleDiffers{Module: m, Why: "composed, and the node does not run it"})
+			dig, _ := ComposedDigestOf(withoutBusUsers(a))
+			out.Added = append(out.Added, ModuleAdded{Module: m, Composed: dig})
 		case !composedHere:
-			differs = append(differs, ModuleDiffers{Module: m, Why: "the node runs it, and it is not composed"})
+			out.Removed = append(out.Removed, m)
 		default:
 			if d, ok := compareOne(m, a, b); ok {
-				same = append(same, m)
+				out.Same = append(out.Same, m)
 			} else {
-				differs = append(differs, d)
+				out.Differs = append(out.Differs, d)
 			}
 		}
 	}
-	return same, differs
+	return out
 }
 
 // compareOne is one module's comparison: true when the digests agree, else what differs first.
 func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) {
 	composed, running = withoutBusUsers(composed), withoutBusUsers(running)
 	dc, errC := ComposedDigestOf(composed)
-	if errC != nil {
-		return ModuleDiffers{Module: module, Why: "the composition has no digest: " + errC.Error()}, false
-	}
 	dr, errR := ComposedDigestOf(running)
+	if errC != nil {
+		return ModuleDiffers{Module: module, Running: dr, Why: "the composition has no digest: " + errC.Error()}, false
+	}
 	if errR == nil && dc == dr {
 		return ModuleDiffers{}, true
 	}
+	differs := func(resource, why string) (ModuleDiffers, bool) {
+		return ModuleDiffers{Module: module, Composed: dc, Running: dr, Resource: resource, Why: why}, false
+	}
 	// Words for the difference: resource by resource, each side with its large files by digest, so a file carried whole
 	// and the same file carried by digest read as one.
 	a, _, errA := composed.ByDigest()
 	b, _, errB := running.ByDigest()
 	if errA != nil || errB != nil {
-		return ModuleDiffers{Module: module, Why: "the compositions differ and cannot be read resource by resource"}, false
+		return differs("", "the compositions differ and cannot be read resource by resource")
 	}
 	ra, rb := resourcesByID(a.Resources), resourcesByID(b.Resources)
 	ids := map[string]bool{}
@@ -89,29 +104,29 @@ func compareOne(module string, composed, running Composed) (ModuleDiffers, bool)
 		y, inB := rb[id]
 		switch {
 		case !inB:
-			return ModuleDiffers{Module: module, Resource: id, Why: "composed, and not in what runs"}, false
+			return differs(id, "composed, and not in what runs")
 		case !inA:
-			return ModuleDiffers{Module: module, Resource: id, Why: "in what runs, and not composed"}, false
+			return differs(id, "in what runs, and not composed")
 		case !bytes.Equal(x, y):
-			return ModuleDiffers{Module: module, Resource: id, Why: "the resource differs"}, false
+			return differs(id, "the resource differs")
 		}
 	}
 	switch {
 	case !jsonEqual(a.Filled, b.Filled):
-		return ModuleDiffers{Module: module, Why: "the files the provisioner fills differ"}, false
+		return differs("", "the files the provisioner fills differ")
 	case a.Data != b.Data:
-		return ModuleDiffers{Module: module, Why: "whether it declares data differs"}, false
+		return differs("", "whether it declares data differs")
 	case !jsonEqual(a.Own, b.Own):
-		return ModuleDiffers{Module: module, Why: "the node's own parts differ"}, false
+		return differs("", "the node's own parts differ")
 	case a.FP != b.FP:
-		return ModuleDiffers{Module: module, Why: "the fingerprint differs"}, false
+		return differs("", "the fingerprint differs")
 	}
 	if dA, err := ComposedDigestOf(a); err == nil {
 		if dB, err := ComposedDigestOf(b); err == nil && dA == dB {
-			return ModuleDiffers{Module: module, Why: "the same content, a large file carried whole on one side and by digest on the other"}, false
+			return differs("", "the same content, a large file carried whole on one side and by digest on the other")
 		}
 	}
-	return ModuleDiffers{Module: module, Why: "the same resources in another order"}, false
+	return differs("", "the same resources in another order")
 }
 
 // withoutBusUsers is c without the bus's user list resource (its id `bus-users`, or `<module>.bus-users`).

go/delivery/placement_candidate.go

@@ -107,58 +107,104 @@ func DecodePlacementCandidate(data []byte) (PlacementCandidate, error) {
 const (
 	// CandidateSame: every composition the node composes with the candidate is the one it runs, and it runs no other.
 	CandidateSame = "same"
-	// CandidateDiffers: some composition differs, is composed and not run, or is run and not composed (Differs).
+	// CandidateDiffers: some composition differs (Differs), is added by the candidate (Added) or removed by it (Removed);
+	// whether the change intends it is the assigner's to judge (ADR 0559 §5).
 	CandidateDiffers = "differs"
-	// CandidateRefused: the node cannot compose with the candidate (Refused says why: an unbound requirement, a stand-in,
-	// a manifest it cannot read).
+	// CandidateRefused: a value is missing for a module the node runs (Refusals), or the node cannot compose with the
+	// candidate at all (Refused says why: an unbound requirement, a manifest it cannot read).
 	CandidateRefused = "refused"
 )
 
-// CandidateAnswer is a node's answer to its candidate of one group, in HaveNode.Candidates: the head and the key
-// revision it answered, its verdict, the modules composed the same, those that differ, or why it could not compose.
-// The node's own composition is named "node" (HaveNodeWord) in Same and Differs.
+// CandidateAnswer is a node's answer to its group's candidate keys, in HaveNode.Candidates (ADR 0559 §4): the head and
+// the key revision it answered, its verdict, the modules composed the same, those that differ with both digests, those
+// the candidate adds (naming the values not yet issued for each, which is no refusal: a module's credential and boxes
+// are made only once it is placed, ADR 0549 §5) or removes, and each refusal. The node's own composition is named
+// "node" (HaveNodeWord).
 type CandidateAnswer struct {
 	Head     string          `json:"head"`
 	Revision uint64          `json:"revision"`
 	Verdict  string          `json:"verdict"`
 	Same     []string        `json:"same,omitempty"`
 	Differs  []ModuleDiffers `json:"differs,omitempty"`
-	Refused  string          `json:"refused,omitempty"`
+	Added    []ModuleAdded   `json:"added,omitempty"`
+	Removed  []string        `json:"removed,omitempty"`
+	Refusals []ModuleRefused `json:"refusals,omitempty"`
+	// Refused is why the node could not compose with the candidate at all; Refusals is then empty.
+	Refused string `json:"refused,omitempty"`
 }
 
-// ModuleDiffers is one module whose composition with the candidate is not the one the node runs: the first resource
-// that differs by its identity (declaration.Resource's Identity), or empty when the whole module is composed and not run
-// or run and not composed, and why in words.
+// ModuleDiffers is one module the node runs and the candidate still places whose composition is not the one it runs:
+// both digests (Composed with the candidate, Running what runs), the first resource that differs by its id, or empty
+// when every resource is equal, and why in words.
 type ModuleDiffers struct {
 	Module   string `json:"module"`
+	Composed string `json:"composed,omitempty"`
+	Running  string `json:"running,omitempty"`
 	Resource string `json:"resource,omitempty"`
 	Why      string `json:"why"`
 }
 
+// ModuleAdded is one module the candidate places on the node and the node does not run: its composition's digest when
+// it composed, and the values not yet issued for it (its own secrets, its grants, a host port), each in the words the
+// composition stood in for it with.
+type ModuleAdded struct {
+	Module   string   `json:"module"`
+	Composed string   `json:"composed,omitempty"`
+	Unissued []string `json:"unissued,omitempty"`
+}
+
+// ModuleRefused is a value missing for a module the node runs, in the words the composition stood in for it with.
+type ModuleRefused struct {
+	Module string `json:"module"`
+	Why    string `json:"why"`
+}
+
+// VerdictOf is the verdict of an answer's lists: refused on any refusal, differs on any difference, addition or
+// removal, same otherwise.
+func VerdictOf(a CandidateAnswer) string {
+	switch {
+	case a.Refused != "" || len(a.Refusals) > 0:
+		return CandidateRefused
+	case len(a.Differs)+len(a.Added)+len(a.Removed) > 0:
+		return CandidateDiffers
+	}
+	return CandidateSame
+}
+
 func (a CandidateAnswer) check() error {
 	if a.Head == "" {
 		return fmt.Errorf("delivery: a candidate answer names no head")
 	}
 	switch a.Verdict {
-	case CandidateSame:
-		if len(a.Differs) > 0 || a.Refused != "" {
-			return fmt.Errorf("delivery: a candidate answered same names a difference or a refusal")
-		}
-	case CandidateDiffers:
-		if len(a.Differs) == 0 || a.Refused != "" {
-			return fmt.Errorf("delivery: a candidate answered differs names no difference, or a refusal")
-		}
-	case CandidateRefused:
-		if a.Refused == "" || len(a.Same)+len(a.Differs) > 0 {
-			return fmt.Errorf("delivery: a candidate answered refused says no reason, or names modules")
-		}
+	case CandidateSame, CandidateDiffers, CandidateRefused:
 	default:
 		return fmt.Errorf("delivery: the candidate verdict %q is not same, differs or refused", a.Verdict)
 	}
+	if got := VerdictOf(a); got != a.Verdict {
+		return fmt.Errorf("delivery: a candidate answered %s names what makes it %s", a.Verdict, got)
+	}
+	if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 {
+		return fmt.Errorf("delivery: a candidate the node could not compose names modules")
+	}
 	for _, d := range a.Differs {
 		if d.Module == "" || d.Why == "" {
 			return fmt.Errorf("delivery: a candidate difference names no module or no why")
 		}
 	}
+	for _, d := range a.Added {
+		if d.Module == "" {
+			return fmt.Errorf("delivery: a module the candidate adds is not named")
+		}
+	}
+	for _, d := range a.Refusals {
+		if d.Module == "" || d.Why == "" {
+			return fmt.Errorf("delivery: a candidate refusal names no module or no why")
+		}
+	}
+	for _, m := range a.Removed {
+		if m == "" {
+			return fmt.Errorf("delivery: a module the candidate removes is not named")
+		}
+	}
 	return nil
 }

go/delivery/placement_test.go

@@ -227,6 +227,8 @@ func TestPlacement(t *testing.T) {
 			"same":    {Head: "h1", Revision: 3, Verdict: CandidateSame, Same: []string{"app", "node"}},
 			"differs": {Head: "h1", Revision: 3, Verdict: CandidateDiffers, Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}},
 			"refused": {Head: "h1", Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"},
+			"added":   {Head: "h1", Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}},
+			"a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}},
 		}
 		n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers}
 		raw, err := EncodeHaveNode(n)
@@ -257,17 +259,21 @@ func TestPlacement(t *testing.T) {
 			"one side":        {comp(file("conf", "a"), file("more", "x")), comp(file("conf", "a")), "more", "composed, and not in what runs"},
 			"order":           {comp(file("conf", "a"), file("more", "x")), comp(file("more", "x"), file("conf", "a")), "", "the same resources in another order"},
 		} {
-			same, differs := CompareComposed(map[string]Composed{"novox.nats." + fp: tc.composed}, []Composed{tc.running})
+			c := CompareComposed(map[string]Composed{"novox.nats." + fp: tc.composed}, []Composed{tc.running})
 			if tc.why == "" {
-				if len(differs) != 0 || len(same) != 1 {
-					t.Errorf("%s: same %v, differs %+v", name, same, differs)
+				if len(c.Differs) != 0 || len(c.Same) != 1 {
+					t.Errorf("%s: %+v", name, c)
 				}
-			} else if len(differs) != 1 || differs[0].Resource != tc.resource || differs[0].Why != tc.why {
-				t.Errorf("%s: differs %+v, want %s: %s", name, differs, tc.resource, tc.why)
+			} else if len(c.Differs) != 1 || c.Differs[0].Resource != tc.resource || c.Differs[0].Why != tc.why ||
+				c.Differs[0].Composed == "" || c.Differs[0].Running == "" || c.Differs[0].Composed == c.Differs[0].Running {
+				t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why)
 			}
 		}
-		if _, differs := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(differs) != 1 || differs[0].Module != "node" {
-			t.Errorf("the node's own composition run and not composed: %+v", differs)
+		if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" {
+			t.Errorf("the node's own composition run and not composed is removed: %+v", c)
+		}
+		if c := CompareComposed(map[string]Composed{"k": comp(file("conf", "a"))}, nil); len(c.Added) != 1 || c.Added[0].Module != "nats" || c.Added[0].Composed == "" {
+			t.Errorf("a composition not run is added: %+v", c)
 		}
 		for name, a := range map[string]CandidateAnswer{
 			"no head":             {Verdict: CandidateSame},
@@ -276,6 +282,9 @@ func TestPlacement(t *testing.T) {
 			"refused no reason":   {Head: "h", Verdict: CandidateRefused},
 			"unknown verdict":     {Head: "h", Verdict: "maybe"},
 			"diff with no module": {Head: "h", Verdict: CandidateDiffers, Differs: []ModuleDiffers{{Why: "x"}}},
+			"same with an added":  {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}},
+			"differs refusing":    {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}},
+			"refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}},
 		} {
 			if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil {
 				t.Errorf("%s: taken", name)