Keep a consumer bound where its data is; only a pin moves it (hq ADR 0232, issue 273)

Issue 258's fix let a mesh seat's holder elsewhere answer before this machine's own provider. Right
for the resolver, which any provider answers alike; for the store's seat it re-bound every database
consumer on a machine running its own store to the holder on another, each was given a fresh, empty
database there, and nothing said so for twenty hours.

- An offer says whether it keeps its consumers' data (`keeps-consumer-data`); unsaid, a provider
  that grants each consumer a credential does. For such a provision the seat's holder no longer
  overrules a provider beside the consumer; a pin still does.
- Where each such consumer was sent is recorded (migration 0071). A resolution that would bind it
  elsewhere keeps the recorded provider and says the move; one whose provider is gone is refused,
  never answered by another.
- A push says a kept move and raises it as an urgent condition at once; the self-check's D12 raises
  it every run, with a pinned move not yet sent as a warning and any unasked move as urgent.

cmd/mesh-controller/bindings.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// A binding to data moves only by a person (novox/hq ADR 0232, issue 273).
+//
+// The resolver keeps each consumer of a provision that keeps its data at the provider it was last
+// sent (catalogue/bound.go) and says what it would have moved. This is where that is said: on the
+// push that composed it, and by the self-check's D12 every run, as an urgent condition naming the
+// consumer, both providers and the pin that confirms the move.
+
+// The condition kinds of D12.
+const (
+	// kindBindingKept is a move the resolver refused: the consumer is still where its data is.
+	kindBindingKept = "binding-kept"
+	// kindBindingMoved is a consumer about to be sent another provider than the one on record with
+	// no pin naming it — what the resolver exists to make impossible, said if it ever is not.
+	kindBindingMoved = "binding-moved"
+	// kindBindingMoving is a move a pin asked for, not yet sent: a person's act, said so that the
+	// data is moved before the push that carries it.
+	kindBindingMoving = "binding-moving"
+)
+
+// probeBindingsID is the self-check's id for this probe, and the source of what it raises.
+const probeBindingsID = "D12"
+
+// keptObservation is the urgent condition for one refused move.
+func keptObservation(k catalogue.KeptBinding) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: bindingID(k.Machine, k.Consumer, k.Provision),
+		Token: kindBindingKept, Kind: kindBindingKept, Machine: k.Machine, Also: otherMachines(k.Machine, k.Bound.Node, k.Would.Node),
+		Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+		Summary: fmt.Sprintf("on %s, the mesh %s", k.Machine, k.String())}
+}
+
+func bindingID(machine, consumer, provision string) string {
+	return machine + "." + consumer + "." + provision
+}
+
+func otherMachines(machine string, nodes ...string) []string {
+	var out []string
+	seen := map[string]bool{machine: true}
+	for _, n := range nodes {
+		if n != "" && !seen[n] {
+			seen[n] = true
+			out = append(out, n)
+		}
+	}
+	return out
+}
+
+// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
+// condition at once where this process keeps the conditions: a push is when a person is looking.
+func reportKept(ctx context.Context, plan catalogue.Resolution) {
+	for _, k := range plan.Kept {
+		fmt.Printf("%s: the mesh %s\n", plan.Node, k)
+		if conditionsFrom != nil {
+			o := keptObservation(k)
+			o.Source = probeBindingsID
+			if _, err := conditionsFrom.Observe(ctx, o); err != nil {
+				fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
+			}
+		}
+	}
+}
+
+// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
+// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
+// moves it (said, so the data goes first), and never anything else.
+func probeBindings(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	inv := d.open.inventory
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var out []conditions.Observation
+	for _, n := range nodes {
+		plan, _, err := planFor(ctx, d.open, n.Name)
+		if err != nil {
+			if unresolvable(err) {
+				// D1 says it, with the binding that refused it when that is why.
+				continue
+			}
+			return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
+		}
+		bound, err := inv.BindingsFor(ctx, n.Name)
+		if err != nil {
+			return nil, err
+		}
+		pins, err := inv.PinsFor(ctx, n.Name)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, bindingFindings(plan, bound, pins)...)
+	}
+	return out, nil
+}
+
+// bindingFindings is what one machine's resolution says against its record.
+func bindingFindings(plan catalogue.Resolution, bound map[string]map[string]catalogue.Chosen,
+	pins map[string]catalogue.Chosen) []conditions.Observation {
+	var out []conditions.Observation
+	for _, k := range plan.Kept {
+		out = append(out, keptObservation(k))
+	}
+	said := map[string]bool{}
+	for _, need := range plan.Needs {
+		if !need.KeepsData || need.ByRecord {
+			continue
+		}
+		was, recorded := bound[need.For][need.Name]
+		now := catalogue.Chosen{Node: need.From, Module: need.Module}
+		if !recorded || was == now || (was.Module == "" && was.Node == now.Node) {
+			continue
+		}
+		id := bindingID(plan.Node, need.For, need.Name)
+		if said[id] {
+			continue
+		}
+		said[id] = true
+		o := conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Machine: plan.Node,
+			Also: otherMachines(plan.Node, was.Node, now.Node), Resolver: conditions.ResolverOperator}
+		if pin, pinned := pins[need.Name]; pinned && pin.Node == now.Node && (pin.Module == "" || pin.Module == now.Module) {
+			o.Token, o.Kind, o.Severity = kindBindingMoving, kindBindingMoving, conditions.Warning
+			o.Summary = fmt.Sprintf("on %s, %s's %s moves from %s to %s at the next push, by the pin — its data "+
+				"is on %s: move it first", plan.Node, need.For, need.Name, was, now, was)
+		} else {
+			o.Token, o.Kind, o.Severity = kindBindingMoved, kindBindingMoved, conditions.Urgent
+			o.Summary = fmt.Sprintf("on %s, %s's %s would be sent %s, and it is bound to %s, where its data is, "+
+				"with no pin naming %s — a move nothing asked for", plan.Node, need.For, need.Name, now, was, now)
+		}
+		out = append(out, o)
+	}
+	return out
+}

cmd/mesh-controller/bindings_test.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// novox/hq issue 273, ADR 0232: a consumer of a provision that keeps its data moves only by a pin.
+
+func storeManifests() []catalogue.Manifest {
+	return []catalogue.Manifest{
+		{Module: "store", Version: "1",
+			Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
+			Claims:   []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
+			Serves:   map[string]map[string]any{"postgres-database": {"port": 5432}},
+			Grants:   map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
+		{Module: "resolver", Version: "1",
+			Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
+			Claims:   []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
+		{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
+		{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
+	}
+}
+
+func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) catalogue.Needed {
+	t.Helper()
+	for _, n := range plan.Needs {
+		if n.For == consumer && n.Name == provision {
+			return n
+		}
+	}
+	t.Fatalf("no %s for %s: %+v", provision, consumer, plan.Needs)
+	return catalogue.Needed{}
+}
+
+// The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's
+// store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the
+// binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it.
+func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range storeManifests() {
+		register(t, open, m)
+	}
+	assignAll := func(pairs ...[2]string) {
+		for _, a := range pairs {
+			if _, err := assign(ctx, open, a[0], a[1]); err != nil {
+				t.Fatalf("assign %s %s: %v", a[0], a[1], err)
+			}
+		}
+	}
+	// The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each.
+	assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
+	for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
+		if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
+		[2]string{"laptop", "board"})
+
+	plan, _, err := planFor(ctx, open, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData {
+		t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData)
+	}
+	if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData {
+		t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData)
+	}
+
+	// Sent, and recorded: only the binding to data.
+	bindings := boundToData(plan, nil)
+	if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) {
+		t.Fatalf("recorded %+v", bindings)
+	}
+	if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil {
+		t.Fatal(err)
+	}
+	if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
+		t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err)
+	}
+
+	// The laptop's store taken away: refused, not moved to the anchor's empty one.
+	if err := inv.Unassign(ctx, "laptop", "store"); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) ||
+		!strings.Contains(err.Error(), "board on laptop is bound to laptop/store") ||
+		!strings.Contains(err.Error(), "pin laptop postgres-database anchor store") {
+		t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err)
+	}
+
+	// A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was.
+	if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
+		t.Fatal(err)
+	}
+	plan, _, err = planFor(ctx, open, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" {
+		t.Fatalf("pinned to the anchor and bound to %s", n.From)
+	}
+	found, err := probeBindings(ctx, &doctor{open: open})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning ||
+		!strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") {
+		t.Fatalf("a pinned move is not said before it is sent: %+v", found)
+	}
+	if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil {
+		t.Fatal(err)
+	}
+	all, err := inv.Bindings(ctx)
+	if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" {
+		t.Fatalf("%+v, %v", all, err)
+	}
+	if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 {
+		t.Fatalf("a move sent is still said: %+v, %v", found, err)
+	}
+}
+
+// What one machine's resolution says against its record.
+func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
+	home, anchor := catalogue.Chosen{Node: "home", Module: "store"}, catalogue.Chosen{Node: "anchor", Module: "store"}
+	plan := catalogue.Resolution{Node: "laptop",
+		Kept: []catalogue.KeptBinding{{Machine: "laptop", Consumer: "board", Provision: "postgres-database",
+			Bound: home, Would: anchor}},
+		Needs: []catalogue.Needed{
+			{Name: "postgres-database", For: "board", From: "home", Module: "store", KeepsData: true},
+			{Name: "postgres-database", For: "game", From: "anchor", Module: "store", KeepsData: true},
+			{Name: "wildcard-resolution", For: "network", From: "anchor", Module: "resolver"},
+		}}
+	bound := map[string]map[string]catalogue.Chosen{
+		"board":   {"postgres-database": home},
+		"game":    {"postgres-database": home},
+		"network": {"wildcard-resolution": {Node: "home", Module: "resolver"}},
+	}
+	found := bindingFindings(plan, bound, nil)
+	if len(found) != 2 {
+		t.Fatalf("found %+v", found)
+	}
+	kept, moved := found[0], found[1]
+	if kept.Kind != kindBindingKept || kept.Severity != conditions.Urgent || kept.Machine != "laptop" ||
+		!strings.Contains(kept.Summary, "would move board's postgres-database from home/store to anchor/store") ||
+		!strings.Contains(kept.Summary, "its data is on home/store") ||
+		!strings.Contains(kept.Summary, "`pin laptop postgres-database anchor store` to confirm a move (and move the data first)") {
+		t.Errorf("kept: %+v", kept)
+	}
+	if moved.Kind != kindBindingMoved || moved.Severity != conditions.Urgent ||
+		!strings.Contains(moved.Summary, "game's postgres-database would be sent anchor/store") {
+		t.Errorf("moved: %+v", moved)
+	}
+	if kept.Key() == moved.Key() {
+		t.Error("two consumers, one condition")
+	}
+}
+
+// A push says the move it refused, and raises its condition at once.
+func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
+		Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
+		Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
+	open, err := k.Open(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
+		open[0].Source != probeBindingsID {
+		t.Fatalf("raised %+v", open)
+	}
+}

cmd/mesh-controller/doctor.go

@@ -99,6 +99,9 @@ var probeRegistry = []probe{
 		"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
 	{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
 		"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
+	{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
+		"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
+		Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
 	{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
 		From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
 }

cmd/mesh-controller/held_back.go

@@ -232,6 +232,7 @@ func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held
 			return sendable{}, err
 		}
 		reportUnhostable(node, plan)
+		reportKept(held, plan)
 		declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
 		if err == nil {
 			reportLeftOut(node, declared)

cmd/mesh-controller/plan.go

@@ -90,6 +90,12 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 	if err != nil {
 		return catalogue.Resolution{}, nil, err
 	}
+	// Where each of its consumers of a provision that keeps data was last sent (novox/hq ADR 0232):
+	// a resolution that would answer one from anywhere else keeps it there, and says so.
+	world.Bound, err = inv.BindingsFor(ctx, nodeName)
+	if err != nil {
+		return catalogue.Resolution{}, nil, err
+	}
 
 	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
 	if err != nil {
@@ -419,10 +425,35 @@ func declarationWith(ctx context.Context, open *stores, node string,
 			names = append(names, m.Module)
 		}
 		out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
+		out.Bindings = boundToData(plan, composed.LeftOut)
 	}
 	return out, nil
 }
 
+// boundToData is every binding of this machine's consumers to a provision that keeps their data
+// (novox/hq ADR 0232), as a send records it. Not a consumer left out of the declaration: the machine
+// is not told anything new about it, so nothing about where it is bound has been sent.
+func boundToData(plan catalogue.Resolution, leftOut map[string]string) []inventory.Binding {
+	var out []inventory.Binding
+	seen := map[[2]string]bool{}
+	for _, n := range plan.Needs {
+		if !n.KeepsData || n.ByRecord || n.Module == "" {
+			continue
+		}
+		if _, left := leftOut[n.For]; left {
+			continue
+		}
+		key := [2]string{n.For, n.Name}
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		out = append(out, inventory.Binding{Machine: plan.Node, Consumer: n.For, Provision: n.Name,
+			Provider: catalogue.Chosen{Node: n.From, Module: n.Module}})
+	}
+	return out
+}
+
 // carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
 // issue 259): the module's current build for each module in it, and for a module left out of it
 // (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when

cmd/mesh-controller/push.go

@@ -512,6 +512,7 @@ func pushCommand(ctx context.Context, args []string) error {
 		// healthy modules beside it are still resolved and sent. Reported so it is not silently
 		// dropped — the remedy is to move it, and until then the rest of the node converges.
 		reportUnhostable(node, plan)
+		reportKept(held, plan)
 		unheld[node] = plan.Unheld
 		// The private network is in here with everything else. It used to be composed separately
 		// and prepended, which meant every machine with an address was on it and no machine could
@@ -847,6 +848,17 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
 	if err != nil {
 		return "", err
 	}
+	if len(s.declared.Bindings) > 0 {
+		// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
+		// what the next resolution keeps it at. On the same outliving context as the send's record.
+		kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+		err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings)
+		cancel()
+		if err != nil {
+			return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+
+				"data could not be recorded: %w", s.node, err)
+		}
+	}
 	if s.declared.BusUsers != "" {
 		// And the user list it carried, so the next send reads whether it must go first from the
 		// list alone (novox/hq issue 249). On the same outliving context as the send's record.
@@ -1007,6 +1019,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
 			continue
 		}
 		reportUnhostable(name, plan)
+		reportKept(ctx, plan)
 		declared, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
 		if err != nil {
 			refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))

cmd/mesh-controller/sendable.go

@@ -55,6 +55,9 @@ type sendable struct {
 	// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
 	// Composed only on the send path; nil records that it is not known.
 	Builds map[string]string
+	// Bindings is where each consumer of a provision that keeps its data is bound in this declaration
+	// (novox/hq ADR 0232), recorded with the send and never on the wire. Composed only on the send path.
+	Bindings []inventory.Binding
 }
 
 // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on