The store's mirror of the assignments may not write while they are switching or moved, asked before every write (issue 1093)

Review round 2. The writer's leave to write is the lease's and AssignmentsMirrorMayWrite, a flag set before the settle of the assignments' switch, kept after the commit and cleared only if the switch fails, so a pass that read its source before the switch lays and purges nothing after it.

cmd/mesh-controller/record_mirrors.go

@@ -63,7 +63,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
 				if m.Name == recordmirror.Assignments.Name && onTheBus {
 					continue
 				}
-				why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
+				why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
 				if why != failing[m.Name] {
 					if why != "" {
 						fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
@@ -83,7 +83,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
 }
 
 // mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
-func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
+func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
 	source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
 	holds func() bool) string {
 	reading, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -95,6 +95,9 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir
 			return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
 		}
 		w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
+		if m.Name == recordmirror.Assignments.Name {
+			w.Holds = assignmentsHolds(inv, holds)
+		}
 		writers[m.Name] = w
 	}
 	want, err := source(reading)
@@ -193,3 +196,10 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
 	events.known = now
 	return true
 }
+
+// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
+// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
+// writes nothing after it (ADR 0408 §4).
+func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
+	return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
+}

cmd/mesh-controller/record_mirrors_switch_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/recordmirror"
+)
+
+// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
+// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
+// switching or moved.
+func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
+	inv := inventory.NewForTest()
+	w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
+	if !w.Holds() {
+		t.Fatal("the writer may not write before the switch")
+	}
+	inv.BeginSwitchForTest(inventory.KindAssignments)
+	if w.Holds() {
+		t.Fatal("the writer may write while the assignments are switching")
+	}
+	// Refused before it reads the bucket (it has none here) or lays or ends a key.
+	r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
+	if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
+		t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
+	}
+}