Keep placement.<node> in step after each commit, so a reader of the form before alone follows every write and the delivery's members need no order (issue 774, research 072 row 37)

go/delivery/placement_keys.go

@@ -30,8 +30,10 @@ import (
 //
 // The form before, one value `placement.<node>` per node (LegacyPlacementKey), is still read during the delivery of
 // this change, so readers and the holder may move in any order: a node with a node key is read from its new keys alone,
-// one without from its legacy key. The holder converts each legacy value once at its start (PlacementKeys.Migrations)
-// and deletes a legacy key with its node key, so a placement handed back never returns from it.
+// one without from its legacy key. The holder converts each legacy value once at its start (PlacementKeys.Migrations),
+// keeps the legacy key in step after every commit as a mirror, so a reader of the form before alone still follows every
+// write while the members of the delivery move in any order, and deletes it first when a node's placement is deleted,
+// so a placement handed back never returns from it. A later change stops the mirror and deletes `placement.*`.
 const (
 	// PlacementNodeWord is the last word of a node key, `<node>.node`.
 	PlacementNodeWord  = HaveNodeWord
@@ -428,13 +430,18 @@ type PlacementKeyWrite struct {
 	Base  uint64
 	// Commit says this write is the node key's: once it is written, the node's placement is the new one.
 	Commit bool
+	// Mirror says this write is the legacy key's, `placement.<node>`, kept in step after the commit for the readers not
+	// yet delivered: a reader of both forms never reads it once the node key is there, so a mirror write that fails
+	// harms no such reader, and is made again at the node's next write.
+	Mirror bool
 }
 
 // Writes are the writes that take node from the placement its keys hold to after (nil: no placement), in the order
 // they are made: the module keys that change, each carrying the value it replaces; then the node key, the commit; then
-// the deletes of the module keys after no longer names. For no placement the legacy key goes first, then the node key,
-// then the module keys, so a crash between them never leaves the legacy value to be read again. A module key whose value
-// stays is not written.
+// the legacy key `placement.<node>` with the whole placement, a mirror for the readers not yet delivered, unless it
+// holds it already; then the deletes of the module keys after no longer names. For no placement the legacy key goes
+// first, then the node key, then the module keys, so a crash between them never leaves the legacy value to be read
+// again. A module key whose value stays is not written.
 func (k *PlacementKeys) Writes(node string, after *Placement) ([]PlacementKeyWrite, error) {
 	nodeKey, err := PlacementNodeKey(node)
 	if err != nil {
@@ -509,12 +516,22 @@ func (k *PlacementKeys) Writes(node string, after *Placement) ([]PlacementKeyWri
 		return nil, err
 	}
 	out = append(out, PlacementKeyWrite{Node: node, Key: nodeKey, Value: raw, Base: of.nodeRev, Commit: true})
+	// The mirror of the form before, while readers of it alone may run (the delivery of this change); a later change
+	// stops writing it and deletes it.
+	if of.legacy == nil || !SamePlacement(*of.legacy, *after) {
+		legacy, err := EncodePlacement(*after)
+		if err != nil {
+			return nil, err
+		}
+		key, _ := LegacyPlacementKey(node)
+		out = append(out, PlacementKeyWrite{Node: node, Key: key, Value: legacy, Base: of.legacyRev, Mirror: true})
+	}
 	deleteModules(keep)
 	return out, nil
 }
 
 // Migrations are, by node, the writes that convert every node held only in the form before (its legacy key, no node
-// key) into its node and module keys, its legacy key left as it is for the readers not yet delivered (Writes); empty
+// key) into its node and module keys, its legacy key kept as the mirror for the readers not yet delivered (Writes); empty
 // once every node is converted, so the holder's conversion at each start is idempotent.
 func (k *PlacementKeys) Migrations() (map[string][]PlacementKeyWrite, error) {
 	k.mu.Lock()
@@ -548,8 +565,8 @@ func (k *PlacementKeys) Legacy(node string) bool {
 	return of != nil && of.node == nil && of.legacy != nil
 }
 
-// PlacementValues is every key and value of p written whole onto no keys before: its module keys and its node key, as
-// a writer that holds no key of the node (a test's bus, a fresh mesh) writes them.
+// PlacementValues is every key and value of p written whole onto no keys before: its module keys, its node key and the
+// legacy mirror, as a writer that holds no key of the node (a test's bus, a fresh mesh) writes them.
 func PlacementValues(p Placement) (map[string][]byte, error) {
 	writes, err := NewPlacementKeys().Writes(p.Node, &p)
 	if err != nil {

go/delivery/placement_test.go

@@ -172,7 +172,8 @@ func TestPlacement(t *testing.T) {
 	// a write, read beside the form before, and converted from it once.
 	t.Run("one key per module and node", func(t *testing.T) {
 		values, err := PlacementValues(g14)
-		if err != nil || len(values) != 3 || values["g14.node"] == nil || values["g14.zsh"] == nil || values["g14.app"] == nil {
+		if err != nil || len(values) != 4 || values["g14.node"] == nil || values["g14.zsh"] == nil || values["g14.app"] == nil ||
+			values["placement.g14"] == nil {
 			t.Fatalf("PlacementValues = %v, %v", values, err)
 		}
 		got, err := PlacementsOf(values)
@@ -211,6 +212,7 @@ func TestPlacement(t *testing.T) {
 		}
 		migrations, err := keys.Migrations()
 		if err != nil || len(migrations["g14"]) != 3 || !migrations["g14"][2].Commit {
+			// The legacy key holds the placement already: no mirror write.
 			t.Fatalf("Migrations = %+v, %v", migrations, err)
 		}
 		apply(migrations["g14"], -1)
@@ -233,7 +235,8 @@ func TestPlacement(t *testing.T) {
 		for _, w := range writes {
 			kinds = append(kinds, fmt.Sprintf("%s:%v:%d", w.Key, w.Value != nil, w.Base))
 		}
-		if strings.Join(kinds, " ") != "g14.app:true:3 g14.new:true:0 g14.node:true:4 g14.zsh:false:2" {
+		if strings.Join(kinds, " ") != "g14.app:true:3 g14.new:true:0 g14.node:true:4 placement.g14:true:1 g14.zsh:false:2" ||
+			!writes[3].Mirror {
 			t.Fatalf("Writes = %v", kinds)
 		}
 		apply(writes, 2)
@@ -244,7 +247,10 @@ func TestPlacement(t *testing.T) {
 		if p, _, _ := keys.Of("g14"); !SamePlacement(p, after) {
 			t.Errorf("the write done is not read whole: %+v", p)
 		}
-		// A module key that stays is not written again.
+		if legacy, _ := PlacementsOf(map[string][]byte{"placement.g14": writes[3].Value}); !SamePlacement(legacy["g14"], after) {
+			t.Errorf("the mirror is not the placement written: %+v", legacy)
+		}
+		// A module key that stays is not written again, nor a mirror that holds it.
 		if again, _ := keys.Writes("g14", &after); len(again) != 1 || !again[0].Commit {
 			t.Errorf("an unchanged placement writes %+v", again)
 		}
@@ -258,7 +264,7 @@ func TestPlacement(t *testing.T) {
 		}
 		// The holder's next write puts the module key right again.
 		repair, _ := keys.Writes("g14", &after)
-		if len(repair) != 2 || repair[0].Key != "g14.app" || repair[0].Base != 99 {
+		if len(repair) != 2 || repair[0].Key != "g14.app" || repair[0].Base != 99 || !repair[1].Commit {
 			t.Fatalf("the repair writes %+v", repair)
 		}
 		apply(repair, -1)