Copy node.sent_builds onto the assignments' keys once, and keep the column; list the new store tests (issues 1109, 1096)

Delivery 1 wrote no commit on the key of a pulled-in module, so after the upgrade those modules fell out of SentBuilds. At the start of the lease holder a copy fills every key that names no commit (creating a standing pulled-in key where none stands), overwrites nothing, empties each column it copied, and raises an urgent condition while it fails. Migration 0105 no longer drops node.sent_builds (issue 1112 does).

cmd/mesh-controller/chain.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/novox/mesh-controller/internal/chain"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 )
@@ -43,6 +44,8 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
 	inv.UseChain(c)
 	// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
 	go askAgainSoon(ctx, inv)
+	// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
+	go copySentBuildsSoon(ctx, inv)
 	return nil
 }
 
@@ -87,3 +90,43 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
 		askAgain(ctx, inv)
 	}
 }
+
+// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
+// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
+const kindSentBuildsNotCopied = "sent-builds-not-copied"
+
+// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
+// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
+// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
+// that names a commit and empties each column it has copied.
+func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
+	failed := false
+	for {
+		_, err := inv.CopySentBuilds(ctx)
+		o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
+			Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+			Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
+		if err != nil {
+			fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
+			if keeper := conditionsFrom; keeper != nil && !failed {
+				o.Summary += err.Error()
+				if _, oerr := keeper.Observe(ctx, o); oerr == nil {
+					failed = true
+				}
+			}
+		} else {
+			if keeper := conditionsFrom; keeper != nil && failed {
+				_, _ = keeper.Clear(ctx, o.Key(), "copied")
+			}
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(chainMoveRetry):
+		}
+	}
+}
+
+// chainMoveRetry is how long the copy waits before it tries again.
+const chainMoveRetry = 30 * time.Second

cmd/mesh-controller/chain_no_table_integration_test.go

@@ -13,8 +13,7 @@ import (
 // **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
 // 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
 // build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
-// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the column that held what
-// a machine was sent, nor the function that raised the generation from the assignment table; the table that stays, stays.
+// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
 func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
 	open := windowed(t)
 	everyHeadWaits(t)
@@ -80,9 +79,6 @@ func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing
 			t.Errorf("the table %s, which stays, is gone", table)
 		}
 	}
-	if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_schema = 'public' and column_name = 'sent_builds'`); rows[0][0] != "0" {
-		t.Errorf("a column sent_builds is in the store: %v", rows)
-	}
 	if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
 		t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
 	}