Wait until the node applied the genesis document before the sdk step asks for a build (hq issue 615)

The packages step sealed the build agent its genesis document and pushed, and the push returned as soon
as it was sent. The sdk step then asked for the SDK's build at once, and the agent reads the document
from the file the node writes when it applies that push: one genesis found it in time, the next was
refused for having neither a binding nor the document. That one push now waits until the node reports
it applied it (push --wait); every other push still returns on sending.

internal/bootstrap/module.go

@@ -137,7 +137,26 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod
 // should hold, seals every secret in them and publishes them, and on a first node that is the
 // slowest thing the mesh does.
 func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
-	said, err := control.within(o.Wait).tell(ctx, "push", o.Node)
+	return pushing(ctx, o, control, false, say)
+}
+
+// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was
+// sent (`push <node> --wait`, novox/hq ADR 0010): for a push whose content the very next step reads
+// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a
+// moment later (issue 615).
+func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
+	return pushing(ctx, o, control, true, say)
+}
+
+func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) {
+	args := []string{"push", o.Node}
+	patience := o.Wait
+	if applied {
+		args = append(args, "--wait", o.Wait.String())
+		// The controller's own wait, and time to compose and send before it starts.
+		patience = 2 * o.Wait
+	}
+	said, err := control.within(patience).tell(ctx, args...)
 	if err != nil {
 		return "", fmt.Errorf(
 			"%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+
@@ -145,7 +164,11 @@ func pushNode(ctx context.Context, o Options, control controlPlane, say func(str
 				"at all. Fix what it named and run this installer again: it will find the module "+
 				"already registered and try the push again", err)
 	}
-	say("  pushed            " + o.Node)
+	if applied {
+		say("  pushed            " + o.Node + ", and it applied it")
+	} else {
+		say("  pushed            " + o.Node)
+	}
 	// **Every push may carry a new bus user, and until the adoption the bus hears of it only from here**
 	// (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's
 	// records and sealed to this node with the push; the bus reads its user list from a file, and until

internal/bootstrap/phase_packages.go

@@ -415,7 +415,12 @@ func deliverGenesisRegistry(ctx context.Context, o Options, control controlPlane
 	}
 	// Push so the sealed document reaches the build agent, which reads it at its next build — and,
 	// like every push before the adoption, the bus hears the users it carries (issue 608).
-	_, err = pushNode(ctx, o, control, say)
+	//
+	// **Waited for until the node applied it** (issue 615). The next step asks the build agent to
+	// publish the SDK, and the agent reads the document from the file the node writes when it applies
+	// this push. A push that returned on sending let the ask race the apply: one genesis found the
+	// document in time, the next was refused, "neither its binding nor the genesis document".
+	_, err = pushNodeApplied(ctx, o, control, say)
 	return err
 }
 

internal/bootstrap/push_places_users_test.go

@@ -42,7 +42,7 @@ func placedAfterThePush(t *testing.T, commands []string) int {
 	t.Helper()
 	pushed, placed := -1, 0
 	for i, c := range commands {
-		if strings.HasSuffix(c, "push anchor") && pushed < 0 {
+		if strings.Contains(c, "/mesh-controller push anchor") && pushed < 0 {
 			pushed = i
 		}
 		if strings.HasPrefix(c, "sh -c") && strings.Contains(c, busAccounts) &&
@@ -95,3 +95,40 @@ func TestOnceTheBusIsTheNatsModulesThePushPlacesNothing(t *testing.T) {
 		t.Fatalf("the installer placed the bus's users over the nats module's: %v", runtime.commands)
 	}
 }
+
+// Defends novox/hq issue 615: the packages step's push is waited for until the node applied it, because
+// the sdk step right after asks the build agent for a build that reads the genesis document from the
+// file that apply writes. A push that returned on sending let the ask race the apply.
+func TestThePackagesStepWaitsUntilTheNodeAppliedTheGenesisDocument(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	runtime, control := placing(false)
+	o := Options{Node: "anchor", Wait: 3 * time.Minute}
+	if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range runtime.commands {
+		if strings.Contains(c, "/mesh-controller push anchor") {
+			if !strings.HasSuffix(c, "push anchor --wait 3m0s") {
+				t.Fatalf("the genesis document's push does not wait for the node to apply it: %s", c)
+			}
+			return
+		}
+	}
+	t.Fatalf("the genesis document was never pushed: %v", runtime.commands)
+}
+
+// Every other push of genesis still returns on sending: only a push whose content the next step reads on
+// the machine waits.
+func TestAnOrdinaryPushDoesNotWait(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	runtime, control := placing(false)
+	o := Options{Node: "anchor", Wait: time.Second}
+	if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range runtime.commands {
+		if strings.Contains(c, "push anchor --wait") {
+			t.Fatalf("an ordinary push waits: %s", c)
+		}
+	}
+}