diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 3140aa9..6b46dd0 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -137,7 +137,26 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod // should hold, seals every secret in them and publishes them, and on a first node that is the // slowest thing the mesh does. func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { - said, err := control.within(o.Wait).tell(ctx, "push", o.Node) + return pushing(ctx, o, control, false, say) +} + +// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was +// sent (`push --wait`, novox/hq ADR 0010): for a push whose content the very next step reads +// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a +// moment later (issue 615). +func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + return pushing(ctx, o, control, true, say) +} + +func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) { + args := []string{"push", o.Node} + patience := o.Wait + if applied { + args = append(args, "--wait", o.Wait.String()) + // The controller's own wait, and time to compose and send before it starts. + patience = 2 * o.Wait + } + said, err := control.within(patience).tell(ctx, args...) if err != nil { return "", fmt.Errorf( "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ @@ -145,7 +164,11 @@ func pushNode(ctx context.Context, o Options, control controlPlane, say func(str "at all. Fix what it named and run this installer again: it will find the module "+ "already registered and try the push again", err) } - say(" pushed " + o.Node) + if applied { + say(" pushed " + o.Node + ", and it applied it") + } else { + say(" pushed " + o.Node) + } // **Every push may carry a new bus user, and until the adoption the bus hears of it only from here** // (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's // records and sealed to this node with the push; the bus reads its user list from a file, and until diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index 6a5db0a..e60036b 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -415,7 +415,12 @@ func deliverGenesisRegistry(ctx context.Context, o Options, control controlPlane } // Push so the sealed document reaches the build agent, which reads it at its next build — and, // like every push before the adoption, the bus hears the users it carries (issue 608). - _, err = pushNode(ctx, o, control, say) + // + // **Waited for until the node applied it** (issue 615). The next step asks the build agent to + // publish the SDK, and the agent reads the document from the file the node writes when it applies + // this push. A push that returned on sending let the ask race the apply: one genesis found the + // document in time, the next was refused, "neither its binding nor the genesis document". + _, err = pushNodeApplied(ctx, o, control, say) return err } diff --git a/internal/bootstrap/push_places_users_test.go b/internal/bootstrap/push_places_users_test.go index 77d5f44..d42912a 100644 --- a/internal/bootstrap/push_places_users_test.go +++ b/internal/bootstrap/push_places_users_test.go @@ -42,7 +42,7 @@ func placedAfterThePush(t *testing.T, commands []string) int { t.Helper() pushed, placed := -1, 0 for i, c := range commands { - if strings.HasSuffix(c, "push anchor") && pushed < 0 { + if strings.Contains(c, "/mesh-controller push anchor") && pushed < 0 { pushed = i } if strings.HasPrefix(c, "sh -c") && strings.Contains(c, busAccounts) && @@ -95,3 +95,40 @@ func TestOnceTheBusIsTheNatsModulesThePushPlacesNothing(t *testing.T) { t.Fatalf("the installer placed the bus's users over the nats module's: %v", runtime.commands) } } + +// Defends novox/hq issue 615: the packages step's push is waited for until the node applied it, because +// the sdk step right after asks the build agent for a build that reads the genesis document from the +// file that apply writes. A push that returned on sending let the ask race the apply. +func TestThePackagesStepWaitsUntilTheNodeAppliedTheGenesisDocument(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: 3 * time.Minute} + if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil { + t.Fatal(err) + } + for _, c := range runtime.commands { + if strings.Contains(c, "/mesh-controller push anchor") { + if !strings.HasSuffix(c, "push anchor --wait 3m0s") { + t.Fatalf("the genesis document's push does not wait for the node to apply it: %s", c) + } + return + } + } + t.Fatalf("the genesis document was never pushed: %v", runtime.commands) +} + +// Every other push of genesis still returns on sending: only a push whose content the next step reads on +// the machine waits. +func TestAnOrdinaryPushDoesNotWait(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + for _, c := range runtime.commands { + if strings.Contains(c, "push anchor --wait") { + t.Fatalf("an ordinary push waits: %s", c) + } + } +}