Land feature 57's controller work on its release branch

The controller plans config, templates and moved providers' provisions,
hands bus accounts to the vault and judges the new manifest words, on the
feature's SDK and node-engine validator (feature 57, config and provisioning).

cmd/mesh-controller/push.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
@@ -194,7 +195,7 @@ func serve(ctx context.Context) (err error) {
 	// And what the catalogue decided a build meant. The builder's own result is already handled
 	// above; this is the other half — the control plane is the only one of the three that knows
 	// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
-	if err := server.Follows(following{open}); err != nil {
+	if err := server.Follows(following{open: open}); err != nil {
 		return err
 	}
 	// And the merges the bus announced and never handed over, read back on a timer and acted on late
@@ -202,7 +203,7 @@ func serve(ctx context.Context) (err error) {
 	go catchingUpOnMerges(ctx, open, server)
 	// And a catalogue that has just started, asking for what it missed. The same type answers
 	// both: what a build meant and what the builds were are two questions about one record.
-	if err := server.Answers(following{open}); err != nil {
+	if err := server.Answers(following{open: open}); err != nil {
 		return err
 	}
 	// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
@@ -256,7 +257,7 @@ func serve(ctx context.Context) (err error) {
 			go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
 		}
 	}
-	if err := server.Checks(following{open}); err != nil {
+	if err := server.Checks(following{open: open}); err != nil {
 		return err
 	}
 	// Composed now and kept current, before the verb that answers from it is served.
@@ -264,6 +265,12 @@ func serve(ctx context.Context) (err error) {
 	// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
 	// controller holds the lease, read by the build seat from the artifact store.
 	go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
+	// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
+	// declared config on the bus, written from the store only when a record changed, by the lease holder.
+	// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
+	// assignment.
+	inv.SetProvisioningPlans(inv.SentProvisioningPlans)
+	go mirroringRecords(ctx, inv, bus.Conn, bus)
 	// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
 	// to-be 45 §6).
 	link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
@@ -296,6 +303,8 @@ func serve(ctx context.Context) (err error) {
 		return err
 	}
 	defer stopCLI()
+	// Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762).
+	go sayReadCalls(ctx)
 	// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
 	stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
 	if err != nil {
@@ -1386,6 +1395,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 	if issued > 0 {
 		fmt.Printf("  issued %d membership(s)\n", issued)
 	}
+	// **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's
+	// setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again
+	// as last issued, with only what it hears changed, so nothing else of it moves before its own send.
+	reheard := reissueHearing(ctx, bus, records, where, sent)
+	if reheard > 0 {
+		fmt.Printf("  issued %d membership(s) again for what they hear of a module's stream\n", reheard)
+	}
 	if len(refused) > 0 {
 		// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
 		// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
@@ -1685,7 +1701,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
 	// send that is away look failed: it is said, loudly, and the machine's own record stands.
 	err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
 		Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
-		Digest: digest, Modules: sent.modules})
+		Digest: digest, Modules: sent.modules, Provisioning: sent.plans})
 	var unrecorded *inventory.SendNotRecordedError
 	switch {
 	case errors.As(err, &unrecorded):
@@ -1744,3 +1760,78 @@ func schemaReach() (int, error) {
 	}
 	return reach, nil
 }
+
+// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose
+// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus,
+// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said
+// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it.
+func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements,
+	sent []readyNode) int {
+	reached := map[string]bool{}
+	for _, s := range sent {
+		reached[s.node] = true
+	}
+	issued := 0
+	for node, declared := range records.Assigned {
+		if reached[node] {
+			continue
+		}
+		for _, d := range declared {
+			if !d.HearsAStream {
+				continue
+			}
+			subject := broker.MembershipSubject(node, d.Module)
+			last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject)
+			if err != nil {
+				if !errors.Is(err, nats.ErrMsgNotFound) {
+					fmt.Printf("  %s on %s: its membership could not be read to issue what it hears again: %v\n",
+						d.Module, node, err)
+				}
+				continue
+			}
+			body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d))
+			if err != nil {
+				fmt.Printf("  %s on %s: its membership could not be read to issue what it hears again: %v\n",
+					d.Module, node, err)
+				continue
+			}
+			if !changed {
+				continue
+			}
+			if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
+				fmt.Printf("  %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err)
+				continue
+			}
+			issued++
+		}
+	}
+	return issued
+}
+
+// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure.
+func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) {
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal(issued, &fields); err != nil {
+		return nil, false, err
+	}
+	var had []broker.HeardIssued
+	if raw, ok := fields["hears"]; ok {
+		if err := json.Unmarshal(raw, &had); err != nil {
+			return nil, false, err
+		}
+	}
+	if slices.Equal(had, hears) {
+		return issued, false, nil
+	}
+	if len(hears) == 0 {
+		delete(fields, "hears")
+	} else {
+		raw, err := json.Marshal(hears)
+		if err != nil {
+			return nil, false, err
+		}
+		fields["hears"] = raw
+	}
+	body, err := json.Marshal(fields)
+	return body, true, err
+}