Feature 57 (config and provisioning): release mesh-controller, the reviewed release branch merged with main, on top of main

cmd/mesh-controller/record_mirrors.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/recordmirror"
+)
+
+// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments,
+// and each registered module's declared config, which mesh-config resolves a module's config values from.
+//
+// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a
+// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made,
+// so no one call site sees every change; what every change has in common is that it is committed to the
+// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
+// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
+// mirror is therefore at most mirrorEvery behind its store.
+
+// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
+const mirrorEvery = 10 * time.Second
+
+// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
+func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
+	api, err := jetstream.New(conn)
+	if err != nil {
+		fmt.Printf("the record mirrors are not kept: %v\n", err)
+		return
+	}
+	holds := func() bool {
+		epoch, err := theLease.epoch(ctx)
+		return err == nil && epoch > 0
+	}
+	epoch := func() uint64 {
+		e, _ := theLease.epoch(ctx)
+		return e
+	}
+	sources := map[string]func(context.Context) (map[string][]byte, error){
+		recordmirror.Assignments.Name:    inv.AssignmentsMirror,
+		recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
+		recordmirror.Predefined.Name:     inv.PredefinedMirror,
+	}
+	writers := map[string]*recordmirror.Writer{}
+	failing := map[string]string{}
+	tick := time.NewTicker(mirrorEvery)
+	defer tick.Stop()
+	for {
+		if holds() {
+			for _, m := range recordmirror.Mirrors() {
+				why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
+				if why != failing[m.Name] {
+					if why != "" {
+						fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
+					} else {
+						fmt.Printf("the %s mirror is kept again\n", m.Name)
+					}
+					failing[m.Name] = why
+				}
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+	}
+}
+
+// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
+func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
+	source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
+	holds func() bool) string {
+	reading, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	w := writers[m.Name]
+	if w == nil {
+		kv, err := api.KeyValue(reading, m.Bucket)
+		if err != nil {
+			return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
+		}
+		w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
+		writers[m.Name] = w
+	}
+	want, err := source(reading)
+	if err != nil {
+		return fmt.Sprintf("the store cannot be read: %v", err)
+	}
+	lost := w.LostEvents
+	r, err := w.Sync(reading, want)
+	if r.Written+r.Removed > 0 {
+		fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
+	}
+	if w.LostEvents > lost {
+		fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
+			m.Name, w.LostEvents-lost)
+	}
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}