Keep assignments in the assignments bucket and move the rows there at start (issue 1093)

Once chain_moved records the kind, Assign, Unassign and Assigned write and read the keys <node>.<module>, and every join of the store that read assignments is made from the bucket: where a module runs, what is behind, what a machine awaits, a given value due for replacement, taking a module, and whether a module may be forgotten (refused while a standing assignment names it). An unassignment ends the key with a marker and keeps its earlier values, as it removed the row; no ended or pending mark is written yet. The generation rises with each change, and a seat held by an assignment goes with it: the foreign key from seat_holding is dropped by migration 0104 and the controller says both. The provisioner's plan is kept in the values by a pass of the tick that writes that one field by compare-and-set; the assignment-changed event is said from what the bucket holds, whoever wrote it. The start moves the rows with what each machine was last sent, keeping a plan the mirror put in a value.

cmd/mesh-controller/record_mirrors.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 
+	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/recordmirror"
 )
@@ -49,9 +52,17 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
 	failing := map[string]string{}
 	tick := time.NewTicker(mirrorEvery)
 	defer tick.Stop()
+	events := &assignmentEvents{}
 	for {
 		if holds() {
+			// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
+			// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
+			// writes them only until the move.
+			onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
 			for _, m := range recordmirror.Mirrors() {
+				if m.Name == recordmirror.Assignments.Name && onTheBus {
+					continue
+				}
 				why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
 				if why != failing[m.Name] {
 					if why != "" {
@@ -104,3 +115,81 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir
 	}
 	return ""
 }
+
+// assignmentEvents is the assignments as the bucket held them at the last look, to say what changed since.
+type assignmentEvents struct {
+	known map[string]inventory.KeptValue
+}
+
+// keepingAssignments is the tick's work for the assignments once they are on the bus: the provisioner's plan written into
+// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
+// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
+func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
+	events *assignmentEvents, failing map[string]string) bool {
+	reading, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	const name = "assignments"
+	if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
+		if err != nil {
+			// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
+			if failing[name] == "" {
+				failing[name] = err.Error()
+				fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
+			}
+			return true
+		}
+		return false
+	}
+	if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
+		if failing[name] != err.Error() {
+			failing[name] = err.Error()
+			fmt.Printf("the provisioner's plans are not kept in the assignments: %v\n", err)
+		}
+	} else if failing[name] != "" {
+		fmt.Printf("the provisioner's plans are kept in the assignments again\n")
+		failing[name] = ""
+	}
+	now, _, err := inv.KeptAssignments(reading)
+	if err != nil {
+		return true
+	}
+	if events.known != nil {
+		keys := make([]string, 0, len(now)+len(events.known))
+		for key := range now {
+			keys = append(keys, key)
+		}
+		for key := range events.known {
+			if _, still := now[key]; !still {
+				keys = append(keys, key)
+			}
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			was, had := events.known[key]
+			is, has := now[key]
+			e := recordmirror.Event{Mirror: recordmirror.Assignments.Name, Key: key, Revision: is.Revision, Epoch: epoch(),
+				At: time.Now().UTC()}
+			switch {
+			case has && !had:
+				e.Change = recordmirror.Added
+			case has && had && was.Revision != is.Revision:
+				e.Change, e.Fields = recordmirror.Changed, recordmirror.ChangedFields(was.Value, is.Value)
+			case !has && had:
+				e.Change, e.Revision = recordmirror.Removed, 0
+			default:
+				continue
+			}
+			body, err := json.Marshal(e)
+			if err != nil {
+				continue
+			}
+			for try := 0; try < 2; try++ {
+				if err = announce.PublishSeatEvent(reading, broker.ControllerSeat, recordmirror.Assignments.Event, body); err == nil {
+					break
+				}
+			}
+		}
+	}
+	events.known = now
+	return true
+}