Keep builds in the builds bucket and move the build rows there at start (issue 1090)

Once chain_moved records the kind, RecordBuild and every build read go through the chain (reuse key and published added); before that the tables are read as before. The serving controller copies the rows at its start, holding the lease: idempotent per key, rows counted against keys, the largest value measured, a refusal (urgent condition, nothing switched) on a count that differs or a value over the bus's largest, and a re-read when the table took a row meanwhile. ForTest attaches the chain in memory with builds moved; ForTestBefore leaves it unmoved. Build requests, gates and the other kinds are still in their tables.

cmd/mesh-controller/chain.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/chain"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
+// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
+// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
+// has recorded it (inventory.ChainReads).
+
+// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
+func chainHolds(ctx context.Context) error {
+	_, err := theLease.epoch(ctx)
+	return err
+}
+
+// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command.
+func openTheChain(ctx context.Context) (*chain.Chain, error) {
+	js, err := aBus()
+	if err != nil {
+		return nil, err
+	}
+	// Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command.
+	return chain.OnTheBus(ctx, js.Conn(), chainHolds)
+}
+
+// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
+const kindChainMoveRefused = "chain-move-refused"
+
+// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
+const chainMoveRetry = 30 * time.Second
+
+// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
+// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
+// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
+// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
+// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
+func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
+	say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
+	for {
+		_, err := inv.MoveChain(ctx, say)
+		var refusal *inventory.Refusal
+		switch {
+		case err == nil:
+			go clearChainMove(ctx)
+			return
+		case errors.As(err, &refusal):
+			say("%v", err)
+			go raiseChainMove(ctx, refusal)
+			return
+		case ctx.Err() != nil:
+			return
+		default:
+			say("the move to the bus did not finish, and the tables are read until it does: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(chainMoveRetry):
+		}
+	}
+}
+
+// chainMoveObservation is the urgent condition for a refused move.
+func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
+		Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+		Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
+}
+
+// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
+func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
+	for conditionsFrom == nil {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(time.Second):
+		}
+	}
+	f(conditionsFrom)
+}
+
+func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
+	whenKept(ctx, func(k *conditions.Keeper) {
+		if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
+			fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
+		}
+	})
+}
+
+func clearChainMove(ctx context.Context) {
+	whenKept(ctx, func(k *conditions.Keeper) {
+		for _, kind := range []string{inventory.KindBuilds} {
+			key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
+			if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
+				fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
+			}
+		}
+	})
+}
+
+// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
+func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
+	bus, ok := server.Bus().(link.OverNATS)
+	if !ok {
+		return errors.New("the chain lives on the bus, and this control plane is not on it")
+	}
+	c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds)
+	if err != nil {
+		return err
+	}
+	inv.UseChain(c)
+	movingTheChain(ctx, inv)
+	return nil
+}

cmd/mesh-controller/push.go

@@ -172,6 +172,11 @@ func serve(ctx context.Context) (err error) {
 		return err
 	}
 	defer server.Close()
+	// The chain on the bus, and the move of what the tables still hold into it (novox/hq ADR 0408 §8): before anything
+	// reads builds, rollouts or assignments.
+	if err := attachTheChain(ctx, inv, server); err != nil {
+		return err
+	}
 
 	// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
 	// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was

cmd/mesh-controller/stores.go

@@ -132,6 +132,9 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
 	}
 	// And where config values are read once the move of config has run (issue 664): mesh-config_resolved.
 	inv.ReadsConfigFrom(readConfigValues)
+	// And the chain, opened when a kind that moved to the bus is first read (novox/hq ADR 0408): a command that reads none
+	// has no use of a connection.
+	inv.UseChainOpener(openTheChain)
 	return inv, nil
 }