Keep the builds every want names and each composition's provisioner plans; find the replaced controller in the build records (issue 1945)

With no plan left to name a build or a send to keep the plans, a want names what a collection must keep, and the
composition is where the provisioner's plans are composed for the assignments mirror (ADR 0539 clauses 1 and 5);
the assignments no longer name a node's build, so the controller replaced is the newest earlier build of it.

cmd/mesh-controller/composed_live.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"sort"
@@ -79,6 +80,10 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
 			continue
 		}
+		// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
+		if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
+			errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err))
+		}
 		comps, moved, err := compositionsOf(node, body, fps, manifests)
 		if err != nil {
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
@@ -196,3 +201,68 @@ func haveOn(ctx context.Context, js jetstream.JetStream, node, module string) (s
 	}
 	return h, true, nil
 }
+
+// wantedOnTheBus is every fingerprint a want names, as `<module> <fingerprint>`: what the keep set keeps the builds of
+// (ADR 0539 clause 1). Read from the assigner's bucket on each look; no bucket yet names none.
+func wantedOnTheBus(ctx context.Context) (map[string]bool, error) {
+	bus, err := aBus()
+	if err != nil {
+		return nil, err
+	}
+	defer bus.Close()
+	js, err := jetstream.New(bus.Conn())
+	if err != nil {
+		return nil, err
+	}
+	kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return map[string]bool{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	lister, err := kv.ListKeys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return map[string]bool{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]bool{}
+	for key := range lister.Keys() {
+		_, module, err := sdkdelivery.ParseWantKey(key)
+		if err != nil {
+			continue
+		}
+		e, err := kv.Get(ctx, key)
+		if err != nil {
+			continue
+		}
+		w, err := sdkdelivery.DecodeWant(e.Value())
+		if err != nil || w.FP == "" {
+			continue
+		}
+		out[module+" "+w.FP] = true
+	}
+	return out, nil
+}
+
+// plansKept is a composition's provisioner's plans as the store keeps them: by module, each plan and what requires a
+// module only pulled in by another (design 57 P6, issue 765); nil when it hands none.
+func plansKept(d sendable) []byte {
+	if len(d.plans) == 0 {
+		return nil
+	}
+	kept := map[string]composedPlan{}
+	for module, p := range d.plans {
+		kept[module] = composedPlan{Plan: p, RequiredBy: d.requiredBy[module]}
+	}
+	raw, _ := json.Marshal(kept) // plain strings and slices: it marshals
+	return raw
+}
+
+// composedPlan is one handed module's plan as the store keeps it, the shape inventory.SentProvisioningPlans reads.
+type composedPlan struct {
+	catalogue.Plan
+	RequiredBy string `json:"required-by,omitempty"`
+}

cmd/mesh-controller/composed_plans_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// 1945 (dev-r6-controller's finding): the provisioner's plans a composition hands are kept in the shape the
+// assignments mirror reads (inventory.SentProvisioningPlans: each module's plan and what requires it), and a
+// composition that hands none keeps none.
+func TestAComposedNodesProvisionerPlansAreKept(t *testing.T) {
+	if plansKept(sendable{}) != nil {
+		t.Errorf("a composition that hands nothing kept plans")
+	}
+	d := sendable{plans: map[string]catalogue.Plan{"umami": {Placement: map[string]string{"dir:state": "/var/lib/umami"}}},
+		requiredBy: map[string]string{"umami": "analytics"}}
+	var kept map[string]struct {
+		Placement  map[string]string `json:"placement"`
+		RequiredBy string            `json:"required-by"`
+	}
+	if err := json.Unmarshal(plansKept(d), &kept); err != nil {
+		t.Fatal(err)
+	}
+	if kept["umami"].Placement["dir:state"] != "/var/lib/umami" || kept["umami"].RequiredBy != "analytics" {
+		t.Errorf("kept %+v", kept)
+	}
+}

cmd/mesh-controller/composed_start.go

@@ -15,6 +15,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 )
 
@@ -109,15 +110,15 @@ func composeAtStart(ctx context.Context, open *stores) {
 	}
 }
 
-// replacedController is the controller a control-node ran before this one: the build the mesh last sent it and that
-// build's fingerprint and manifest.
+// replacedController is the controller a control-node ran before this one: its fingerprint and manifest.
 type replacedController struct {
 	node, fp string
 	manifest catalogue.Manifest
 }
 
-// replacedControllers are, for each node the controller is assigned to, the controller build it was last sent when its
-// fingerprint is of the new form and not the one the controller is registered at now.
+// replacedControllers are, for each node the controller is assigned to, the controller it replaced: the newest build of
+// the controller that worked, whose fingerprint is of the new form and not the one the controller is registered at now
+// (the assignments name no build since ADR 0539 §12, so the build records say it). None when there is no such build.
 func replacedControllers(ctx context.Context, open *stores) ([]replacedController, error) {
 	inv := open.inventory
 	fps, err := currentFingerprints(ctx, inv)
@@ -134,26 +135,40 @@ func replacedControllers(ctx context.Context, open *stores) ([]replacedControlle
 			on = e.On
 		}
 	}
+	if len(on) == 0 {
+		return nil, nil
+	}
+	builds, err := inv.Builds(ctx, controllerModule, 20)
+	if err != nil {
+		return nil, err
+	}
+	before, found := replacedBuild(builds, fps[controllerModule])
+	if !found {
+		return nil, nil
+	}
+	var m catalogue.Manifest
+	if err := json.Unmarshal(before.Manifest, &m); err != nil {
+		return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, before.ID, err)
+	}
 	var out []replacedController
 	for _, node := range on {
-		sent, known, err := inv.SentBuilds(ctx, node)
-		if err != nil || !known || sent[controllerModule] == "" {
-			continue
-		}
-		build, found, err := inv.BuildByID(ctx, sent[controllerModule])
-		if err != nil || !found || build.SourceFingerprint == fps[controllerModule] ||
-			!strings.HasPrefix(build.SourceFingerprint, sdkdelivery.SourcePrefix2) || len(build.Manifest) == 0 {
-			continue
-		}
-		var m catalogue.Manifest
-		if err := json.Unmarshal(build.Manifest, &m); err != nil {
-			return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, build.ID, err)
-		}
-		out = append(out, replacedController{node: node, fp: build.SourceFingerprint, manifest: m})
+		out = append(out, replacedController{node: node, fp: before.SourceFingerprint, manifest: m})
 	}
 	return out, nil
 }
 
+// replacedBuild is, of a module's builds newest first, the newest that worked, kept its manifest, and names a
+// fingerprint of the new form other than current.
+func replacedBuild(builds []inventory.Build, current string) (inventory.Build, bool) {
+	for _, b := range builds {
+		if b.Worked() && b.SourceFingerprint != current && strings.HasPrefix(b.SourceFingerprint, sdkdelivery.SourcePrefix2) &&
+			len(b.Manifest) > 0 {
+			return b, true
+		}
+	}
+	return inventory.Build{}, false
+}
+
 // composeReplacedController writes, on each control-node, the controller's composition at the fingerprint of the one
 // it replaced: the node composed with that build's manifest in place of the controller's registered one, in the
 // reading mode (nothing made), and only the controller's composition of it kept.

cmd/mesh-controller/composed_start_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
 )
 
 // fakeBuckets is the bus's buckets by name, for deleting.
@@ -70,3 +71,23 @@ func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) {
 		t.Errorf("composing the replaced controller changed the Input's catalogue")
 	}
 }
+
+// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a
+// fingerprint of the new form, not the one registered now.
+func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) {
+	now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64),
+		sdkdelivery.SourcePrefix2+strings.Repeat("4", 64)
+	m := []byte(`{"module":"mesh-controller","version":"5"}`)
+	builds := []inventory.Build{
+		{ID: "c6", SourceFingerprint: now, Manifest: m},
+		{ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m},
+		{ID: "c5", SourceFingerprint: before, Manifest: m},
+		{ID: "c4", SourceFingerprint: older, Manifest: m},
+	}
+	if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" {
+		t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
+	}
+	if _, ok := replacedBuild(builds[:1], now); ok {
+		t.Errorf("a controller with no earlier build replaced one")
+	}
+}

cmd/mesh-controller/new_modules.go

@@ -209,6 +209,8 @@ func watchNewModules(ctx context.Context, open *stores) {
 	inBackground(func() { composeAtStart(ctx, open) })
 	watchDoneKeys(ctx, open)
 	watchHavesForJudgement(ctx, open)
+	// And the keep set keeps the builds every want names (ADR 0539 clause 1), with no plan left to name them.
+	open.inventory.SetWanted(wantedOnTheBus)
 	keys, closeKeys, err := openBuilderKeys()
 	if err != nil {
 		fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)

internal/inventory/collection.go

@@ -2,6 +2,7 @@ package inventory
 
 import (
 	"context"
+	"fmt"
 	"slices"
 	"sort"
 	"strings"
@@ -47,6 +48,9 @@ const (
 	KeptByRecentRecord = "recorded-lately"
 	// KeptByOpenPlan: a plan that has not ended names the build that made it (novox/hq issue 1933).
 	KeptByOpenPlan = "open-plan"
+	// KeptByWant: a want names the fingerprint of the build that made it, so a node may be told to run it (novox/hq
+	// ADR 0539 clause 1): with no plan, the want is what names a build a node runs or moves to.
+	KeptByWant = "wanted"
 )
 
 // KeptLately is how long after a build is recorded its artifacts are kept whatever else the records say (novox/hq
@@ -458,7 +462,8 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t
 	return out
 }
 
-// keptLatelyOrPlanned is keptLately of every build the records hold, (no plan names a build any more, ADR 0539 §17).
+// keptLatelyOrPlanned is keptLately of every build the records hold, and the builds of every fingerprint a want names
+// (no plan names a build any more, ADR 0539 §17; a want does, clause 1).
 func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) {
 	c, err := i.theChain(ctx)
 	if err != nil {
@@ -468,7 +473,44 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin
 	if err != nil {
 		return nil, err
 	}
-	return keptLately(all, nil, time.Now().UTC(), i.lately()), nil
+	out := keptLately(all, nil, time.Now().UTC(), i.lately())
+	if i.wanted == nil {
+		return out, nil
+	}
+	wanted, err := i.wanted(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the fingerprints the wants name: %w", err)
+	}
+	for reference, whys := range keptByWants(all, wanted) {
+		for _, why := range whys {
+			if !slices.Contains(out[reference], why) {
+				out[reference] = append(out[reference], why)
+			}
+		}
+	}
+	return out, nil
+}
+
+// keptByWants is, of these builds, the artifacts of each that worked and whose module and fingerprint a want names
+// (`<module> <fingerprint>`), each kept as wanted.
+func keptByWants(builds []Build, wanted map[string]bool) map[string][]string {
+	out := map[string][]string{}
+	for _, b := range builds {
+		if !b.Worked() || b.SourceFingerprint == "" || !wanted[b.Module+" "+b.SourceFingerprint] {
+			continue
+		}
+		for _, a := range b.Made {
+			if reference := asRecorded(a.Reference); reference != "" && !slices.Contains(out[reference], KeptByWant) {
+				out[reference] = append(out[reference], KeptByWant)
+			}
+		}
+	}
+	return out
+}
+
+// SetWanted gives the keep set every fingerprint a want names, as `<module> <fingerprint>` (ADR 0539 clause 1).
+func (i *Inventory) SetWanted(wanted func(ctx context.Context) (map[string]bool, error)) {
+	i.wanted = wanted
 }
 
 // digestIn is the `sha256:<hex>` a reference names, empty when it names none.

internal/inventory/collection_wanted_test.go

@@ -0,0 +1,37 @@
+package inventory
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// 1945 (dev-r6-controller's finding, for ADR 0539 clause 1): with no plan left to name a build, the builds of every
+// fingerprint a want names are kept: a collection never lets go of what a node is told to run. A build of a
+// fingerprint no want names, one of another module at that fingerprint, and a failed one keep nothing this way.
+func TestTheBuildsOfAWantedFingerprintAreKept(t *testing.T) {
+	fp := "src2:" + strings.Repeat("a", 64)
+	other := "src2:" + strings.Repeat("b", 64)
+	at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	made := func(id, module, fingerprint, reference string) Build {
+		return Build{ID: id, Module: module, SourceFingerprint: fingerprint, At: at,
+			Made: []Artifact{{Name: "files", Kind: "archive", Reference: reference}}}
+	}
+	const (
+		wanted   = "artifact-store://route-proxy/files/blobs/sha256:a1"
+		unwanted = "artifact-store://route-proxy/files/blobs/sha256:b2"
+		elsewise = "artifact-store://mesh-mail/files/blobs/sha256:c3"
+	)
+	builds := []Build{made("key-a-1", "route-proxy", fp, wanted), made("key-b-1", "route-proxy", other, unwanted),
+		made("key-c-1", "mesh-mail", fp, elsewise),
+		{ID: "failed", Module: "route-proxy", SourceFingerprint: fp, Failed: "no", Made: []Artifact{{Reference: "artifact-store://f@sha256:ff"}}}}
+	kept := keptByWants(builds, map[string]bool{"route-proxy " + fp: true})
+	if !has(kept[wanted], KeptByWant) {
+		t.Errorf("the archive of a wanted fingerprint is not kept: %v", kept)
+	}
+	for _, r := range []string{unwanted, elsewise, "artifact-store://f@sha256:ff"} {
+		if len(kept[r]) > 0 {
+			t.Errorf("%s is kept as wanted: %v", r, kept[r])
+		}
+	}
+}

internal/inventory/nodes.go

@@ -43,6 +43,9 @@ type Inventory struct {
 	// keptLately is how long a build's artifacts are kept after it is recorded (KeptLately, novox/hq issue 1933); zero
 	// is KeptLately, negative none (ForTest: the tests of the keep set's other rules record their builds a moment apart).
 	keptLately time.Duration
+	// wanted is every fingerprint a want names, as `<module> <fingerprint>` (novox/hq ADR 0539 clause 1; SetWanted): the
+	// builds of each are kept, so a collection never lets go of what a node is told to run. Nil names none.
+	wanted func(ctx context.Context) (map[string]bool, error)
 }
 
 // lately is how long a build's artifacts are kept after it is recorded, whatever else the records say.

internal/inventory/provisioning.go

@@ -408,6 +408,18 @@ func sortedPlanKeys(plans map[string]Provisioning) []string {
 // (migration 0100) by module, and carried by the assignments mirror in each handed module's assignment. So the
 // client is handed the plan of the build its machine was sent, and a controller started afresh mirrors the same.
 
+// RecordComposedPlans keeps the provisioner's plan of each module a node's composition hands to it, as JSON by module
+// (`<module>: plan and required-by`), or none: what SentProvisioningPlans reads for the assignments mirror (design 57 P6).
+// With no send left, the composition is where the plans are composed (novox/hq ADR 0539 clause 5).
+func (i *Inventory) RecordComposedPlans(ctx context.Context, node string, plans []byte) error {
+	var kept any
+	if len(plans) > 0 {
+		kept = plans
+	}
+	_, err := i.store.Pool().Exec(ctx, `update node set sent_provisioning = $2 where name = $1`, node, kept)
+	return err
+}
+
 // SentProvisioningPlans is every plan the machines were last sent, keyed `<node>.<module>`: what
 // SetProvisioningPlans is given. A plan of a module only pulled in by another is kept with what required it, and is
 // mirrored in a value of its own until a send drops it (issue 765). A plan of a module assigned when it was sent and