cmd/mesh-controller/composed_start.go

@@ -112,6 +112,14 @@ func composeAtStart(ctx context.Context, open *stores) {
 	if err := composeReplacedController(ctx, open); err != nil {
 		fmt.Printf("start: the controller this one replaced is not composed: %v\n", err)
 	}
+	// And every kept composition with today's inputs, once: a change of config while no controller heard it (ADR 0545).
+	// One reading of the inputs for every node.
+	ctx = withInputCache(ctx)
+	for _, node := range names {
+		if _, err := followKept(ctx, open, node, nil); err != nil {
+			fmt.Printf("start: %s's kept compositions are not composed again: %v\n", node, err)
+		}
+	}
 	if _, err := retireDeclarations(ctx, open); err != nil {
 		fmt.Printf("start: whether the declarations' stream may go cannot be read: %v\n", err)
 	}
@@ -316,20 +324,26 @@ func composeReplacedController(ctx context.Context, open *stores) error {
 // replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest
 // in place of the controller's registered one.
 func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) {
+	return keptComposition(in, r.node, controllerModule, r.fp, r.manifest)
+}
+
+// keptComposition is module's composition on node at fp: in composed with manifest, that fingerprint's own, in place of
+// the module's registered one, in the reading mode (nothing made). in is not changed.
+func keptComposition(in compose.Input, node, module, fp string, manifest catalogue.Manifest) (sdkdelivery.Composed, error) {
 	shelf := make(map[string]catalogue.Manifest, len(in.Catalogue))
 	for k, v := range in.Catalogue {
 		shelf[k] = v
 	}
-	shelf[controllerModule] = r.manifest
+	shelf[module] = manifest
 	in.Catalogue = shelf
-	all, _, err := compose.Compositions(in, r.node, map[string]string{controllerModule: r.fp}, compose.Reading)
+	all, _, err := compose.Compositions(in, node, map[string]string{module: fp}, compose.Reading)
 	if err != nil {
 		return sdkdelivery.Composed{}, err
 	}
-	key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
+	key, _ := sdkdelivery.ComposedKey(node, module, fp)
 	c, ok := all[key]
 	if !ok {
-		return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the replaced build's manifest", r.node, controllerModule)
+		return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the manifest of %s", node, module, short(fp))
 	}
 	return c, nil
 }

cmd/mesh-controller/composed_start_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strings"
 	"testing"
 
@@ -46,51 +47,91 @@ func TestTheFirstStartDeletesThePlansBucketsAndSaysEachOnce(t *testing.T) {
 	}
 }
 
-// 1945/AC6 (unit half): the controller this one replaced is composed on its node from its own build's manifest at its
-// own fingerprint: the composition holds what that manifest declares, not what the controller registered now declares.
-func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) {
-	now := catalogue.Manifest{Module: controllerModule, Version: "6", Resources: []map[string]any{
-		{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "six\n"}}}
-	before := catalogue.Manifest{Module: controllerModule, Version: "5", Resources: []map[string]any{
-		{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "five\n"}}}
-	in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
-		Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true}}}
-	fp := sdkdelivery.SourcePrefix2 + strings.Repeat("5", 64)
-	c, err := replacedComposition(in, replacedController{node: "anchor", fp: fp, manifest: before})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if c.Module != controllerModule || c.FP != fp || c.Node != "anchor" || len(c.Resources) != 1 {
-		t.Fatalf("composed %s at %s on %s with %d resources", c.Module, c.FP, c.Node, len(c.Resources))
-	}
-	var f struct{ Content string }
-	_ = json.Unmarshal(c.Resources[0], &f)
-	if f.Content != "five\n" {
-		t.Errorf("the replaced controller's composition holds %q, its own manifest says five", f.Content)
-	}
-	if in.Catalogue[controllerModule].Version != "6" {
-		t.Errorf("composing the replaced controller changed the Input's catalogue")
-	}
-}
+// An earlier fingerprint is composed from its own build's manifest with today's inputs: the controller this one
+// replaced (1945/AC6), and every kept composition of a module whose config changed (ADR 0545: on 2026-10-25 a want back
+// to an earlier mesh-build-go on g14 applied the capacity of 0 its kept composition was made with). The composition
+// holds what that manifest declares and today's config, the kept keys are those of the touched modules at a
+// fingerprint other than the registered one, the build of a fingerprint is its newest that worked, and a write of
+// mesh-config's resolved values names the node and module it touches.
+func TestAnEarlierFingerprintIsComposedFromItsOwnManifest(t *testing.T) {
+	fp := func(c string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(c, 64) }
+	t.Run("from its own manifest, with today's inputs", func(t *testing.T) {
+		manifest := func(version string) catalogue.Manifest {
+			return catalogue.Manifest{Module: controllerModule, Version: version,
+				Settings: map[string]catalogue.SettingDeclaration{
+					"capacity": {Kind: catalogue.KindPreference, Default: float64(0), Why: "how many builds at once"}},
+				Resources: []map[string]any{{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings",
+					"content": version + " ${setting:capacity}\n"}}}
+		}
+		now, before := manifest("six"), manifest("five")
+		in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
+			Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true,
+				Settings: map[string][]catalogue.Layer{controllerModule: {{From: "mesh", Values: map[string]any{"capacity": float64(4)}}}}}}}
+		for name, compose := range map[string]func() (sdkdelivery.Composed, error){
+			"the replaced controller": func() (sdkdelivery.Composed, error) {
+				return replacedComposition(in, replacedController{node: "anchor", fp: fp("5"), manifest: before})
+			},
+			"a kept composition": func() (sdkdelivery.Composed, error) {
+				return keptComposition(in, "anchor", controllerModule, fp("5"), before)
+			},
+		} {
+			c, err := compose()
+			if err != nil {
+				t.Fatalf("%s: %v", name, err)
+			}
+			if c.Module != controllerModule || c.FP != fp("5") || c.Node != "anchor" || len(c.Resources) != 1 {
+				t.Fatalf("%s: composed %s at %s on %s with %d resources", name, c.Module, c.FP, c.Node, len(c.Resources))
+			}
+			var f struct{ Content string }
+			_ = json.Unmarshal(c.Resources[0], &f)
+			if f.Content != "five 4\n" {
+				t.Errorf("%s holds %q, want its own manifest's five with today's capacity 4", name, f.Content)
+			}
+		}
+		if in.Catalogue[controllerModule].Version != "six" {
+			t.Errorf("composing an earlier fingerprint changed the Input's catalogue")
+		}
+	})
 
-// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a
-// fingerprint of the new form, not the one registered now.
-func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) {
-	now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64),
-		sdkdelivery.SourcePrefix2+strings.Repeat("4", 64)
-	m := []byte(`{"module":"mesh-controller","version":"5"}`)
-	builds := []inventory.Build{
-		{ID: "c6", SourceFingerprint: now, Manifest: m},
-		{ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m},
-		{ID: "c5", SourceFingerprint: before, Manifest: m},
-		{ID: "c4", SourceFingerprint: older, Manifest: m},
-	}
-	if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" {
-		t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
-	}
-	if _, ok := replacedBuild(builds[:1], now); ok {
-		t.Errorf("a controller with no earlier build replaced one")
-	}
+	t.Run("which build, which keys, which touch", func(t *testing.T) {
+		m := []byte(`{"module":"mesh-controller","version":"5"}`)
+		builds := []inventory.Build{
+			{ID: "c6", SourceFingerprint: fp("6"), Manifest: m},
+			{ID: "c5-failed", SourceFingerprint: fp("5"), Failed: "no", Manifest: m},
+			{ID: "c5", SourceFingerprint: fp("5"), Manifest: m},
+			{ID: "c4", SourceFingerprint: fp("4"), Manifest: m},
+		}
+		if b, ok := replacedBuild(builds, fp("6")); !ok || b.ID != "c5" {
+			t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
+		}
+		if _, ok := replacedBuild(builds[:1], fp("6")); ok {
+			t.Errorf("a controller with no earlier build replaced one")
+		}
+		if b, ok := buildAt(builds, fp("5")); !ok || b.ID != "c5" {
+			t.Errorf("the build at 5 is %+v (%v), want c5, the one that worked", b, ok)
+		}
+		if _, ok := buildAt(builds, fp("3")); ok {
+			t.Errorf("a fingerprint with no build has one")
+		}
+		key := func(node, module, f string) string { k, _ := sdkdelivery.ComposedKey(node, module, f); return k }
+		keys := []string{key("g14", "mesh-build-go", fp("2")), key("g14", "mesh-build-go", fp("1")),
+			key("g14", "mesh-build-go", fp("3")), key("g14", "hello-web", fp("1")), key("ace", "mesh-build-go", fp("1"))}
+		registered := map[string]string{"mesh-build-go": fp("3"), "hello-web": fp("9")}
+		got := keptToFollow(keys, "g14", map[string]bool{"mesh-build-go": true}, registered)
+		if want := []keptKey{{"mesh-build-go", fp("1")}, {"mesh-build-go", fp("2")}}; !slices.Equal(got, want) {
+			t.Errorf("kept to follow %v, want %v", got, want)
+		}
+		if got := keptToFollow(keys, "g14", nil, registered); len(got) != 3 {
+			t.Errorf("every module's kept compositions: %v", got)
+		}
+		for k, want := range map[string][2]string{"g14.mesh-build-go.capacity": {"g14", "mesh-build-go"},
+			"g14.mesh-build-go.limits.cpu": {"g14", "mesh-build-go"}, "g14.mesh-build-go": {}, "": {}} {
+			node, module, ok := configTouched(k)
+			if [2]string{node, module} != want || ok != (want[0] != "") {
+				t.Errorf("%q touches %s %s (%v), want %v", k, node, module, ok, want)
+			}
+		}
+	})
 }
 
 // 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one

cmd/mesh-controller/kept_compositions.go

@@ -0,0 +1,263 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// **A kept composition follows its inputs** (novox/hq ADR 0545, issue 1945). The controller composes a module only at
+// the fingerprint it is registered at, and keeps the compositions of up to five fingerprints (ADR 0539 §4) so that
+// going back is one want. A kept one was never composed again, so it carried the config of the moment it was made: on
+// 2026-10-25 a want back to an earlier mesh-build-go on g14 applied its capacity of 0, the value from before it was
+// set. Now a write of mesh-config's resolved values for a module on a node, heard from that bucket, composes the node
+// and then every kept fingerprint of that module on that node again, each from its own build's manifest with today's
+// inputs, writing only those whose composition differs (writeComposed). A starting controller does it once for every
+// node, for a change made while no controller heard it.
+
+// configTouched is the node and module a key of mesh-config_resolved (`<node>.<module>.<key>`) names.
+func configTouched(key string) (node, module string, ok bool) {
+	parts := strings.SplitN(key, ".", 3)
+	if len(parts) != 3 || parts[0] == "" || parts[1] == "" {
+		return "", "", false
+	}
+	return parts[0], parts[1], true
+}
+
+// keptKey is one kept composition to compose again.
+type keptKey struct{ module, fp string }
+
+// keptToFollow is, of a node's composition keys, each of modules (every module when nil) at a fingerprint other than
+// the one it is registered at, sorted: the kept compositions. The node's own composition and other nodes' are not.
+func keptToFollow(keys []string, node string, modules map[string]bool, registered map[string]string) []keptKey {
+	var out []keptKey
+	for _, key := range keys {
+		n, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key)
+		if err != nil || isNode || n != node || (modules != nil && !modules[module]) {
+			continue
+		}
+		if fp == registered[module] || !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
+			continue
+		}
+		out = append(out, keptKey{module, fp})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].module != out[j].module {
+			return out[i].module < out[j].module
+		}
+		return out[i].fp < out[j].fp
+	})
+	return out
+}
+
+// buildAt is, of a module's builds newest first, the newest that worked at fp and kept its manifest.
+func buildAt(builds []inventory.Build, fp string) (inventory.Build, bool) {
+	for _, b := range builds {
+		if b.Worked() && b.SourceFingerprint == fp && len(b.Manifest) > 0 {
+			return b, true
+		}
+	}
+	return inventory.Build{}, false
+}
+
+// followKept composes again, on node, every kept composition of modules (every module when nil) with today's inputs,
+// and writes those that differ. It answers the keys written; a fingerprint whose build's manifest is gone is said and
+// left as it is.
+func followKept(ctx context.Context, open *stores, node string, modules map[string]bool) ([]string, error) {
+	if _, err := theLease.epoch(ctx); err != nil {
+		return nil, err
+	}
+	bus, err := aBus()
+	if err != nil {
+		return nil, err
+	}
+	defer bus.Close()
+	js, err := jetstream.New(bus.Conn())
+	if err != nil {
+		return nil, err
+	}
+	inv := open.inventory
+	registered, err := currentFingerprints(ctx, inv)
+	if err != nil {
+		return nil, err
+	}
+	unlock := lockComposing(node)
+	defer unlock()
+	at, err := openComposedPlace(ctx, js, bus.Conn(), node)
+	if err != nil {
+		return nil, err
+	}
+	lister, err := at.bucket.ListKeys(ctx)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var keys []string
+	for k := range lister.Keys() {
+		keys = append(keys, k)
+	}
+	kept := keptToFollow(keys, node, modules, registered)
+	if len(kept) == 0 {
+		return nil, nil
+	}
+	sign, err := controllerSigner(ctx, open)
+	if err != nil {
+		return nil, err
+	}
+	upgrades, err := inv.Upgrades(ctx)
+	if err != nil {
+		return nil, err
+	}
+	at.sign = sign
+	at.policy = func(module string) (string, string, string) {
+		u, ok := upgrades[module]
+		if !ok {
+			return "", "", ""
+		}
+		return u.Policy(), u.From, u.Why
+	}
+	in, err := inputFor(ctx, open)
+	if err != nil {
+		return nil, err
+	}
+	var wrote []string
+	var errs []error
+	builds := map[string][]inventory.Build{}
+	for _, k := range kept {
+		if _, read := builds[k.module]; !read {
+			if builds[k.module], err = inv.Builds(ctx, k.module, 20); err != nil {
+				errs = append(errs, fmt.Errorf("%s's builds: %w", k.module, err))
+				continue
+			}
+		}
+		b, found := buildAt(builds[k.module], k.fp)
+		if !found {
+			fmt.Printf("compositions: %s's kept %s on %s has no build whose manifest is kept, and is not composed again\n",
+				k.module, short(k.fp), node)
+			continue
+		}
+		var m catalogue.Manifest
+		if err := json.Unmarshal(b.Manifest, &m); err != nil {
+			errs = append(errs, fmt.Errorf("the manifest of %s's build %s: %w", k.module, b.ID, err))
+			continue
+		}
+		c, err := keptComposition(in.Input, node, k.module, k.fp, m)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s at %s on %s: %w", k.module, short(k.fp), node, err))
+			continue
+		}
+		key, _ := sdkdelivery.ComposedKey(node, k.module, k.fp)
+		w, err := writeComposed(ctx, at, key, c)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if w {
+			wrote = append(wrote, key)
+		}
+	}
+	if len(wrote) > 0 {
+		fmt.Printf("compositions: %s: kept compositions composed again with today's inputs: %s\n", node, strings.Join(wrote, ", "))
+	}
+	return wrote, errors.Join(errs...)
+}
+
+// configFollowGather is how long the config watch waits after a write for more before it composes: mesh-config writes
+// a module's keys one by one.
+var configFollowGather = time.Second
+
+// watchConfigFollow hears every write of mesh-config's resolved values and, for each node and module it touches,
+// composes the node and the module's kept compositions there again, until ctx ends (ADR 0545). A bucket not there is
+// said: a change is then composed by the next act that touches the node and the next start.
+func watchConfigFollow(ctx context.Context, open *stores) {
+	bus, err := aBus()
+	if err != nil {
+		fmt.Printf("config: the resolved values are not heard: %v\n", err)
+		return
+	}
+	js, err := jetstream.New(bus.Conn())
+	if err != nil {
+		bus.Close()
+		fmt.Printf("config: the resolved values are not heard: %v\n", err)
+		return
+	}
+	kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket)
+	if err != nil {
+		bus.Close()
+		fmt.Printf("config: %s is not heard, so a change of config composes nothing until the next act: %v\n",
+			broker.ConfigResolvedBucket, err)
+		return
+	}
+	w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
+	if err != nil {
+		bus.Close()
+		fmt.Printf("config: %s is not heard: %v\n", broker.ConfigResolvedBucket, err)
+		return
+	}
+	goServing(ctx, func() {
+		defer bus.Close()
+		defer func() { _ = w.Stop() }()
+		touched := map[string]map[string]bool{}
+		var gather <-chan time.Time
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case e, ok := <-w.Updates():
+				if !ok {
+					if ctx.Err() == nil {
+						fmt.Printf("config: the watch of %s ended; a change is composed by the next start\n", broker.ConfigResolvedBucket)
+					}
+					return
+				}
+				if e == nil {
+					continue
+				}
+				node, module, ok := configTouched(e.Key())
+				if !ok {
+					continue
+				}
+				if touched[node] == nil {
+					touched[node] = map[string]bool{}
+				}
+				touched[node][module] = true
+				if gather == nil {
+					gather = time.After(configFollowGather)
+				}
+			case <-gather:
+				gather = nil
+				now := touched
+				touched = map[string]map[string]bool{}
+				forgetResolved() // the next composition reads the values just written, not a read of the last 5 s
+				for node, modules := range now {
+					if err := composerFor(open).compose(ctx, []string{node}, false); err != nil {
+						fmt.Printf("config: %s is not composed after its config changed: %v\n", node, err)
+					}
+					if _, err := followKept(ctx, open, node, modules); err != nil {
+						fmt.Printf("config: %s's kept compositions are not composed again: %v\n", node, err)
+					}
+				}
+			}
+		}
+	})
+}
+
+// forgetResolved drops the last read of mesh-config's resolved values.
+func forgetResolved() {
+	theResolved.mu.Lock()
+	theResolved.entries = nil
+	theResolved.mu.Unlock()
+}

cmd/mesh-controller/new_modules.go

@@ -208,6 +208,9 @@ func watchNewModules(ctx context.Context, open *stores) {
 	// composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted.
 	inBackground(ctx, func() { composeAtStart(ctx, open) })
 	watchDoneKeys(ctx, open)
+	// And a change of config composes the node and the touched modules' kept compositions again, heard from
+	// mesh-config's resolved values (ADR 0545, kept_compositions.go).
+	watchConfigFollow(ctx, open)
 	// And each assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543): a module
 	// built long before is assigned to one more node with no build done again (issue 1945, assignment_accounts.go).
 	watchAssignmentAccounts(ctx, open)