Say each wait of a done key's take-in once with its reason and look again every 30 s, one follower per module (issue 1945)

cmd/mesh-controller/asker_wire.go

@@ -12,7 +12,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -28,7 +27,7 @@ import (
 )
 
 // askerFrom is the serving controller's asker; nil in any other process.
-var askerFrom atomic.Pointer[asker]
+var askerFrom *asker
 
 // askWithin is how long a verb a warrant chose is given to answer.
 const askWithin = time.Minute
@@ -574,8 +573,8 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
 		fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
 		return
 	}
-	askerFrom.Store(a)
-	goServing(ctx, func() { a.keep(ctx) })
+	askerFrom = a
+	go a.keep(ctx)
 }
 
 // servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and

cmd/mesh-controller/chain.go

@@ -49,7 +49,7 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
 	// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
 	if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
 		fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
-		goServing(ctx, func() { layBuildsOutSoon(ctx, inv) })
+		go layBuildsOutSoon(ctx, inv)
 	}
 	// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
 	// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
@@ -80,11 +80,7 @@ func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
 			Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
 		if err != nil {
 			fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
-			if ctx.Err() != nil {
-				return // stopped: the next controller lays them out
-			}
-			// A cancelled layout is tried again and raises nothing: a cancellation is no fault of the chain (issue 1945).
-			if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) {
+			if keeper := conditionsFrom; keeper != nil && !failed {
 				o.Summary += err.Error()
 				if _, oerr := keeper.Observe(ctx, o); oerr == nil {
 					failed = true

cmd/mesh-controller/composed_digest_race_test.go

@@ -1,78 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-
-	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// raceKV is a digests bucket in which another writer writes its composition and digest just before this writer reads
-// the digest key: what two take-ins composing one node at once do.
-type raceKV struct {
-	*fakeComposedKV
-	before func()
-}
-
-func (r *raceKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
-	if r.before != nil {
-		b := r.before
-		r.before = nil
-		b()
-	}
-	return r.fakeComposedKV.Get(ctx, key)
-}
-
-// The digest kept beside a composition is always the digest of the composition the bucket holds: when another writer
-// writes its composition and its digest between this writer's composition and its digest, this writer's digest must not
-// win, or the assigner's want names a composition the bucket no longer holds and the node applies nothing (issue 1945,
-// live on 2026-10-25: shanks' node-tools composition had digest 575a5c96, its want named a36e63c8).
-func TestTheDigestKeptIsTheDigestOfTheCompositionHeld(t *testing.T) {
-	at, _, _, _ := aPlace(t, "shanks")
-	key, _ := sdkdelivery.ComposedKey("shanks", "node-tools", fp2Of('a'))
-	file := func(content string) json.RawMessage {
-		return json.RawMessage(`{"id":"node-tools.f","type":"file","module":"node-tools","path":"/etc/nt","mode":"0644","content":"` +
-			content + `"}`)
-	}
-	ours := sdkdelivery.Composed{Node: "shanks", Module: "node-tools", FP: fp2Of('a'), Resources: []json.RawMessage{file("ours")}}
-	theirs := sdkdelivery.Composed{Node: "shanks", Module: "node-tools", FP: fp2Of('a'), Resources: []json.RawMessage{file("theirs")}}
-	race := &raceKV{fakeComposedKV: at.digests.(*fakeComposedKV)}
-	at.digests = race
-	race.before = func() {
-		// The other writer: its composition over ours, then its digest.
-		other := at
-		other.digests = race.fakeComposedKV
-		if _, err := writeComposed(context.Background(), other, key, theirs); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if _, err := writeComposed(context.Background(), at, key, ours); err != nil {
-		t.Fatal(err)
-	}
-	held, _, err := readComposed(context.Background(), at.bucket, key)
-	if err != nil {
-		t.Fatal(err)
-	}
-	heldDigest, _ := sdkdelivery.ComposedDigestOf(held)
-	e, err := race.fakeComposedKV.Get(context.Background(), key)
-	if err != nil {
-		t.Fatal(err)
-	}
-	kept, err := sdkdelivery.DecodeComposedDigest(e.Value())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if kept.Digest != heldDigest {
-		t.Fatalf("the digest kept is %s and the composition held has %s", kept.Digest, heldDigest)
-	}
-}
-
-func fp2Of(c byte) string {
-	b := make([]byte, 64)
-	for i := range b {
-		b[i] = c
-	}
-	return sdkdelivery.SourcePrefix2 + string(b)
-}

cmd/mesh-controller/composed_keys.go

@@ -120,13 +120,6 @@ var (
 	doneKeysSay     = func(s string) { fmt.Println(s) }
 )
 
-// doneKeysBeforeLetGo is called just before a follower lets go of its module; a variable so a test can wake the module
-// at that moment. Nil does nothing.
-var doneKeysBeforeLetGo func(module string)
-
-// doneKeysLookAgain marks a module woken while a take-in of it ran: that take-in goes round once more.
-var doneKeysLookAgain sync.Map
-
 // doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
 var doneKeysMu sync.Map
 
@@ -136,42 +129,18 @@ var doneKeysMu sync.Map
 // yet is no failure: it is said once for each reason it waits for and looked at again every doneKeyRetry.
 func takeDoneKey(ctx context.Context, open *stores, module string) {
 	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
-	for {
-		if !mu.(*sync.Mutex).TryLock() {
-			// A take-in of the module already follows it: it is told to look again once it is done, so a newer key
-			// written meanwhile is never lost (the review of issue 1945).
-			doneKeysLookAgain.Store(module, true)
-			return
-		}
-		followDoneKey(ctx, open, module)
-		if doneKeysBeforeLetGo != nil {
-			doneKeysBeforeLetGo(module)
-		}
-		mu.(*sync.Mutex).Unlock()
-		// A wake between this follower's last look and its letting go failed its TryLock and only marked the module:
-		// looked at again now that it is let go, and taken once more (the review of issue 1945).
-		if _, again := doneKeysLookAgain.LoadAndDelete(module); !again || ctx.Err() != nil {
-			return
-		}
+	if !mu.(*sync.Mutex).TryLock() {
+		return // a take-in of the module already follows it, and looks again within doneKeyRetry
 	}
-}
-
-// followDoneKey takes a module's done key in, under the module's lock, until it is taken in or ctx ends: a failure or a
-// wait is tried again every doneKeyRetry.
-func followDoneKey(ctx context.Context, open *stores, module string) {
+	defer mu.(*sync.Mutex).Unlock()
 	failed := false
 	saidWait := ""
 	for {
-		doneKeysLookAgain.Delete(module) // what this look reads covers every wake before it
 		err := doneKeyTakeOnce(ctx, open, module)
 		o := doneKeyObservation(module)
 		if err == nil {
 			if keeper := conditionsFrom; keeper != nil && failed {
 				_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
-				failed = false
-			}
-			if _, again := doneKeysLookAgain.LoadAndDelete(module); again {
-				continue // a wake came while this one ran: once more, for the key written meanwhile
 			}
 			return
 		}
@@ -194,12 +163,7 @@ func followDoneKey(ctx context.Context, open *stores, module string) {
 			}
 			continue
 		}
-		if ctx.Err() != nil {
-			return // stopped: a take-in cut by the stop is no fault of the build, and the next controller takes it in
-		}
-		if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) {
-			// A take-in whose own step was cancelled is tried again, and raises nothing: a cancellation is no fault
-			// of the build (issue 1945: 30 urgent conditions live, each a take-in cut by a restart).
+		if keeper := conditionsFrom; keeper != nil && !failed {
 			o.Summary += err.Error()
 			if _, oerr := keeper.Observe(ctx, o); oerr == nil {
 				failed = true
@@ -287,13 +251,21 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
 			}
 			return holders[catalogue.BrokerSeat].Node, nil
 		},
-		// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
-		// grants once (issue 1945, composer.go).
 		compose: func(ctx context.Context, nodes []string) error {
-			return composerFor(open).compose(ctx, nodes, false)
+			_, err := composeModules(ctx, open, nodes)
+			return err
 		},
 		grants: func(ctx context.Context) error {
-			return composerFor(open).compose(ctx, nil, true)
+			bus, err := aBus()
+			if err != nil {
+				return err
+			}
+			defer bus.Close()
+			js, err := jetstream.New(bus.Conn())
+			if err != nil {
+				return err
+			}
+			return grantsIssued(ctx, inv, js, "done keys: ")
 		},
 	}
 }
@@ -313,10 +285,10 @@ func watchDoneKeys(ctx context.Context, open *stores) {
 		fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
 		return
 	}
-	goServing(ctx, func() {
+	go func() {
 		<-ctx.Done()
 		closeKeys()
-	})
+	}()
 	// And every module once now, after the watch is open so no write falls between the two.
 	inBackground(func() {
 		doneKeysAtStart(ctx, catalogedModules(open), func(module string) { takeDoneKeyAtStart(ctx, open, module) })
@@ -335,44 +307,23 @@ func doneKeysAtStart(ctx context.Context, modules func(context.Context) ([]strin
 		return
 	}
 	sort.Strings(names)
-	// Side by side, a few at a time: their compositions meet in the composer's passes, each node composed once per pass
-	// (issue 1945), so the start's look is not one module's composition after another.
-	sem := make(chan struct{}, startTakeIns)
-	var wg sync.WaitGroup
 	for _, m := range names {
 		if ctx.Err() != nil {
-			break
+			return
 		}
-		sem <- struct{}{}
-		wg.Add(1)
-		go func() {
-			defer func() { <-sem; wg.Done() }()
-			take(m)
-		}()
+		take(m)
 	}
-	wg.Wait()
 }
 
-// startTakeIns is how many take-ins the start's look runs at once.
-const startTakeIns = 16
-
 // takeDoneKeyAtStart is one take-in of the start's look, under the module's lock; one that fails is handed to takeDoneKey,
 // which tries again every doneKeyRetry, loudly, so a failing module never holds up the look at the others.
 func takeDoneKeyAtStart(ctx context.Context, open *stores, module string) {
 	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
-	if !mu.(*sync.Mutex).TryLock() {
-		// A take-in already follows the module: told to look again, it takes what the start would have (issue 1945).
-		doneKeysLookAgain.Store(module, true)
-		return
-	}
-	doneKeysLookAgain.Delete(module) // what this look reads covers every wake before it
-	err := doneKeyTakeOnce(ctx, open, module)
+	mu.(*sync.Mutex).Lock()
+	err := takeDoneKeyOnce(ctx, open, module)
 	mu.(*sync.Mutex).Unlock()
-	// Read after letting go, as every holder of a module's lock does: a wake during this look only marked the module
-	// (the review of issue 1945).
-	_, again := doneKeysLookAgain.LoadAndDelete(module)
-	if err != nil || again {
-		// A failure, a wait or a wake meanwhile is followed by takeDoneKey, which says it and looks again.
+	if err != nil {
+		// A failure or a wait is followed by takeDoneKey, which says it and looks again (issue 1945).
 		inBackground(func() { takeDoneKey(ctx, open, module) })
 	}
 }
@@ -391,34 +342,3 @@ func catalogedModules(open *stores) func(context.Context) ([]string, error) {
 		return out, nil
 	}
 }
-
-// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
-// runs several meshes), each started at its first use and run for as long as the process serves.
-var doneKeysComposers sync.Map
-
-// composerFor is the composer of the stores, its passes composing on them and writing the grants on the bus.
-func composerFor(open *stores) *composer {
-	if c, ok := doneKeysComposers.Load(open); ok {
-		return c.(*composer)
-	}
-	c := newComposer(func(ctx context.Context, nodes []string) map[string]error {
-		_, failed := composeModulesByNode(ctx, open, nodes)
-		return failed
-	}, func(ctx context.Context) error {
-		bus, err := aBus()
-		if err != nil {
-			return err
-		}
-		defer bus.Close()
-		js, err := jetstream.New(bus.Conn())
-		if err != nil {
-			return err
-		}
-		return grantsIssued(ctx, open.inventory, js, "done keys: ")
-	})
-	if was, loaded := doneKeysComposers.LoadOrStore(open, c); loaded {
-		return was.(*composer)
-	}
-	go c.run(context.Background())
-	return c
-}

cmd/mesh-controller/composed_keys_cancel_test.go

@@ -1,61 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/novox/mesh-controller/internal/conditions"
-)
-
-// A take-in cut by a cancellation raises no condition (issue 1945: on 2026-10-25 thirty urgent fingerprint-not-composed
-// conditions were live, each a take-in a stop or a restart cut with "context canceled"): when the controller stops, the
-// next one takes it in; when a step of its own was cancelled, it is tried again.
-func TestATakeInCutByACancellationRaisesNothing(t *testing.T) {
-	for _, tc := range []struct {
-		name string
-		stop bool // the controller's context ends, else only the step's error is a cancellation
-	}{{"the controller stops", true}, {"a step was cancelled", false}} {
-		t.Run(tc.name, func(t *testing.T) {
-			was, wasRetry, wasSay, wasKeeper := doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom
-			t.Cleanup(func() {
-				doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom = was, wasRetry, wasSay, wasKeeper
-			})
-			store := conditions.NewInMemory()
-			k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
-				Teller: &conditions.Told{}})
-			t.Cleanup(func() { k.Close(context.Background()) })
-			conditionsFrom = k
-			doneKeyRetry = time.Millisecond
-			doneKeysSay = func(string) {}
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			calls := 0
-			doneKeyTakeOnce = func(context.Context, *stores, string) error {
-				calls++
-				if tc.stop {
-					cancel() // stopped mid-way
-					return fmt.Errorf("the module cannot be composed: %w", context.Canceled)
-				}
-				if calls == 1 {
-					return fmt.Errorf("the module cannot be composed: %w", context.Canceled)
-				}
-				return nil
-			}
-			done := make(chan struct{})
-			go func() { takeDoneKey(ctx, nil, "cut-module"); close(done) }()
-			select {
-			case <-done:
-			case <-time.After(5 * time.Second):
-				t.Fatal("the take-in did not end")
-			}
-			if !tc.stop && calls != 2 {
-				t.Errorf("a take-in whose step was cancelled was tried %d time(s), want 2", calls)
-			}
-			if c, found, err := conditions.ReadOne(t.Context(), store, doneKeyObservation("cut-module").Key()); err != nil || found {
-				t.Fatalf("a take-in cut by a cancellation raised %+v (found %v, %v)", c, found, err)
-			}
-		})
-	}
-}

cmd/mesh-controller/composed_keys_start_test.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"reflect"
 	"sort"
-	"sync"
 	"testing"
 )
 
@@ -16,10 +15,9 @@ import (
 // new fingerprints, and the new one, watching updates only, never looked).
 func TestAStartingControllerTakesEveryModulesDoneKeyOnce(t *testing.T) {
 	var took []string
-	var mu sync.Mutex
 	doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) {
 		return []string{"mesh-provisioner-client", "gitea", "postgres"}, nil
-	}, func(module string) { mu.Lock(); took = append(took, module); mu.Unlock() })
+	}, func(module string) { took = append(took, module) })
 	sort.Strings(took)
 	if want := []string{"gitea", "mesh-provisioner-client", "postgres"}; !reflect.DeepEqual(took, want) {
 		t.Fatalf("took %v, want %v", took, want)

cmd/mesh-controller/composed_keys_test.go

@@ -99,7 +99,7 @@ func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testin
 		d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file",
 			"module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"})
 		raw, _ := json.Marshal(d)
-		comps, _, _, err := compositionsOf("g14", raw, fps, nil, nil, nil)
+		comps, _, err := compositionsOf("g14", raw, fps, nil, nil)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/mesh-controller/composed_keys_waits_test.go

@@ -73,8 +73,7 @@ func TestAWakeWhileATakeInFollowsAddsNone(t *testing.T) {
 			return fmt.Errorf("%w: the key says building", errKeyWaits)
 		}
 	}
-	follower := make(chan struct{})
-	go func() { takeDoneKey(context.Background(), nil, "board"); close(follower) }()
+	go takeDoneKey(context.Background(), nil, "board")
 	time.Sleep(5 * time.Millisecond)
 	returned := make(chan struct{})
 	go func() { takeDoneKey(context.Background(), nil, "board"); close(returned) }()
@@ -84,101 +83,4 @@ func TestAWakeWhileATakeInFollowsAddsNone(t *testing.T) {
 		t.Fatal("a second wake waited for the follower")
 	}
 	close(release)
-	<-follower
-}
-
-// A wake while a take-in of the module follows it is not lost: the follower goes round once more after it succeeds, so
-// a newer build's key written meanwhile is taken in (the review of issue 1945).
-func TestAWakeDuringATakeInIsTakenAfterIt(t *testing.T) {
-	was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
-	t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
-	doneKeyRetry = 20 * time.Millisecond
-	doneKeysSay = func(string) {}
-	var mu sync.Mutex
-	calls := 0
-	inFirst := make(chan struct{})
-	release := make(chan struct{})
-	doneKeyTakeOnce = func(context.Context, *stores, string) error {
-		mu.Lock()
-		calls++
-		n := calls
-		mu.Unlock()
-		if n == 1 {
-			close(inFirst)
-			<-release
-		}
-		return nil
-	}
-	follower := make(chan struct{})
-	go func() { takeDoneKey(context.Background(), nil, "mesh-host"); close(follower) }()
-	<-inFirst
-	takeDoneKey(context.Background(), nil, "mesh-host") // the newer key's wake, while the first runs
-	close(release)
-	<-follower
-	mu.Lock()
-	defer mu.Unlock()
-	if calls != 2 {
-		t.Fatalf("taken %d times: the wake during the take-in was lost", calls)
-	}
-}
-
-// A wake that comes after the follower's last look but before it lets go of the module is not lost: the follower looks
-// at the mark again once it has let go, and takes the module once more (the review of issue 1945).
-func TestAWakeJustBeforeTheFollowerLetsGoIsTaken(t *testing.T) {
-	was, wasRetry, wasSay, wasHook := doneKeyTakeOnce, doneKeyRetry, doneKeysSay, doneKeysBeforeLetGo
-	t.Cleanup(func() {
-		doneKeyTakeOnce, doneKeyRetry, doneKeysSay, doneKeysBeforeLetGo = was, wasRetry, wasSay, wasHook
-	})
-	doneKeyRetry = 10 * time.Millisecond
-	doneKeysSay = func(string) {}
-	var mu sync.Mutex
-	calls := 0
-	doneKeyTakeOnce = func(context.Context, *stores, string) error {
-		mu.Lock()
-		calls++
-		mu.Unlock()
-		return nil
-	}
-	woke := false
-	doneKeysBeforeLetGo = func(module string) {
-		if !woke {
-			woke = true
-			takeDoneKey(context.Background(), nil, module) // fails its TryLock and marks the module
-		}
-	}
-	takeDoneKey(context.Background(), nil, "mesh-host")
-	mu.Lock()
-	defer mu.Unlock()
-	if calls != 2 {
-		t.Fatalf("taken %d times: the wake just before the follower let go was lost", calls)
-	}
-}
-
-// A wake during the start's take-in of a module is not lost: the start pass reads the mark once it lets go of the module,
-// and hands the module to takeDoneKey, which takes it once more (the review of issue 1945: the third dropped wake of its
-// kind).
-func TestAWakeDuringTheStartsTakeInIsTaken(t *testing.T) {
-	was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
-	t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
-	doneKeyRetry = 10 * time.Millisecond
-	doneKeysSay = func(string) {}
-	var mu sync.Mutex
-	calls := 0
-	doneKeyTakeOnce = func(context.Context, *stores, string) error {
-		mu.Lock()
-		calls++
-		n := calls
-		mu.Unlock()
-		if n == 1 {
-			takeDoneKey(context.Background(), nil, "mesh-host") // the newer key's wake, during the start's take-in
-		}
-		return nil
-	}
-	takeDoneKeyAtStart(context.Background(), nil, "mesh-host")
-	background.Wait()
-	mu.Lock()
-	defer mu.Unlock()
-	if calls != 2 {
-		t.Fatalf("taken %d times: the wake during the start's take-in was lost", calls)
-	}
 }

cmd/mesh-controller/composed_live.go

@@ -26,52 +26,29 @@ import (
 // every refusal is named in the error. It answers the keys it wrote, by node. Under the lease only: a controller that
 // does not hold it writes nothing and says so.
 func composeModules(ctx context.Context, open *stores, nodes []string) (map[string][]string, error) {
-	out, byNode := composeModulesByNode(ctx, open, nodes)
-	names := make([]string, 0, len(byNode))
-	for n := range byNode {
-		names = append(names, n)
-	}
-	sort.Strings(names)
-	var errs []error
-	for _, n := range names {
-		errs = append(errs, byNode[n])
-	}
-	return out, errors.Join(errs...)
-}
-
-// composeModulesByNode is composeModules with each node's error apart: a failure before any node is composed is every
-// node's (the composer gives each caller only the errors of the nodes it asked for, issue 1945).
-func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (map[string][]string, map[string]error) {
-	all := func(err error) (map[string][]string, map[string]error) {
-		byNode := map[string]error{}
-		for _, n := range nodes {
-			byNode[n] = err
-		}
-		return nil, byNode
-	}
 	if _, err := theLease.epoch(ctx); err != nil {
-		return all(fmt.Errorf("nothing is composed: %w", err))
+		return nil, fmt.Errorf("nothing is composed: %w", err)
 	}
 	bus, err := aBus()
 	if err != nil {
-		return all(fmt.Errorf("nothing is composed: the bus: %w", err))
+		return nil, fmt.Errorf("nothing is composed: the bus: %w", err)
 	}
 	defer bus.Close()
 	js, err := jetstream.New(bus.Conn())
 	if err != nil {
-		return all(err)
+		return nil, err
 	}
 	sign, err := controllerSigner(ctx, open)
 	if err != nil {
-		return all(fmt.Errorf("nothing is composed: the controller's signing key: %w", err))
+		return nil, fmt.Errorf("nothing is composed: the controller's signing key: %w", err)
 	}
 	fps, err := currentFingerprints(ctx, open.inventory)
 	if err != nil {
-		return all(err)
+		return nil, err
 	}
 	entries, err := open.inventory.Catalogued(ctx)
 	if err != nil {
-		return all(err)
+		return nil, err
 	}
 	manifests := make(map[string]catalogue.Manifest, len(entries))
 	for _, e := range entries {
@@ -79,7 +56,7 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
 	}
 	upgrades, err := open.inventory.Upgrades(ctx)
 	if err != nil {
-		return all(err)
+		return nil, err
 	}
 	policy := func(module string) (string, string, string) {
 		u, ok := upgrades[module]
@@ -90,101 +67,59 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
 	}
 	sort.Strings(nodes)
 	out := map[string][]string{}
-	errsOf := map[string][]error{}
+	var errs []error
 	for _, node := range nodes {
-		// One composition of a node at a time in this process: two at once collided on its keys' revisions (issue 1945).
-		unlock := lockComposing(node)
-		func() {
-			defer unlock()
-			at, err := openComposedPlace(ctx, js, bus.Conn(), node)
-			if err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
-				return
-			}
-			at.sign, at.policy = sign, policy
-			if composeTrace != nil {
-				traceInput(ctx, open, node)
-			}
-			declared, err := composeSend(ctx, open, node)
-			if err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
-				return
-			}
-			body, err := compositionBody(declared)
-			if err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
-				return
-			}
-			// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
-			if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: its provisioner's plans: %w", node, err))
-			}
-			comps, moved, left, err := compositionsOf(node, body, fps, manifests, declared.modules, namedByOwnComposition(ctx, at))
-			if err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
-				return
-			}
-			if len(moved) > 0 {
-				fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
-					node, strings.Join(moved, ", "))
-			}
-			if len(left) > 0 {
-				fmt.Printf("compositions: %s's %s are left out of the node's own: the controller knows no fingerprint of the "+
-					"new form for them, and the node's want or have names their own composition (issue 1945)\n",
-					node, strings.Join(left, ", "))
-			}
-			if composeTrace != nil {
-				traceComposed(ctx, at, node, comps)
-			}
-			wrote, err := writeNodeCompositions(ctx, at, comps)
-			if len(wrote) > 0 {
-				out[node] = wrote
-				fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", "))
-			}
-			// And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232).
-			if len(declared.Bindings) > 0 {
-				if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil {
-					errsOf[node] = append(errsOf[node], fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err))
-				}
-			}
-			if err != nil {
-				errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
-			}
-		}()
-	}
-	byNode := map[string]error{}
-	for node, errs := range errsOf {
-		if len(errs) > 0 {
-			byNode[node] = errors.Join(errs...)
+		at, err := openComposedPlace(ctx, js, bus.Conn(), node)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", node, err))
+			continue
 		}
-	}
-	return out, byNode
-}
-
-// namedByOwnComposition says a module's want or have on the node names a composition of its own: a fingerprint of the
-// new form (issue 1945).
-func namedByOwnComposition(ctx context.Context, at composedPlace) func(module string) (bool, error) {
-	return func(module string) (bool, error) {
-		if at.wanted != nil {
-			fp, err := at.wanted(ctx, module)
-			if err != nil {
-				return false, err
-			}
-			if strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
-				return true, nil
+		at.sign, at.policy = sign, policy
+		if composeTrace != nil {
+			traceInput(ctx, open, node)
+		}
+		declared, err := composeSend(ctx, open, node)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", node, err))
+			continue
+		}
+		body, err := compositionBody(declared)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", node, err))
+			continue
+		}
+		// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
+		if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
+			errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err))
+		}
+		comps, moved, err := compositionsOf(node, body, fps, manifests, declared.modules)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", node, err))
+			continue
+		}
+		if len(moved) > 0 {
+			fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
+				node, strings.Join(moved, ", "))
+		}
+		if composeTrace != nil {
+			traceComposed(ctx, at, node, comps)
+		}
+		wrote, err := writeNodeCompositions(ctx, at, comps)
+		if len(wrote) > 0 {
+			out[node] = wrote
+			fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", "))
+		}
+		// And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232).
+		if len(declared.Bindings) > 0 {
+			if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil {
+				errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err))
 			}
 		}
-		if at.ran != nil {
-			h, found, err := at.ran(ctx, module)
-			if err != nil {
-				return false, err
-			}
-			if found && strings.HasPrefix(h.FP, sdkdelivery.SourcePrefix2) {
-				return true, nil
-			}
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", node, err))
 		}
-		return false, nil
 	}
+	return out, errors.Join(errs...)
 }
 
 // composeAndSay is composeModules for an act that has nothing to do with its answer but say it: what each former send
@@ -435,13 +370,3 @@ func compositionBody(declared sendable) ([]byte, error) {
 	declared.fileReferences = false
 	return declared.Body()
 }
-
-// composingLocks hold one composition of a node at a time in this process (issue 1945).
-var composingLocks sync.Map
-
-func lockComposing(node string) func() {
-	l, _ := composingLocks.LoadOrStore(node, &sync.Mutex{})
-	mu := l.(*sync.Mutex)
-	mu.Lock()
-	return mu.Unlock
-}

cmd/mesh-controller/composed_objects_test.go

@@ -26,7 +26,7 @@ func TestALargeFileOfACompositionHasItsObjectInTheBucket(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	comps, _, _, err := compositionsOf("novox", body, nil, nil, nil, nil)
+	comps, _, err := compositionsOf("novox", body, nil, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -50,7 +50,7 @@ func TestACompositionNamingAnObjectTheBucketLacksIsRefused(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	comps, _, _, err := compositionsOf("novox", body, nil, nil, nil, nil)
+	comps, _, err := compositionsOf("novox", body, nil, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -78,7 +78,7 @@ func TestEveryFilledFileStaysFilledInTheCompositions(t *testing.T) {
 		for _, m := range []string{"power", "claude-code", "node-env"} {
 			delete(fps, m)
 		}
-		comps, _, _, err := compositionsOf(node, body, fps, nil, nil, nil)
+		comps, _, err := compositionsOf(node, body, fps, nil, nil)
 		if err != nil {
 			t.Fatalf("%s: %v", node, err)
 		}

cmd/mesh-controller/composed_rejudge.go

@@ -134,7 +134,7 @@ func watchHavesForJudgement(ctx context.Context, open *stores) {
 		fmt.Printf("compositions: the haves are not watched: %v\n", err)
 		return
 	}
-	goServing(ctx, func() {
+	go func() {
 		defer bus.Close()
 		defer w.Stop()
 		for {
@@ -165,5 +165,5 @@ func watchHavesForJudgement(ctx context.Context, open *stores) {
 				})
 			}
 		}
-	})
+	}()
 }

cmd/mesh-controller/composed_served_integration_test.go

@@ -608,11 +608,8 @@ func keepWhatServeSets(t *testing.T) {
 	t.Setenv(servedVar, "")
 	unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents,
 		link.Calls.Follow, conditionsFrom, handActConn
-	doctor, asker := doctorFrom, askerFrom.Load()
 	t.Cleanup(func() {
 		logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn =
 			unheld, status, given, check, follow, keeper, hand
-		doctorFrom = doctor
-		askerFrom.Store(asker)
 	})
 }

cmd/mesh-controller/composed_writer.go

@@ -67,15 +67,8 @@ type composedPlace struct {
 // fingerprint fps gives it, the node's own with its adoption and provisioner, and every resource of a module fps names
 // no fingerprint for in the node's own, its module taken off it and named in moved. Data says the module's manifest
 // declares data.
-//
-// **A module a want or a have names by its own composition is never in the node's own** (issue 1945): the node-engine
-// runs that module's composition, and the same resources in the node's own were declared twice, which it refuses
-// whole ("2 resources declare X", every node's apply failing from 2026-10-25 12:43). named says, for a module fps names
-// no fingerprint of the new form for, that the node's want or have names a fingerprint of the new form; such a module's
-// resources and filled paths are left out of this pass (its own composition, as written before, stays what the node
-// runs) and it is named in left.
-func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string,
-	named func(module string) (bool, error)) (out map[string]sdkdelivery.Composed, moved, left []string, err error) {
+func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string) (
+	out map[string]sdkdelivery.Composed, moved []string, err error) {
 
 	var envelope struct {
 		Resources   []map[string]json.RawMessage `json:"resources"`
@@ -84,9 +77,9 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 		Provisioner string                       `json:"provisioner_account"`
 	}
 	if err := json.Unmarshal(body, &envelope); err != nil {
-		return nil, nil, nil, fmt.Errorf("%s's declaration: %w", node, err)
+		return nil, nil, fmt.Errorf("%s's declaration: %w", node, err)
 	}
-	inModule, ownComposition := map[string]bool{}, map[string]bool{}
+	inModule := map[string]bool{}
 	resources := make([]json.RawMessage, 0, len(envelope.Resources))
 	for _, r := range envelope.Resources {
 		var module string
@@ -94,30 +87,15 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 			_ = json.Unmarshal(raw, &module)
 		}
 		if module != "" && !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) {
-			if !inModule[module] && !ownComposition[module] {
-				isNamed := false
-				if named != nil {
-					if isNamed, err = named(module); err != nil {
-						return nil, nil, nil, fmt.Errorf("whether %s's want or have names %s's own composition: %w",
-							node, module, err)
-					}
-				}
-				if isNamed {
-					ownComposition[module] = true
-					left = append(left, module)
-				} else {
-					inModule[module] = true
-					moved = append(moved, module)
-				}
-			}
-			if ownComposition[module] {
-				continue
-			}
 			delete(r, "module")
+			if !inModule[module] {
+				inModule[module] = true
+				moved = append(moved, module)
+			}
 		}
 		raw, err := json.Marshal(r)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		resources = append(resources, raw)
 	}
@@ -128,26 +106,22 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 	// and shanks).
 	var filled, ownFilled []string
 	for _, p := range envelope.Filled {
-		switch {
-		case filledByMoved(envelope.Resources, p, ownComposition):
-			// The file of a module whose own composition the node runs: that composition names it, not this pass.
-		case filledByMoved(envelope.Resources, p, inModule):
+		if filledByMoved(envelope.Resources, p, inModule) {
 			ownFilled = append(ownFilled, p)
-		default:
+		} else {
 			filled = append(filled, p)
 		}
 	}
 	sort.Strings(moved)
-	sort.Strings(left)
 	out, err = sdkdelivery.SplitComposed(node, resources, fps, filled,
 		sdkdelivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 	if len(ownFilled) > 0 {
 		own, err := sdkdelivery.ComposedNodeKey(node)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		c := out[own]
 		c.Filled = append(c.Filled, ownFilled...)
@@ -175,7 +149,7 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 			out[key] = c
 		}
 	}
-	return out, moved, left, nil
+	return out, moved, nil
 }
 
 // filledByMoved says a filled path is a file of a module whose resources were moved into the node's own composition.
@@ -244,39 +218,16 @@ func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdeliv
 		}
 		wrote = true
 	}
-	// **The digest kept is the digest of the composition the bucket holds** (issue 1945): another writer may write its
-	// composition and its digest between this one's composition and its digest, and the digest written last must not be
-	// of a composition the bucket no longer holds, or the assigner's want names it and the node applies nothing. After
-	// its digest, this writer reads the composition again and, when another holds the key, keeps that one's digest.
-	written := false
-	for follows := 0; ; follows++ {
-		judged, err := judgeMove(ctx, at, key, c, digest)
-		if composeTrace != nil {
-			composeTrace(fmt.Sprintf("%s: %s: written %v, judged %s %q user-list-only %v against %.19s, err %v", at.node, key,
-				wrote, judged.Judgement, judged.Words, judged.UserListOnly, judged.Against, err))
-		}
-		if err != nil {
-			return wrote || written, err
-		}
-		w, err := keepDigest(ctx, at.digests, key, judged)
-		written = written || w
-		if err != nil {
-			return wrote || written, err
-		}
-		now, rev, err := readComposed(ctx, at.bucket, key)
-		if err != nil {
-			return wrote || written, err
-		}
-		nowDigest, err := sdkdelivery.ComposedDigestOf(now)
-		if rev == 0 || err != nil || nowDigest == digest {
-			return wrote || written, nil
-		}
-		if follows >= 3 {
-			return wrote || written, fmt.Errorf("%s moved under this writer %d times; its digest is kept by the next pass",
-				key, follows+1)
-		}
-		c, digest = now, nowDigest
+	judged, err := judgeMove(ctx, at, key, c, digest)
+	if composeTrace != nil {
+		composeTrace(fmt.Sprintf("%s: %s: written %v, judged %s %q user-list-only %v against %.19s, err %v", at.node, key, wrote,
+			judged.Judgement, judged.Words, judged.UserListOnly, judged.Against, err))
 	}
+	if err != nil {
+		return wrote, err
+	}
+	written, err := keepDigest(ctx, at.digests, key, judged)
+	return wrote || written, err
 }
 
 // objectsHeld refuses a composition that names a file by a digest whose object the node's bucket does not hold: a node

cmd/mesh-controller/composed_writer_test.go

@@ -218,7 +218,7 @@ func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) {
 			Resources []json.RawMessage `json:"resources"`
 		}
 		_ = json.Unmarshal(body, &d)
-		comps, moved, _, err := compositionsOf(node, body, fps, nil, nil, nil)
+		comps, moved, err := compositionsOf(node, body, fps, nil, nil)
 		if err != nil {
 			t.Fatalf("%s: %v", node, err)
 		}
@@ -253,7 +253,7 @@ func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) {
 // unchanged it writes nothing; a changed input writes a new revision of that key alone, by compare-and-set.
 func TestACompositionIsWrittenOnceAndAgainOnlyWhenItChanges(t *testing.T) {
 	body, fps := capturedBody(t, "g14")
-	comps, _, _, err := compositionsOf("g14", body, fps, nil, nil, nil)
+	comps, _, err := compositionsOf("g14", body, fps, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -342,7 +342,7 @@ func TestCompositionsOutsideTheFiveNewestAreDeleted(t *testing.T) {
 func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) {
 	body := []byte(`{"declaration":1,"resources":[]}`)
 	fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)
-	comps, _, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"}, nil)
+	comps, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -360,54 +360,3 @@ func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) {
 		}
 	}
 }
-
-// 1945/AC16 (unit): a module the controller knows no fingerprint of the new form for, whose want or have on the node
-// names its own composition, is never in the node's own composition, neither its resources nor its filled paths, and is
-// named as left; a module neither names still goes with the node's own, its filled path with it. (Live from 2026-10-25
-// 12:43: zsh, power, restic and more were in both, and every node refused "2 resources declare X".)
-func TestAModuleItsWantNamesIsNeverInTheNodesOwnComposition(t *testing.T) {
-	body := []byte(`{"declaration":1,"resources":[
-		{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zsh/zshrc"},
-		{"id":"power.unit","type":"file","module":"power","path":"/etc/power.conf"},
-		{"id":"mesh-wireguard.conf","type":"file","module":"mesh-wireguard","path":"/etc/wireguard/mesh0.conf"}],
-		"filled_files":["/etc/zsh/zshrc","/etc/power.conf","/etc/wireguard/mesh0.conf"]}`)
-	asked := map[string]int{}
-	named := func(module string) (bool, error) {
-		asked[module]++
-		return module == "zsh" || module == "power", nil
-	}
-	comps, moved, left, err := compositionsOf("ace", body, map[string]string{}, nil, nil, named)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if strings.Join(left, ",") != "power,zsh" || strings.Join(moved, ",") != "mesh-wireguard" {
-		t.Errorf("left %v, moved %v; want power,zsh left and mesh-wireguard moved", left, moved)
-	}
-	if asked["zsh"] != 1 || asked["power"] != 1 {
-		t.Errorf("asked each module once: %v", asked)
-	}
-	own, _ := sdkdelivery.ComposedNodeKey("ace")
-	for key, c := range comps {
-		for _, id := range resourceIDs(t, c.Resources) {
-			if strings.HasPrefix(id, "zsh.") || strings.HasPrefix(id, "power.") {
-				t.Errorf("%s holds %s, which the module's own composition declares", key, id)
-			}
-		}
-		for _, p := range c.Filled {
-			if p == "/etc/zsh/zshrc" || p == "/etc/power.conf" {
-				t.Errorf("%s names %s as filled, which the module's own composition names", key, p)
-			}
-		}
-	}
-	if ids := strings.Join(resourceIDs(t, comps[own].Resources), " "); ids != "mesh-wireguard.conf" {
-		t.Errorf("the node's own composition holds %q, want mesh-wireguard.conf alone", ids)
-	}
-	if f := strings.Join(comps[own].Filled, " "); f != "/etc/wireguard/mesh0.conf" {
-		t.Errorf("the node's own composition names %q as filled, want mesh-wireguard's file alone", f)
-	}
-
-	if _, _, _, err := compositionsOf("ace", body, map[string]string{}, nil, nil,
-		func(string) (bool, error) { return false, fmt.Errorf("the wants bucket is gone") }); err == nil {
-		t.Errorf("a want that cannot be read composed the node anyway")
-	}
-}

cmd/mesh-controller/composer.go

@@ -1,105 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"sort"
-	"sync"
-)
-
-// **One composer, one pass at a time, each node once per pass** (issue 1945). A done key's take-in registers its module
-// and asks the composer for its nodes; the composer gathers every node asked for while a pass runs and composes them in
-// the next pass, in one composition of each node, then writes the grants once. Before, each take-in of a burst composed
-// every node of its module itself, at once: they collided on the compositions' revisions ("wrong last sequence … key
-// revision mismatch"), retried every 30 s, and the take-in that mattered waited behind them (2026-10-25 at 11:18 and
-// 12:32). A caller waits for the pass that began after it asked, and hears that pass's error.
-type composer struct {
-	composeNodes func(ctx context.Context, nodes []string) map[string]error
-	writeGrants  func(ctx context.Context) error
-
-	mu      sync.Mutex
-	pending map[string]bool
-	grants  bool
-	waiters []waiter
-	wake    chan struct{}
-}
-
-// waiter is a caller waiting for its pass: the nodes it asked for, whether it asked for the grants, and where its answer
-// goes.
-type waiter struct {
-	nodes  []string
-	grants bool
-	done   chan error
-}
-
-func newComposer(compose func(context.Context, []string) map[string]error, grants func(context.Context) error) *composer {
-	return &composer{composeNodes: compose, writeGrants: grants, pending: map[string]bool{}, wake: make(chan struct{}, 1)}
-}
-
-// compose asks for the nodes to be composed, and the grants written after when grants is true, and waits for the pass
-// that does it, with the errors of its own nodes alone.
-func (c *composer) compose(ctx context.Context, nodes []string, grants bool) error {
-	done := make(chan error, 1)
-	c.mu.Lock()
-	for _, n := range nodes {
-		c.pending[n] = true
-	}
-	c.grants = c.grants || grants
-	c.waiters = append(c.waiters, waiter{nodes: nodes, grants: grants, done: done})
-	c.mu.Unlock()
-	select {
-	case c.wake <- struct{}{}:
-	default:
-	}
-	select {
-	case err := <-done:
-		return err
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-// run makes the passes until ctx ends: each takes everything asked for since the last began.
-func (c *composer) run(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-c.wake:
-		}
-		c.mu.Lock()
-		nodes := make([]string, 0, len(c.pending))
-		for n := range c.pending {
-			nodes = append(nodes, n)
-		}
-		grants, waiters := c.grants, c.waiters
-		c.pending, c.grants, c.waiters = map[string]bool{}, false, nil
-		c.mu.Unlock()
-		if len(waiters) == 0 {
-			continue
-		}
-		sort.Strings(nodes)
-		var failed map[string]error
-		if len(nodes) > 0 {
-			failed = c.composeNodes(ctx, nodes)
-		}
-		// The grants whenever a caller asked for them, whatever a node's composition said: a node that failed fails its
-		// own callers alone (issue 1945).
-		var grantsErr error
-		if grants {
-			grantsErr = c.writeGrants(ctx)
-		}
-		for _, w := range waiters {
-			var errs []error
-			for _, n := range w.nodes {
-				if err := failed[n]; err != nil {
-					errs = append(errs, err)
-				}
-			}
-			if w.grants && grantsErr != nil {
-				errs = append(errs, grantsErr)
-			}
-			w.done <- errors.Join(errs...)
-		}
-	}
-}

cmd/mesh-controller/composer_test.go

@@ -1,143 +0,0 @@
-package main
-
-import (
-	"context"
-	"slices"
-	"sort"
-	"sync"
-	"testing"
-	"time"
-)
-
-// A burst of done keys composes each node once per pass and writes the grants once, never two passes at a time: the
-// take-ins of a burst each composed every node of their module at once, collided on the compositions' revisions and
-// retried every 30 s, while the one that mattered (mesh-host's) waited (issue 1945, live on 2026-10-25 at 11:18 and
-// 12:32).
-func TestABurstComposesEachNodeOncePerPass(t *testing.T) {
-	var mu sync.Mutex
-	var passes [][]string
-	grants := 0
-	running, maxRunning := 0, 0
-	release := make(chan struct{})
-	first := true
-	c := newComposer(func(_ context.Context, nodes []string) map[string]error {
-		mu.Lock()
-		running++
-		maxRunning = max(maxRunning, running)
-		passes = append(passes, slices.Clone(nodes))
-		wait := first
-		first = false
-		mu.Unlock()
-		if wait {
-			<-release // the first pass runs while the rest of the burst arrives
-		}
-		mu.Lock()
-		running--
-		mu.Unlock()
-		return nil
-	}, func(context.Context) error {
-		mu.Lock()
-		grants++
-		mu.Unlock()
-		return nil
-	})
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go c.run(ctx)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() { defer wg.Done(); _ = c.compose(ctx, []string{"ace"}, true) }()
-	time.Sleep(20 * time.Millisecond)
-	for _, nodes := range [][]string{{"ace", "g14"}, {"g14"}, {"shanks", "ace"}, {"novox"}, {"ace"}} {
-		wg.Add(1)
-		go func() { defer wg.Done(); _ = c.compose(ctx, nodes, true) }()
-	}
-	time.Sleep(20 * time.Millisecond)
-	close(release)
-	wg.Wait()
-
-	mu.Lock()
-	defer mu.Unlock()
-	if maxRunning != 1 {
-		t.Fatalf("%d passes ran at once", maxRunning)
-	}
-	if len(passes) != 2 {
-		t.Fatalf("passes %v: want the first and one more for the rest of the burst", passes)
-	}
-	second := slices.Clone(passes[1])
-	sort.Strings(second)
-	if !slices.Equal(second, []string{"ace", "g14", "novox", "shanks"}) {
-		t.Fatalf("the second pass composed %v: each node once", passes[1])
-	}
-	if grants != 2 {
-		t.Fatalf("the grants were written %d times for two passes", grants)
-	}
-}
-
-// A caller hears its own pass's failure, and only a pass that began after it asked counts as its own.
-func TestACallerHearsThePassThatComposedItsNodes(t *testing.T) {
-	calls := 0
-	c := newComposer(func(_ context.Context, nodes []string) map[string]error {
-		calls++
-		if calls == 1 {
-			return map[string]error{"ace": context.DeadlineExceeded}
-		}
-		return nil
-	}, func(context.Context) error { return nil })
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go c.run(ctx)
-	if err := c.compose(ctx, []string{"ace"}, false); err == nil {
-		t.Fatal("a failed pass was not heard")
-	}
-	if err := c.compose(ctx, []string{"ace"}, false); err != nil {
-		t.Fatalf("a good pass said %v", err)
-	}
-}
-
-// One node that fails its composition fails only the callers that asked for it, and the grants are written whenever a
-// caller asked for them (the review of issue 1945: one joined error failed every take-in of the pass and stopped their
-// grants).
-func TestANodeThatFailsFailsOnlyItsCallers(t *testing.T) {
-	grants := 0
-	release := make(chan struct{})
-	first := true
-	c := newComposer(func(_ context.Context, nodes []string) map[string]error {
-		if first {
-			first = false
-			<-release
-		}
-		out := map[string]error{}
-		for _, n := range nodes {
-			if n == "broken" {
-				out[n] = context.DeadlineExceeded
-			}
-		}
-		return out
-	}, func(context.Context) error { grants++; return nil })
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go c.run(ctx)
-	go func() { _ = c.compose(ctx, []string{"warm"}, false) }()
-	time.Sleep(20 * time.Millisecond)
-	errs := make(chan error, 2)
-	go func() { errs <- c.compose(ctx, []string{"ace"}, true) }()
-	go func() { errs <- c.compose(ctx, []string{"broken"}, false) }()
-	time.Sleep(20 * time.Millisecond)
-	close(release)
-	var failed, fine int
-	for range 2 {
-		if err := <-errs; err != nil {
-			failed++
-		} else {
-			fine++
-		}
-	}
-	if failed != 1 || fine != 1 {
-		t.Fatalf("failed %d, fine %d: only the caller of the broken node fails", failed, fine)
-	}
-	if grants != 1 {
-		t.Fatalf("the grants were written %d times; a caller asked for them", grants)
-	}
-}

cmd/mesh-controller/conditions.go

@@ -49,7 +49,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore)
 		Say:    func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
 		// What status leads with changed: composed again soon (a nudge outside the serving controller
 		// does nothing).
-		Changed: func() { statusFrom.nudge(); askerFrom.Load().nudge() },
+		Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
 		// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
 		Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
 }

cmd/mesh-controller/declaration_files.go

@@ -225,9 +225,6 @@ func filesNotPutObservation(node, summary string) conditions.Observation {
 // notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
 // clears the count and the condition.
 func notePut(ctx context.Context, node string, put error) {
-	if errors.Is(put, context.Canceled) || ctx.Err() != nil {
-		return // a send cut by a cancellation or a stop says nothing of the bucket (issue 1945)
-	}
 	filesFailMu.Lock()
 	if put == nil {
 		delete(filesFailed, node)

cmd/mesh-controller/declaration_files_test.go

@@ -415,7 +415,6 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
 	was := conditionsFrom
 	conditionsFrom = k
 	t.Cleanup(func() { conditionsFrom = was })
-	forgetPutFailures(t, "cond-node")
 	key := filesNotPutObservation("cond-node", "").Key()
 	open := func() bool {
 		_, found, err := conditions.ReadOne(t.Context(), store, key)
@@ -443,36 +442,6 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
 	}
 }
 
-// A send cut by a cancellation counts as no failure to put its files, however often (issue 1945: a cancellation is no fault
-// of the bucket).
-func TestAPutCutByACancellationRaisesNothing(t *testing.T) {
-	store := conditions.NewInMemory()
-	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}})
-	t.Cleanup(func() { k.Close(context.Background()) })
-	was := conditionsFrom
-	conditionsFrom = k
-	t.Cleanup(func() { conditionsFrom = was })
-	forgetPutFailures(t, "cut-node")
-	cut := fmt.Errorf("the files could not be put: %w", context.Canceled)
-	for range filesNotPutAfter + 1 {
-		notePut(t.Context(), "cut-node", cut)
-	}
-	if _, found, err := conditions.ReadOne(t.Context(), store, filesNotPutObservation("cut-node", "").Key()); err != nil || found {
-		t.Fatalf("sends cut by a cancellation raised the condition (found %v, %v)", found, err)
-	}
-}
-
-// forgetPutFailures starts the node's count of failed puts at nothing and leaves it so, as a run of the test again finds it.
-func forgetPutFailures(t *testing.T, node string) {
-	forget := func() {
-		filesFailMu.Lock()
-		delete(filesFailed, node)
-		filesFailMu.Unlock()
-	}
-	forget()
-	t.Cleanup(forget)
-}
-
 // aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
 type aBusThatSays struct {
 	link.Bus

cmd/mesh-controller/declared_packages_integration_test.go

@@ -87,29 +87,3 @@ func TestTheDeclaredPackagesMirrorNamesWhatEachNodesModulesDeclare(t *testing.T)
 		}
 	}
 }
-
-// 1945/AC-mirror-5: the digest the declared-packages mirror skips an unchanged pass by is made from a real reading of the
-// stores, is the same over two readings of the same records, and moves when an assignment does.
-func TestTheInputDigestOfTheStoresIsStable(t *testing.T) {
-	open := aMesh(t)
-	ctx := t.Context()
-	register(t, open, catalogue.Manifest{Module: "viewer", Version: "1", Resources: []map[string]any{
-		{"id": "pkg-curl", "type": "package", "package": "curl"}}})
-	read := func() string {
-		in, err := readComposeInput(ctx, open)
-		if err != nil {
-			t.Fatal(err)
-		}
-		return inputDigest(in)
-	}
-	first, second := read(), read()
-	if first == "" || first != second {
-		t.Fatalf("two readings of the same records gave the digests %q and %q", first, second)
-	}
-	if _, err := assign(ctx, open, "anchor", "viewer"); err != nil {
-		t.Fatalf("assign: %v", err)
-	}
-	if read() == first {
-		t.Errorf("an assignment left the digest of the stores' Input at %s", first)
-	}
-}

cmd/mesh-controller/fingerprints_at_test.go

@@ -1,93 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-
-	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
-
-	"github.com/novox/mesh-controller/internal/builderkeys"
-)
-
-// 1945/AC15 (unit): a module registered at a commit no build record points at (a catalogue merge re-indexed it with
-// the fingerprint it had, and the take-in registered it from the done key whose record stands at its first commit)
-// has, as its current fingerprint, the one the indexer's trunk value at that commit names; a module whose commit has a
-// build keeps that build's; a module the indexer names nothing of the new form for has none; an indexer error is the
-// answer's, never a module silently left out.
-func TestACommitWithNoBuildOfItsOwnTakesTheIndexersFingerprint(t *testing.T) {
-	zsh, board := grantFP('a'), grantFP('b')
-	commits := map[string]string{"zsh": "68f7431d98d7", "board": "2167b95a1111", "legacy": "abcdef012345", "held": ""}
-	built := func(_ context.Context, module, commit string) (string, bool, error) {
-		if module == "board" {
-			return board, true, nil
-		}
-		return "", false, nil
-	}
-	indexed := func(_ context.Context, module, commit string) (string, error) {
-		switch module {
-		case "zsh":
-			return zsh, nil
-		case "board":
-			t.Errorf("the indexer was asked for board, whose commit has a build")
-		case "held":
-			t.Errorf("the indexer was asked for a module with no commit")
-		}
-		return "", nil
-	}
-	got, err := fingerprintsAt(context.Background(), commits, built, indexed)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got["zsh"] != zsh || got["board"] != board || len(got) != 2 {
-		t.Errorf("got %v, want zsh at %s and board at %s alone", got, zsh, board)
-	}
-
-	broken := func(context.Context, string, string) (string, error) { return "", errors.New("the bus is gone") }
-	if _, err := fingerprintsAt(context.Background(), commits, built, broken); err == nil {
-		t.Errorf("an indexer error was not the answer's")
-	}
-}
-
-// fakeTrunk is the indexer's trunk values of one module: now, and its history by commit.
-type fakeTrunk struct {
-	now     *sdkdelivery.Module
-	history map[string]sdkdelivery.Module
-	asked   []string
-}
-
-func (f *fakeTrunk) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) {
-	f.asked = append(f.asked, commit)
-	if commit == "" {
-		if f.now == nil {
-			return sdkdelivery.Module{}, fmt.Errorf("%w (%s)", builderkeys.ErrNoValue, module)
-		}
-		return *f.now, nil
-	}
-	m, ok := f.history[commit]
-	if !ok {
-		return sdkdelivery.Module{}, fmt.Errorf("%w (%s at %s)", builderkeys.ErrNoValue, module, commit)
-	}
-	return m, nil
-}
-
-// 1945/AC15 (unit): the indexer's fingerprint at a commit is read from the value it holds now when that is the
-// commit's (one read), else from the key's history at the commit; none held is no fingerprint.
-func TestTheIndexersFingerprintAtACommit(t *testing.T) {
-	now := sdkdelivery.Module{Commit: "68f7431d98d7a1279cba3a615b6e9f22c3295783", FP2: grantFP('a')}
-	f := &fakeTrunk{now: &now}
-	if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "68f7431d98d7a1279cba3a615b6e9f22c3295783"); err != nil || fp != grantFP('a') || len(f.asked) != 1 {
-		t.Errorf("now's commit: %q (%v), asked %v", fp, err, f.asked)
-	}
-	f = &fakeTrunk{now: &now, history: map[string]sdkdelivery.Module{"2167b95a": {Commit: "2167b95a", FP2: grantFP('c')}}}
-	if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "2167b95a"); err != nil || fp != grantFP('c') {
-		t.Errorf("an older commit: %q (%v)", fp, err)
-	}
-	if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "99999999"); err != nil || fp != "" {
-		t.Errorf("a commit the indexer never wrote: %q (%v)", fp, err)
-	}
-	if fp, err := indexedFingerprintAt(context.Background(), &fakeTrunk{}, "zsh", "68f7431d"); err != nil || fp != "" {
-		t.Errorf("no value at all: %q (%v)", fp, err)
-	}
-}

cmd/mesh-controller/given.go

@@ -53,61 +53,12 @@ func startedOn(h sdkdelivery.Have, written time.Time) time.Time {
 
 const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
 
-// replaceGiven replaces what a module's good start makes due, composes the machines, and says so. One at a time for a
-// node: a call that finds one running leaves what it was asked for as a mark, and the one running takes it once it is
-// done, so a start heard meanwhile is never lost (issue 1945, the kind of the dropped wake).
+// replaceGiven replaces what a module's good start makes due, composes the machines, and says so.
 func replaceGiven(ctx context.Context, open *stores, node string, started map[string]time.Time) {
 	if _, busy := replacing.LoadOrStore(node, true); busy {
-		markGivenAgain(node, started)
 		return
 	}
-	for {
-		replaceGivenOnce(ctx, open, node, started)
-		replacing.Delete(node)
-		// Read after letting go: a call meanwhile only marked the node.
-		pending := takeGivenMark(node)
-		if pending == nil || ctx.Err() != nil {
-			return
-		}
-		if _, busy := replacing.LoadOrStore(node, true); busy {
-			markGivenAgain(node, pending) // the one running now takes it
-			return
-		}
-		started = pending
-	}
-}
-
-// givenAgain is, by node, the starts asked for while a replacement ran there, merged.
-var givenAgain = struct {
-	sync.Mutex
-	by map[string]map[string]time.Time
-}{by: map[string]map[string]time.Time{}}
-
-func markGivenAgain(node string, started map[string]time.Time) {
-	givenAgain.Lock()
-	defer givenAgain.Unlock()
-	m := givenAgain.by[node]
-	if m == nil {
-		m = map[string]time.Time{}
-		givenAgain.by[node] = m
-	}
-	for module, at := range started {
-		if at.After(m[module]) {
-			m[module] = at
-		}
-	}
-}
-
-func takeGivenMark(node string) map[string]time.Time {
-	givenAgain.Lock()
-	defer givenAgain.Unlock()
-	m := givenAgain.by[node]
-	delete(givenAgain.by, node)
-	return m
-}
-
-// replaceGivenOnce is one replacement on a node; a variable so a test can stand in for it.
-var replaceGivenOnce = func(ctx context.Context, open *stores, node string, started map[string]time.Time) {
+	defer replacing.Delete(node)
 	replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, node, started)
 	if err != nil {
 		log.Printf("a value given by hand on %s could not be replaced after its module started: %v", node, err)
@@ -169,7 +120,7 @@ func watchHavesForGiven(ctx context.Context, open *stores) {
 		log.Printf("given values: the haves are not watched: %v", err)
 		return
 	}
-	goServing(ctx, func() {
+	go func() {
 		defer bus.Close()
 		defer w.Stop()
 		for {
@@ -198,5 +149,5 @@ func watchHavesForGiven(ctx context.Context, open *stores) {
 				inBackground(func() { replaceGiven(ctx, open, node, map[string]time.Time{module: at}) })
 			}
 		}
-	})
+	}()
 }

cmd/mesh-controller/given_again_test.go

@@ -1,34 +0,0 @@
-package main
-
-import (
-	"context"
-	"sync"
-	"testing"
-	"time"
-)
-
-// A start heard while a replacement of given values runs on the node is not lost: it is marked, and the replacement
-// running takes it once it is done (issue 1945, the kind of the dropped wake: replaceGiven returned at once when busy).
-func TestAStartHeardDuringAReplacementIsTaken(t *testing.T) {
-	was := replaceGivenOnce
-	t.Cleanup(func() { replaceGivenOnce = was })
-	var mu sync.Mutex
-	var asked []map[string]time.Time
-	at := time.Date(2026, 10, 25, 13, 0, 0, 0, time.UTC)
-	replaceGivenOnce = func(ctx context.Context, open *stores, node string, started map[string]time.Time) {
-		mu.Lock()
-		asked = append(asked, started)
-		first := len(asked) == 1
-		mu.Unlock()
-		if first {
-			// Another module starts on the node while this one is being replaced.
-			replaceGiven(ctx, open, node, map[string]time.Time{"redis": at})
-		}
-	}
-	replaceGiven(context.Background(), nil, "ace", map[string]time.Time{"postgres": at})
-	mu.Lock()
-	defer mu.Unlock()
-	if len(asked) != 2 || !asked[1]["redis"].Equal(at) {
-		t.Fatalf("asked %v: the start heard during the replacement was lost", asked)
-	}
-}

cmd/mesh-controller/grants.go

@@ -11,7 +11,6 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"github.com/novox/mesh-controller/internal/broker"
-	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -56,128 +55,29 @@ func grantsFrom(principals []broker.Principal, hashes map[string]string, current
 
 // currentFingerprints is, by module, the fingerprint of the build each registered module is registered at: its
 // manifest's commit (built_from), that commit's newest successful build, and the fingerprint that build names.
-//
-// **A commit with no build of its own asks the indexer** (issue 1945, live from 2026-10-25 12:43). A catalogue merge
-// that leaves a module's sources alone re-indexes it at the merge's commit with the fingerprint it had: the take-in
-// registers the module at the new commit from the same done key, whose build record (one per key and revision) already
-// stands at the commit it was first taken in at, so the new commit has no build pointing at it. Read only by the
-// commit, such a module had no fingerprint of the new form: it was composed into the node's own composition while its
-// want named its own, and every node declared its files twice. The indexer's trunk value at the commit names the
-// fingerprint the module was registered from, so that is its fingerprint.
 func currentFingerprints(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
 	builds, err := inv.CurrentBuilds(ctx)
 	if err != nil {
 		return nil, err
 	}
-	commits := make(map[string]string, len(builds))
-	for module, b := range builds {
-		commits[module] = b.Commit
-	}
-	var indexed func(ctx context.Context, module, commit string) (string, error)
-	var closeIndexed func()
-	defer func() {
-		if closeIndexed != nil {
-			closeIndexed()
-		}
-	}()
-	return fingerprintsAt(ctx, commits,
-		func(ctx context.Context, module, commit string) (string, bool, error) {
-			id, err := inv.BuildOf(ctx, module, commit)
-			if err != nil || id == "" {
-				return "", false, nil
-			}
-			build, found, err := inv.BuildByID(ctx, id)
-			if err != nil || !found {
-				return "", false, nil
-			}
-			return build.SourceFingerprint, true, nil
-		},
-		func(ctx context.Context, module, commit string) (string, error) {
-			if indexed == nil {
-				read, done, err := openIndexedFingerprints()
-				if err != nil {
-					return "", err
-				}
-				indexed, closeIndexed = read, done
-			}
-			return indexed(ctx, module, commit)
-		})
-}
-
-// fingerprintsAt is, by module, the fingerprint of the build registered at its commit (built), or else the fingerprint
-// of the new form the indexer's trunk value at that commit names (indexed). A module with no commit (held without a
-// source) has none, and a module neither knows has none. An error of the indexer's is the answer's: a module left out
-// for it would be composed into the node's own composition beside its own (issue 1945).
-func fingerprintsAt(ctx context.Context, commits map[string]string,
-	built func(ctx context.Context, module, commit string) (string, bool, error),
-	indexed func(ctx context.Context, module, commit string) (string, error)) (map[string]string, error) {
-	modules := make([]string, 0, len(commits))
-	for module := range commits {
-		modules = append(modules, module)
-	}
-	sort.Strings(modules)
 	out := map[string]string{}
-	for _, module := range modules {
-		commit := commits[module]
-		if commit == "" {
+	for module, b := range builds {
+		if b.Commit == "" {
 			continue
 		}
-		fp, found, err := built(ctx, module, commit)
-		if err != nil {
-			return nil, err
-		}
-		if found {
-			out[module] = fp
+		id, err := inv.BuildOf(ctx, module, b.Commit)
+		if err != nil || id == "" {
 			continue
 		}
-		fp, err = indexed(ctx, module, commit)
-		if err != nil {
-			return nil, fmt.Errorf("the fingerprint %s is registered at (%.8s): %w", module, commit, err)
-		}
-		if strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
-			out[module] = fp
+		build, found, err := inv.BuildByID(ctx, id)
+		if err != nil || !found {
+			continue
 		}
+		out[module] = build.SourceFingerprint
 	}
 	return out, nil
 }
 
-// openIndexedFingerprints opens the indexer's trunk values: what reads the fingerprint of the new form a module's value
-// at a commit names ("" when it holds none for that commit), and what closes the bus connection it opened.
-var openIndexedFingerprints = func() (func(ctx context.Context, module, commit string) (string, error), func(), error) {
-	keys, closeKeys, err := openBuilderKeys()
-	if err != nil {
-		return nil, nil, fmt.Errorf("the indexer's trunk values: %w", err)
-	}
-	return func(ctx context.Context, module, commit string) (string, error) {
-		return indexedFingerprintAt(ctx, keys, module, commit)
-	}, closeKeys, nil
-}
-
-// indexedFingerprintAt is the fingerprint of the new form the indexer's value of module at commit names: the value it
-// holds now when that is the commit's (one read), else the newest of the key's history at the commit; "" when none.
-func indexedFingerprintAt(ctx context.Context, keys interface {
-	ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
-}, module, commit string) (string, error) {
-	m, err := keys.ModuleAt(ctx, module, "")
-	if err == nil && sameCommitID(m.Commit, commit) {
-		return m.FP2, nil
-	}
-	if err != nil && !errors.Is(err, builderkeys.ErrNoValue) {
-		return "", err
-	}
-	if errors.Is(err, builderkeys.ErrNoValue) {
-		return "", nil
-	}
-	m, err = keys.ModuleAt(ctx, module, commit)
-	if errors.Is(err, builderkeys.ErrNoValue) {
-		return "", nil
-	}
-	if err != nil {
-		return "", err
-	}
-	return m.FP2, nil
-}
-
 // grantsKV is what writing the grants needs of their bucket.
 type grantsKV interface {
 	Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)

cmd/mesh-controller/new_modules.go

@@ -223,10 +223,10 @@ func watchNewModules(ctx context.Context, open *stores) {
 		fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
 		return
 	}
-	goServing(ctx, func() {
+	go func() {
 		<-ctx.Done()
 		closeKeys()
-	})
+	}()
 }
 
 // wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep

cmd/mesh-controller/plain_words.go

@@ -59,17 +59,6 @@ func dataWords(o conditions.Observation) (what, module string) {
 }
 
 var plainWordings = map[string]func(conditions.Observation) words{
-	// A module's new build whose compositions or grant the controller could not write (composed_keys.go).
-	kindNotComposed: worded(func(o conditions.Observation) words {
-		m := "a module"
-		if p := idPart(o, 0); o.Scope == conditions.ScopeBuild && p != "" {
-			m = p // the module: raised in the build's scope, its id the module's name
-		}
-		return words{Headline: "A new build of " + m + " cannot be delivered",
-			Explanation: fmt.Sprintf("A new build of %s was made, and the mesh could not prepare it for the machines, so "+
-				"none can be given it yet. It is tried again every half minute; the details say what fails.", m),
-			Resolved: "the new build of " + m + " can be delivered"}
-	}),
 	// Machines and their node-engine.
 	"silent": worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")

cmd/mesh-controller/push.go

@@ -122,15 +122,7 @@ func serve(ctx context.Context) (err error) {
 	defer theLease.release()
 	ctx, stopActing := context.WithCancel(ctx)
 	defer stopActing()
-	// Every goroutine started from here is counted, and ended and waited for before serve returns (serving_group.go):
-	// on the way out by error here, and at the end before anything else is undone.
-	ctx, running := withServingGroup(ctx)
-	stopRunning := func() {
-		stopActing()
-		running.stop()
-	}
-	defer stopRunning()
-	goServing(ctx, func() {
+	go func() {
 		select {
 		case <-lost:
 			stopActing()
@@ -140,7 +132,7 @@ func serve(ctx context.Context) (err error) {
 		// asks the controller after this one rather than have a command started and killed with this
 		// process (novox/hq issue 289).
 		handingOver.Store(true)
-	})
+	}()
 	defer func() {
 		select {
 		case <-lost:
@@ -167,16 +159,6 @@ func serve(ctx context.Context) (err error) {
 	if err := attachTheChain(ctx, inv, server); err != nil {
 		return err
 	}
-	bus, isNATS := server.Bus().(link.OverNATS)
-	if !isNATS {
-		return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
-	}
-	// The condition store, opened before anything that raises or reads a condition starts beside the serving, so none
-	// of them meets conditionsFrom being written (issue 1945); closed after they all ended.
-	keeper := openServingConditions(ctx, open, bus)
-	if keeper != nil {
-		defer closeConditions(keeper)
-	}
 
 	// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
 	// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
@@ -191,7 +173,7 @@ func serve(ctx context.Context) (err error) {
 	// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
 	// is registered, ended with why when its build will not register it, raised and cleared as conditions.
 	// Never by a read.
-	goServing(ctx, func() { settlingPending(ctx, open) })
+	go settlingPending(ctx, open)
 	// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
 	// And what the catalogue decided a build meant. The builder's own result is already handled
 	// above; this is the other half — the control plane is the only one of the three that knows
@@ -239,6 +221,10 @@ func serve(ctx context.Context) (err error) {
 			"Those answer the reason when called; everything else is served as usual\n",
 			catalogue.ControllerSeatName, strings.Join(behind, ", "))
 	}
+	bus, isNATS := server.Bus().(link.OverNATS)
+	if !isNATS {
+		return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
+	}
 	// The hand-act log is counted for `status` on this connection rather than a new one a minute; no longer serving, the
 	// connection is closed, and a verb run after in this process dials its own (a served test's leftover failed the
 	// next test's `conditions` with "connection closed", issue 1945).
@@ -258,18 +244,18 @@ func serve(ctx context.Context) (err error) {
 		return err
 	}
 	// Composed now and kept current, before the verb that answers from it is served.
-	goServing(ctx, func() { statusFrom.keep(ctx) })
+	go statusFrom.keep(ctx)
 	// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
 	// controller holds the lease, read by the build seat from the artifact store.
-	goServing(ctx, func() { exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion) })
+	go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
 	// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
 	// declared config on the bus, written from the store only when a record changed, by the lease holder.
 	// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
 	// assignment.
 	inv.SetProvisioningPlans(inv.SentProvisioningPlans)
-	goServing(ctx, func() { mirroringRecords(ctx, open, inv, bus.Conn, bus) })
+	go mirroringRecords(ctx, open, inv, bus.Conn, bus)
 	// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
-	goServing(ctx, func() { sweepingDeclarationFiles(ctx, inv, bus.Conn) })
+	go sweepingDeclarationFiles(ctx, inv, bus.Conn)
 	// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
 	// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
 	watchNewModules(ctx, open)
@@ -301,7 +287,7 @@ func serve(ctx context.Context) (err error) {
 	// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
 	// is said and the controller serves on: status then says the conditions cannot be read, and is
 	// not well — louder than not serving at all, and the push that repairs the bus still runs.
-	if stopWatching := watchTheMesh(ctx, open, server, bus, keeper); stopWatching != nil {
+	if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
 		defer stopWatching()
 	}
 	stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
@@ -316,7 +302,7 @@ func serve(ctx context.Context) (err error) {
 	}
 	defer stopCLI()
 	// Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762).
-	goServing(ctx, func() { sayReadCalls(ctx) })
+	go sayReadCalls(ctx)
 	// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
 	stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
 	if err != nil {
@@ -324,10 +310,7 @@ func serve(ctx context.Context) (err error) {
 	}
 	defer stopAnnouncing()
 
-	err = server.Serve(ctx)
-	// Ended and waited for before the verbs stop being served and the globals are put back, not after.
-	stopRunning()
-	return err
+	return server.Serve(ctx)
 }
 
 // OverlayCIDRVar is the range the mesh allocates node addresses from.

cmd/mesh-controller/record_mirrors.go

@@ -2,15 +2,12 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"sort"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/compose"
-	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 
@@ -27,8 +24,7 @@ import (
 // so no one call site sees every change; what every change has in common is that it is committed to the
 // store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
 // the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
-// mirror is therefore at most mirrorEvery behind its store. The declared-packages mirror is the exception: it composes
-// every node, so it passes on a change of what it composes from, not on this tick (keepingDeclaredPackages, issue 1945).
+// mirror is therefore at most mirrorEvery behind its store.
 
 // mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
 const mirrorEvery = 10 * time.Second
@@ -49,31 +45,21 @@ func mirroringRecords(ctx context.Context, open *stores, inv *inventory.Inventor
 		return e
 	}
 	sources := map[string]func(context.Context) (map[string][]byte, error){
-		recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
-		recordmirror.Predefined.Name:     inv.PredefinedMirror,
+		recordmirror.DeclaredConfig.Name:   inv.DeclaredConfigMirror,
+		recordmirror.Predefined.Name:       inv.PredefinedMirror,
+		recordmirror.DeclaredPackages.Name: declaredPackagesMirror(open),
 	}
-	packages := keepingDeclaredPackages(ctx, open, api, announce, epoch, holds)
-	held := false
 	writers := map[string]*recordmirror.Writer{}
 	failing := map[string]string{}
 	tick := time.NewTicker(mirrorEvery)
 	defer tick.Stop()
 	events := &assignmentEvents{}
 	for {
-		now := holds()
-		if now && !held {
-			// The lease taken: the declared-packages mirror is brought to the stores at once, not at its next change.
-			packages.poke()
-		}
-		held = now
-		if now {
+		if holds() {
 			// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
 			// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
 			keepingAssignments(ctx, inv, announce, epoch, events, failing)
 			for _, m := range recordmirror.Mirrors() {
-				if sources[m.Name] == nil {
-					continue // the declared packages, kept on a change (keepingDeclaredPackages)
-				}
 				why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
 				if why != failing[m.Name] {
 					if why != "" {
@@ -193,263 +179,35 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
 	events.known = now
 }
 
-// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5) over a fresh reading of the stores.
+// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5): each node's composition, asked as a
+// question that makes nothing, and the packages it declares with the modules that declare them. A node whose own set does
+// not compose has no key, and any other failure to read is the pass's error, so a key is never purged for a store that
+// could not be read.
 func declaredPackagesMirror(open *stores) func(context.Context) (map[string][]byte, error) {
 	return func(ctx context.Context) (map[string][]byte, error) {
 		in, err := inputFor(ctx, open)
 		if err != nil {
 			return nil, err
 		}
-		return declaredPackagesFrom(ctx, in)
-	}
-}
-
-// declaredPackagesFrom is the declared-packages mirror over an Input: each node's composition, asked as a question that makes
-// nothing, and the packages it declares with the modules that declare them. A node whose own set does not compose has no key,
-// and any other failure to read is the pass's error, so a key is never purged for a store that could not be read.
-func declaredPackagesFrom(_ context.Context, in *meshInput) (map[string][]byte, error) {
-	// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
-	asked := in.Input
-	asked.Nodes = append([]compose.Node(nil), in.Nodes...)
-	for i := range asked.Nodes {
-		asked.Nodes[i].ReadsModules = true
-	}
-	out := map[string][]byte{}
-	for _, n := range in.sortedNodes() {
-		d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
-		if compose.IsUnresolvable(err) {
-			continue
+		// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
+		asked := in.Input
+		asked.Nodes = append([]compose.Node(nil), in.Nodes...)
+		for i := range asked.Nodes {
+			asked.Nodes[i].ReadsModules = true
 		}
-		if err != nil {
-			return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
-		}
-		if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
-			return nil, err
-		}
-	}
-	return out, nil
-}
-
-// **The declared-packages mirror passes on a change, not on a tick** (issue 1945 note 22). Its source composes every node in
-// full, about 45 s on the control node; on the 10 s tick it never rested and held the controller at 130-175 % CPU. Every
-// change that can move a node's packages is a change of its composition, which the controller writes into the bucket of the
-// compositions' digests, and of the assignments, which the verbs write into theirs: a watch of those two buckets wakes the
-// keeper. A pass reads the stores' Input once and composes only when the Input's digest differs from the one it last
-// mirrored, so a wake that changed nothing it composes from costs one reading. One pass at a time; the wakes during a pass
-// are one pass after it, at least packagesGap after the last began. A rare safety pass (packagesSafetyEvery) covers a
-// change no watched key says, and with an unchanged digest composes nothing either.
-const (
-	packagesSafetyEvery = 10 * time.Minute
-	packagesGap         = 30 * time.Second
-)
-
-// packagesTiming is when the declared-packages mirror passes: the safety pass's period and the least time between the
-// starts of two passes.
-type packagesTiming struct{ safety, gap time.Duration }
-
-// packagesKeeper keeps the declared-packages mirror: read is the Input, compose the mirror's values from it, sync writes
-// those that differ from the bucket, holds says this controller holds the lease. Each a seam a test stands in for.
-type packagesKeeper struct {
-	read    func(ctx context.Context) (*meshInput, error)
-	compose func(ctx context.Context, in *meshInput) (map[string][]byte, error)
-	sync    func(ctx context.Context, want map[string][]byte) error
-	holds   func() bool
-	wake    chan struct{}
-	// last is the digest of the Input last mirrored whole; empty after a failure or while the lease is not held, so the
-	// next pass composes.
-	last string
-}
-
-func newPackagesKeeper(read func(context.Context) (*meshInput, error),
-	compose func(context.Context, *meshInput) (map[string][]byte, error),
-	sync func(context.Context, map[string][]byte) error, holds func() bool) *packagesKeeper {
-	return &packagesKeeper{read: read, compose: compose, sync: sync, holds: holds, wake: make(chan struct{}, 1)}
-}
-
-// poke says something the mirror composes from may have changed; never blocks, and pokes before a pass are one.
-func (k *packagesKeeper) poke() {
-	select {
-	case k.wake <- struct{}{}:
-	default:
-	}
-}
-
-// pass brings the mirror to the Input, composing only when the Input differs from the one last mirrored.
-func (k *packagesKeeper) pass(ctx context.Context) error {
-	if !k.holds() {
-		k.last = ""
-		return nil
-	}
-	in, err := k.read(ctx)
-	if err != nil {
-		k.last = ""
-		return fmt.Errorf("the store cannot be read: %w", err)
-	}
-	digest := inputDigest(in)
-	if digest != "" && digest == k.last {
-		return nil
-	}
-	want, err := k.compose(ctx, in)
-	if err != nil {
-		k.last = ""
-		return err
-	}
-	if err := k.sync(ctx, want); err != nil {
-		k.last = ""
-		return err
-	}
-	k.last = digest
-	return nil
-}
-
-// run passes once at its start, then on each wake and each safety period, until ctx ends. said hears why a pass failed,
-// and an empty line when one succeeds again.
-func (k *packagesKeeper) run(ctx context.Context, at packagesTiming, said func(why string)) {
-	safety := time.NewTicker(at.safety)
-	defer safety.Stop()
-	failing := ""
-	var began time.Time
-	for {
-		// The wakes so far are this pass's: drained before it reads, so a change after is a pass of its own.
-		select {
-		case <-k.wake:
-		default:
-		}
-		began = time.Now()
-		why := ""
-		if err := k.pass(ctx); err != nil {
-			why = err.Error()
-		}
-		if why != failing {
-			said(why)
-			failing = why
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-k.wake:
-		case <-safety.C:
-		}
-		if rest := at.gap - time.Since(began); rest > 0 {
-			select {
-			case <-ctx.Done():
-				return
-			case <-time.After(rest):
+		out := map[string][]byte{}
+		for _, n := range in.sortedNodes() {
+			d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
+			if compose.IsUnresolvable(err) {
+				continue
 			}
-		}
-	}
-}
-
-// inputDigest is a digest of everything the composition reads (the Input, and why a node or the bus is refused): the same
-// for the same Input. Empty when it cannot be made, and then the pass composes.
-func inputDigest(in *meshInput) string {
-	refused := map[string]string{}
-	for node, err := range in.refused {
-		if err != nil {
-			refused[node] = err.Error()
-		}
-	}
-	bus := ""
-	if in.busRefused != nil {
-		bus = in.busRefused.Error()
-	}
-	body, err := json.Marshal(struct {
-		Input      compose.Input
-		Refused    map[string]string
-		BusRefused string
-	}{in.Input, refused, bus})
-	if err != nil {
-		return ""
-	}
-	sum := sha256.Sum256(body)
-	return hex.EncodeToString(sum[:])
-}
-
-// keepingDeclaredPackages keeps the declared-packages mirror for as long as ctx: woken by the watch of the compositions'
-// digests and of the assignments. It answers the keeper, whose poke a caller may also wake it with.
-func keepingDeclaredPackages(ctx context.Context, open *stores, api jetstream.JetStream, announce recordmirror.Announcer,
-	epoch func() uint64, holds func() bool) *packagesKeeper {
-	m := recordmirror.DeclaredPackages
-	var w *recordmirror.Writer
-	read := func(ctx context.Context) (*meshInput, error) {
-		reading, cancel := context.WithTimeout(ctx, 2*time.Minute)
-		defer cancel()
-		return readComposeInput(reading, open)
-	}
-	sync := func(ctx context.Context, want map[string][]byte) error {
-		writing, cancel := context.WithTimeout(ctx, 30*time.Second)
-		defer cancel()
-		if w == nil {
-			kv, err := api.KeyValue(writing, m.Bucket)
 			if err != nil {
-				return fmt.Errorf("the bucket %s cannot be read: %v", m.Bucket, err)
+				return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
 			}
-			w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
-		}
-		lost := w.LostEvents
-		r, err := w.Sync(writing, want)
-		if r.Written+r.Removed > 0 {
-			fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
-		}
-		if w.LostEvents > lost {
-			fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
-				m.Name, w.LostEvents-lost)
-		}
-		return err
-	}
-	k := newPackagesKeeper(read, declaredPackagesFrom, sync, holds)
-	for _, bucket := range []string{sdkdelivery.ControllerComposedBucket, broker.AssignmentsBucket} {
-		goServing(ctx, func() { wakeOnBucket(ctx, api, bucket, k.poke) })
-	}
-	goServing(ctx, func() {
-		k.run(ctx, packagesTiming{safety: packagesSafetyEvery, gap: packagesGap}, func(why string) {
-			if why != "" {
-				fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
-			} else {
-				fmt.Printf("the %s mirror is kept again\n", m.Name)
-			}
-		})
-	})
-	return k
-}
-
-// wakeOnBucket calls wake on every write or delete of a key in the bucket, for as long as ctx: only the keys' metadata is
-// watched. A bucket not there yet, or a watch that ends, is watched again a minute later; the safety pass covers the gap.
-func wakeOnBucket(ctx context.Context, api jetstream.JetStream, bucket string, wake func()) {
-	for {
-		if err := watchBucketOnce(ctx, api, bucket, wake); err != nil && ctx.Err() == nil {
-			fmt.Printf("the declared-packages mirror does not hear %s: %v\n", bucket, err)
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(time.Minute):
-		}
-	}
-}
-
-// watchBucketOnce watches the bucket until its watch ends or ctx does.
-func watchBucketOnce(ctx context.Context, api jetstream.JetStream, bucket string, wake func()) error {
-	kv, err := api.KeyValue(ctx, bucket)
-	if err != nil {
-		return err
-	}
-	watch, err := kv.WatchAll(ctx, jetstream.UpdatesOnly(), jetstream.MetaOnly())
-	if err != nil {
-		return err
-	}
-	defer func() { _ = watch.Stop() }()
-	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		case e, ok := <-watch.Updates():
-			if !ok {
-				return fmt.Errorf("the watch ended")
-			}
-			if e != nil {
-				wake()
+			if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
+				return nil, err
 			}
 		}
+		return out, nil
 	}
 }

cmd/mesh-controller/record_mirrors_test.go

@@ -1,191 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"git.novox.be/novox/mesh-sdk/go/catalogue"
-	"git.novox.be/novox/mesh-sdk/go/compose"
-)
-
-// The declared-packages mirror composes only when what it composes from changed (issue 1945 note 22: it composed every node
-// in full every 10 s, a pass of about 45 s, so the controller on the control node never rested at 130-175 % CPU).
-
-// scaled is a cadence of the mirror with a second taken as a millisecond, so a test's minute is 60 ms.
-func scaled() packagesTiming {
-	return packagesTiming{safety: packagesSafetyEvery / 1000, gap: packagesGap / 1000}
-}
-
-const scaledMinute = 60 * time.Millisecond
-
-// fakeMesh is the keeper's input as a test changes it: each node's assigned modules, and the compositions counted by node.
-type fakeMesh struct {
-	mu       sync.Mutex
-	assigned map[string][]string
-	composed atomic.Int64 // node compositions
-	synced   atomic.Int64
-	lastSync map[string][]byte
-	// block, when set, holds the next composition until it is closed; entered is closed when that composition began.
-	block   chan struct{}
-	entered chan struct{}
-}
-
-func newFakeMesh(nodes int) *fakeMesh {
-	f := &fakeMesh{assigned: map[string][]string{}}
-	for i := 0; i < nodes; i++ {
-		f.assigned[fmt.Sprintf("node%d", i)] = []string{"base"}
-	}
-	return f
-}
-
-func (f *fakeMesh) assign(node, module string) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.assigned[node] = append(append([]string(nil), f.assigned[node]...), module)
-}
-
-func (f *fakeMesh) keeper() *packagesKeeper {
-	read := func(context.Context) (*meshInput, error) {
-		f.mu.Lock()
-		defer f.mu.Unlock()
-		in := compose.Input{Catalogue: map[string]catalogue.Manifest{"base": {Module: "base", Version: "1"}}}
-		for _, n := range nodesOfFake(f.assigned) {
-			in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: append([]string(nil), f.assigned[n]...)})
-		}
-		return &meshInput{Input: in}, nil
-	}
-	composeAll := func(_ context.Context, in *meshInput) (map[string][]byte, error) {
-		f.mu.Lock()
-		block, entered := f.block, f.entered
-		f.block = nil
-		f.mu.Unlock()
-		if block != nil {
-			close(entered)
-			<-block
-		}
-		out := map[string][]byte{}
-		for _, n := range in.Nodes {
-			f.composed.Add(1)
-			out[n.Name] = []byte(fmt.Sprint(n.Assigned))
-		}
-		return out, nil
-	}
-	syncAll := func(_ context.Context, want map[string][]byte) error {
-		f.mu.Lock()
-		f.lastSync = want
-		f.mu.Unlock()
-		f.synced.Add(1)
-		return nil
-	}
-	return newPackagesKeeper(read, composeAll, syncAll, func() bool { return true })
-}
-
-func nodesOfFake(m map[string][]string) []string {
-	out := make([]string, 0, len(m))
-	for k := range m {
-		out = append(out, k)
-	}
-	for i := range out {
-		for j := i + 1; j < len(out); j++ {
-			if out[j] < out[i] {
-				out[i], out[j] = out[j], out[i]
-			}
-		}
-	}
-	return out
-}
-
-func runKeeper(t *testing.T, k *packagesKeeper) context.CancelFunc {
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		k.run(ctx, scaled(), func(string) {})
-	}()
-	t.Cleanup(func() {
-		cancel()
-		<-done
-	})
-	return cancel
-}
-
-func eventually(t *testing.T, what string, cond func() bool) {
-	t.Helper()
-	deadline := time.Now().Add(time.Second)
-	for !cond() {
-		if time.Now().After(deadline) {
-			t.Fatalf("not within a second: %s", what)
-		}
-		time.Sleep(time.Millisecond)
-	}
-}
-
-// 1945/AC-mirror-1: given a mesh whose input does not change, when a minute passes after the mirror's first pass, then it
-// composes no node (before: a pass every 10 s, each composing every node). The count per minute is said.
-func TestTheDeclaredPackagesMirrorComposesNothingWithoutAChange(t *testing.T) {
-	const nodes = 5
-	f := newFakeMesh(nodes)
-	runKeeper(t, f.keeper())
-	eventually(t, "the first pass", func() bool { return f.synced.Load() >= 1 })
-	start := f.composed.Load()
-	time.Sleep(scaledMinute)
-	perMinute := f.composed.Load() - start
-	t.Logf("node compositions in a minute of an unchanged mesh of %d nodes: %d", nodes, perMinute)
-	if perMinute != 0 {
-		t.Errorf("an unchanged mesh of %d nodes was composed %d times in a minute; want 0", nodes, perMinute)
-	}
-}
-
-// 1945/AC-mirror-2: given a pass that is running, when fifty changes arrive during it, then exactly one more pass follows,
-// never two at once and never one per change.
-func TestTheDeclaredPackagesMirrorPassesOncePerBurst(t *testing.T) {
-	f := newFakeMesh(1)
-	release := make(chan struct{})
-	f.block, f.entered = release, make(chan struct{})
-	k := f.keeper()
-	runKeeper(t, k)
-	<-f.entered
-	for i := 0; i < 50; i++ {
-		f.assign("node0", fmt.Sprintf("m%d", i))
-		k.poke()
-	}
-	close(release)
-	eventually(t, "the pass after the burst", func() bool { return f.synced.Load() >= 2 })
-	time.Sleep(2*scaled().gap + 20*time.Millisecond)
-	if got := f.synced.Load(); got != 2 {
-		t.Errorf("a burst of 50 changes during a pass gave %d passes in all; want 2 (the running one and one after)", got)
-	}
-}
-
-// 1945/AC-mirror-3: given the mirror at rest, when an input changes and its event arrives, then the change is mirrored.
-func TestTheDeclaredPackagesMirrorMirrorsAChange(t *testing.T) {
-	f := newFakeMesh(2)
-	k := f.keeper()
-	runKeeper(t, k)
-	eventually(t, "the first pass", func() bool { return f.synced.Load() >= 1 })
-	f.assign("node1", "viewer")
-	k.poke()
-	eventually(t, "the change mirrored", func() bool {
-		f.mu.Lock()
-		defer f.mu.Unlock()
-		return string(f.lastSync["node1"]) == "[base viewer]"
-	})
-}
-
-// 1945/AC-mirror-4: the digest of the input is the same for the same input and differs when an assignment differs.
-func TestTheInputDigestFollowsTheInput(t *testing.T) {
-	of := func(assigned ...string) *meshInput {
-		return &meshInput{Input: compose.Input{Nodes: []compose.Node{{Name: "anchor", Assigned: assigned}}}}
-	}
-	a, b, c := inputDigest(of("base")), inputDigest(of("base")), inputDigest(of("base", "viewer"))
-	if a == "" || a != b {
-		t.Errorf("the same input gave the digests %q and %q", a, b)
-	}
-	if a == c {
-		t.Errorf("an added assignment left the digest at %q", a)
-	}
-}

cmd/mesh-controller/serving_group.go

@@ -1,59 +0,0 @@
-package main
-
-import (
-	"context"
-	"sync"
-)
-
-// **Every goroutine the serving controller starts ends before serve returns** (issue 1945). A loop or a watch started
-// beside the serving runs until serve's context ends, and serve's stop waits for it before it closes the stores, the
-// bus and the condition keeper those loops read: a loop that outlived its serve met the next serve's globals, its
-// closed stores and its bus, and in the test process that is the next test (a race on conditionsFrom, a package that
-// passed every test and still exited failing). Where it is checked: the tests that run serve whole, under -race.
-
-// servingGroup counts the goroutines one serve started. Once its stop began it starts nothing more: what would start is
-// work for a controller that is stopping.
-type servingGroup struct {
-	mu      sync.Mutex
-	stopped bool
-	wg      sync.WaitGroup
-}
-
-type servingGroupKey struct{}
-
-// withServingGroup is ctx carrying a new group, which goServing under it counts.
-func withServingGroup(ctx context.Context) (context.Context, *servingGroup) {
-	g := &servingGroup{}
-	return context.WithValue(ctx, servingGroupKey{}, g), g
-}
-
-// goServing runs work beside the caller, counted in the group ctx carries, so serve's stop waits for it. Under a group
-// whose stop began it runs nothing; under a context with no group (a part run alone, by a test or a verb) it is a plain
-// goroutine, as before.
-func goServing(ctx context.Context, work func()) {
-	g, _ := ctx.Value(servingGroupKey{}).(*servingGroup)
-	if g == nil {
-		go work()
-		return
-	}
-	g.mu.Lock()
-	if g.stopped {
-		g.mu.Unlock()
-		return
-	}
-	g.wg.Add(1)
-	g.mu.Unlock()
-	go func() {
-		defer g.wg.Done()
-		work()
-	}()
-}
-
-// stop starts nothing more and waits for what was started; the caller ends their context first. A second stop waits
-// for nothing.
-func (g *servingGroup) stop() {
-	g.mu.Lock()
-	g.stopped = true
-	g.mu.Unlock()
-	g.wg.Wait()
-}

cmd/mesh-controller/states_announced.go

@@ -73,10 +73,10 @@ func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores)
 	if err != nil {
 		return fmt.Errorf("following the states announced: %w", err)
 	}
-	goServing(ctx, func() {
+	go func() {
 		<-ctx.Done()
 		consuming.Stop()
-	})
+	}()
 	return nil
 }
 

cmd/mesh-controller/status_summary.go

@@ -210,7 +210,7 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
 	// And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3,
 	// issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path.
 	if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" {
-		goServing(ctx, func() { moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node) })
+		go moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node)
 	}
 	return news, err
 }

cmd/mesh-controller/watchdogs.go

@@ -543,9 +543,10 @@ func later(a, b time.Time) time.Time {
 	return b
 }
 
-// openServingConditions opens the condition store for the serving controller and makes it conditionsFrom. Nil when the store
-// could not be opened, which is said: status then says the conditions cannot be read.
-func openServingConditions(ctx context.Context, open *stores, bus link.OverNATS) *conditions.Keeper {
+// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
+// self-check, for the serving controller; the returned function stops them. Nil when the store could
+// not be opened, which is said.
+func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
 	keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory})
 	if err != nil {
 		fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
@@ -553,22 +554,6 @@ func openServingConditions(ctx context.Context, open *stores, bus link.OverNATS)
 		return nil
 	}
 	conditionsFrom = keeper
-	return keeper
-}
-
-// closeConditions flushes what the keeper holds and closes it, once nothing that raises a condition runs.
-func closeConditions(keeper *conditions.Keeper) {
-	flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	keeper.Close(flushing)
-}
-
-// watchTheMesh starts the watchdogs, the bus's advisories and the self-check over the condition store's keeper, for the
-// serving controller; the returned function stops them and waits for them to end. Nil when there is no keeper.
-func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS, keeper *conditions.Keeper) func() {
-	if keeper == nil {
-		return nil
-	}
 	logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
 	stopHearing, err := server.HearAdvisories(logf)
 	if err != nil {
@@ -581,26 +566,26 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
 	d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
 	w.doctor = d
 	doctorFrom = d
-	// Its own group as well as serve's: its stop waits for what it started.
 	watching, stop := context.WithCancel(ctx)
-	watching, running := withServingGroup(watching)
-	goServing(watching, func() { w.keep(watching) })
-	goServing(watching, func() { d.keep(watching) })
+	go w.keep(watching)
+	go d.keep(watching)
 	// And the healers (novox/hq to-be 45 §7): what is open that a registered healer answers, repaired
 	// under the lease and the brake, every act said.
 	healers := newHealing(open, keeper, bus, server.JetStream())
-	goServing(watching, func() { healers.keep(watching) })
+	go healers.keep(watching)
 	// And the asker (novox/hq ADR 0259): what needs the operator and names its answers is asked of them,
 	// and the answer chosen is performed on its warrant.
 	startAsking(watching, open, server, bus.Conn, keeper)
-	goServing(watching, func() { forgettingOldHeals(watching, open.inventory) })
+	go forgettingOldHeals(watching, open.inventory)
 	fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
 		"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
 		broker.ConditionsBucket, conditions.Seat)
 	return func() {
 		stop()
-		running.stop()
 		stopHearing()
+		flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		keeper.Close(flushing)
 	}
 }