Compose each node once per pass in one composer for the done keys' take-ins, one composition of a node at a time, and the start's look side by side (issue 1945)

cmd/mesh-controller/composed_keys.go

@@ -251,21 +251,13 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
 			}
 			return holders[catalogue.BrokerSeat].Node, nil
 		},
+		// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
+		// grants once (issue 1945, composer.go).
 		compose: func(ctx context.Context, nodes []string) error {
-			_, err := composeModules(ctx, open, nodes)
-			return err
+			return composerFor(open).compose(ctx, nodes, false)
 		},
 		grants: func(ctx context.Context) error {
-			bus, err := aBus()
-			if err != nil {
-				return err
-			}
-			defer bus.Close()
-			js, err := jetstream.New(bus.Conn())
-			if err != nil {
-				return err
-			}
-			return grantsIssued(ctx, inv, js, "done keys: ")
+			return composerFor(open).compose(ctx, nil, true)
 		},
 	}
 }
@@ -307,14 +299,27 @@ func doneKeysAtStart(ctx context.Context, modules func(context.Context) ([]strin
 		return
 	}
 	sort.Strings(names)
+	// Side by side, a few at a time: their compositions meet in the composer's passes, each node composed once per pass
+	// (issue 1945), so the start's look is not one module's composition after another.
+	sem := make(chan struct{}, startTakeIns)
+	var wg sync.WaitGroup
 	for _, m := range names {
 		if ctx.Err() != nil {
-			return
+			break
 		}
-		take(m)
+		sem <- struct{}{}
+		wg.Add(1)
+		go func() {
+			defer func() { <-sem; wg.Done() }()
+			take(m)
+		}()
 	}
+	wg.Wait()
 }
 
+// startTakeIns is how many take-ins the start's look runs at once.
+const startTakeIns = 16
+
 // takeDoneKeyAtStart is one take-in of the start's look, under the module's lock; one that fails is handed to takeDoneKey,
 // which tries again every doneKeyRetry, loudly, so a failing module never holds up the look at the others.
 func takeDoneKeyAtStart(ctx context.Context, open *stores, module string) {
@@ -342,3 +347,34 @@ func catalogedModules(open *stores) func(context.Context) ([]string, error) {
 		return out, nil
 	}
 }
+
+// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
+// runs several meshes), each started at its first use and run for as long as the process serves.
+var doneKeysComposers sync.Map
+
+// composerFor is the composer of the stores, its passes composing on them and writing the grants on the bus.
+func composerFor(open *stores) *composer {
+	if c, ok := doneKeysComposers.Load(open); ok {
+		return c.(*composer)
+	}
+	c := newComposer(func(ctx context.Context, nodes []string) error {
+		_, err := composeModules(ctx, open, nodes)
+		return err
+	}, func(ctx context.Context) error {
+		bus, err := aBus()
+		if err != nil {
+			return err
+		}
+		defer bus.Close()
+		js, err := jetstream.New(bus.Conn())
+		if err != nil {
+			return err
+		}
+		return grantsIssued(ctx, open.inventory, js, "done keys: ")
+	})
+	if was, loaded := doneKeysComposers.LoadOrStore(open, c); loaded {
+		return was.(*composer)
+	}
+	go c.run(context.Background())
+	return c
+}