Trace a rotation's credential writes and compositions, record the bindings a composition carries, drop SchemaReachOf, and re-pin mesh-sdk to e45bdc0891bb (issue 1945)

cmd/mesh-controller/composed_served_integration_test.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -434,6 +435,28 @@ func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
 // composition keeps its own, and nothing is sent.
 func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
 	m := aServedMesh(t)
+	// Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the
+	// callers: one failure shows the cause (issue 1945).
+	var traceMu sync.Mutex
+	var trace []string
+	note := func(line string) {
+		traceMu.Lock()
+		defer traceMu.Unlock()
+		trace = append(trace, time.Now().Format("15:04:05.000")+" "+line)
+	}
+	inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) }
+	composeTrace = func(line string) { note("compose " + line) }
+	t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil })
+	traced := func() string {
+		traceMu.Lock()
+		defer traceMu.Unlock()
+		return strings.Join(trace, "\n    ")
+	}
+	defer func() {
+		if t.Failed() {
+			t.Logf("TRACE\n    %s", traced())
+		}
+	}()
 	before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
 	holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
 	var waited uint64
@@ -485,8 +508,8 @@ func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
 			}
 		}
 		diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr)
-		t.Fatalf("the board's composition did not move within a minute after the rotation; LINES %s; holders before %+v, after %+v; DIAG %s; %s",
-			strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop"))
+		t.Fatalf("the board's composition did not move within a minute after the rotation; TRACE\n    %s\nLINES %s; holders before %+v, after %+v; DIAG %s; %s",
+			traced(), strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop"))
 	}
 	moved := map[string]bool{}
 	for node, was := range before {