Trace a rotation's credential writes and compositions, record the bindings a composition carries, drop SchemaReachOf, and re-pin mesh-sdk to e45bdc0891bb (issue 1945)

cmd/mesh-controller/composed_live.go

@@ -70,6 +70,9 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri
 			continue
 		}
 		at.sign, at.policy = sign, policy
+		if composeTrace != nil {
+			traceInput(ctx, open, node)
+		}
 		declared, err := composeSend(ctx, open, node)
 		if err != nil {
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
@@ -93,11 +96,20 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri
 			fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
 				node, strings.Join(moved, ", "))
 		}
+		if composeTrace != nil {
+			traceComposed(ctx, at, node, comps)
+		}
 		wrote, err := writeNodeCompositions(ctx, at, comps)
 		if len(wrote) > 0 {
 			out[node] = wrote
 			fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", "))
 		}
+		// And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232).
+		if len(declared.Bindings) > 0 {
+			if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil {
+				errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err))
+			}
+		}
 		if err != nil {
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
 		}

cmd/mesh-controller/composed_served_integration_test.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -434,6 +435,28 @@ func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
 // composition keeps its own, and nothing is sent.
 func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
 	m := aServedMesh(t)
+	// Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the
+	// callers: one failure shows the cause (issue 1945).
+	var traceMu sync.Mutex
+	var trace []string
+	note := func(line string) {
+		traceMu.Lock()
+		defer traceMu.Unlock()
+		trace = append(trace, time.Now().Format("15:04:05.000")+" "+line)
+	}
+	inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) }
+	composeTrace = func(line string) { note("compose " + line) }
+	t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil })
+	traced := func() string {
+		traceMu.Lock()
+		defer traceMu.Unlock()
+		return strings.Join(trace, "\n    ")
+	}
+	defer func() {
+		if t.Failed() {
+			t.Logf("TRACE\n    %s", traced())
+		}
+	}()
 	before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
 	holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
 	var waited uint64
@@ -485,8 +508,8 @@ func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
 			}
 		}
 		diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr)
-		t.Fatalf("the board's composition did not move within a minute after the rotation; LINES %s; holders before %+v, after %+v; DIAG %s; %s",
+		t.Fatalf("the board's composition did not move within a minute after the rotation; TRACE\n    %s\nLINES %s; holders before %+v, after %+v; DIAG %s; %s",
-			strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop"))
+			traced(), strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop"))
 	}
 	moved := map[string]bool{}
 	for node, was := range before {

cmd/mesh-controller/composed_trace.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// composeTrace, when set, is told what composing a node saw and decided: the Input's credential state before it composes, and for
+// each module the digest it composed against the digest already written. A test sets it to read why a composition did or did not
+// move; the controller never does.
+var composeTrace func(string)
+
+// traceInput says what the node's composition will read of the pair credentials: those no row holds (would-make) and those held.
+func traceInput(ctx context.Context, open *stores, node string) {
+	in, err := inputFor(ctx, open)
+	if err != nil {
+		composeTrace(fmt.Sprintf("%s: the Input cannot be read: %v", node, err))
+		return
+	}
+	composeTrace(fmt.Sprintf("%s: before composing, would make %+v; credentials held %+v", node, in.wouldMake, in.Credentials))
+}
+
+// traceComposed says, per composition of a node, the digest composed now against the one written.
+func traceComposed(ctx context.Context, at composedPlace, node string, comps map[string]sdkdelivery.Composed) {
+	keys := make([]string, 0, len(comps))
+	for k := range comps {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		now, err := sdkdelivery.ComposedDigestOf(comps[key])
+		if err != nil {
+			composeTrace(fmt.Sprintf("%s: %s: no digest: %v", node, key, err))
+			continue
+		}
+		written := "none"
+		if e, err := at.digests.Get(ctx, key); err == nil {
+			if v, err := sdkdelivery.DecodeComposedDigest(e.Value()); err == nil {
+				written = v.Digest
+			}
+		}
+		same := "CHANGED"
+		if written == now {
+			same = "same"
+		}
+		composeTrace(fmt.Sprintf("%s: %s: composed %.19s, written %.19s: %s", node, key, now, written, same))
+	}
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

go.sum

@@ -14,6 +14,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024150354-5da87a0dda89 h1:9FENYqhy
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024150354-5da87a0dda89/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8 h1:aFfcEXmd24GpY5na6Xaic0A9xmAAa/jbrrX5wErPDdU=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb h1:uwU95N4tmDypH4B4ZcUOtg18PChDR3xODaZxdG0Ssl4=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/inventory/schema.go

@@ -3,8 +3,6 @@ package inventory
 import (
 	"context"
 	"errors"
-
-	"github.com/jackc/pgx/v5"
 )
 
 // What a controller build knows of the store's schema (novox/hq issue 352): the highest migration it
@@ -23,17 +21,6 @@ func (i *Inventory) RecordSchemaReach(ctx context.Context, build string, reach i
 	return err
 }
 
-// SchemaReachOf is the highest migration a build carries, as it recorded when it served; false for a
-// build that never did.
-func (i *Inventory) SchemaReachOf(ctx context.Context, build string) (int, bool, error) {
-	var reach int
-	err := i.store.Pool().QueryRow(ctx, `select reach from controller_schema where build = $1`, build).Scan(&reach)
-	if errors.Is(err, pgx.ErrNoRows) {
-		return 0, false, nil
-	}
-	return reach, err == nil, err
-}
-
 // SchemaApplied is the highest migration the store has applied.
 func (i *Inventory) SchemaApplied(ctx context.Context) (int, error) {
 	var n *int

internal/inventory/secrets.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"runtime"
 	"slices"
 	"sort"
 	"strings"
@@ -21,6 +22,31 @@ import (
 // The table holds nothing usable — see the migration and internal/secrets for why that is the
 // design rather than an inconvenience.
 
+// SecretTrace, when set, is told of every write of a pair credential's row — a made or accepted one, a rotated one discarded —
+// with the callers that led to it. A test sets it to see who wrote what; the controller never does.
+var SecretTrace func(op, what string)
+
+func traceSecret(op, name, consumer, module, provider string) {
+	if SecretTrace == nil {
+		return
+	}
+	pcs := make([]uintptr, 8)
+	n := runtime.Callers(3, pcs)
+	frames := runtime.CallersFrames(pcs[:n])
+	var callers []string
+	for {
+		f, more := frames.Next()
+		if i := strings.LastIndex(f.Function, "/"); i >= 0 {
+			f.Function = f.Function[i+1:]
+		}
+		callers = append(callers, f.Function)
+		if !more || len(callers) >= 5 {
+			break
+		}
+	}
+	SecretTrace(op, fmt.Sprintf("%s for %s/%s from %s, by %s", name, consumer, module, provider, strings.Join(callers, " < ")))
+}
+
 // Secret is one provision's credential, sealed to each end.
 type Secret struct {
 	Name     string
@@ -146,6 +172,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul
 	if err != nil {
 		return Secret{}, err
 	}
+	traceSecret("made", name, consumer, consumerModule, provider)
 	return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule, Local: local,
 		Provider:    provider,
 		ForConsumer: made.ForConsumer, ForProvider: made.ForProvider,
@@ -276,6 +303,9 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerMo
 		`delete from secret where name = $1 and consumer = $2 and consumer_module = $3
 		   and provider = $4 and local = $5`,
 		name, consumerNode.ID, consumerModule, providerNode.ID, local)
+	if err == nil {
+		traceSecret("discarded", name, consumer, consumerModule, provider)
+	}
 	return err
 }
 

testdata/beside/CAPTURED

@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
 
 mesh-catalog  2246ccc492e59579a140752e6f574b2121d296af  modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
 mesh-host     9d9e3a46c4f5943cfffb413d962ce48b75e2fd39  examples/foundation-first-node.lock
-mesh-sdk      a523a8cbb0c8b1a83fcccd1790224e1d94380785  conformance/events/module-event.json
+mesh-sdk      e45bdc0891bb7c90be639c4903d51bb5dc61a2ac  conformance/events/module-event.json
 
 To move them, from this repository's root, with the two repositories checked out beside it: