From 8c54139905b08cdf42082b23b1c0acdb5bf8a17c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 20:23:05 +0200 Subject: [PATCH] Trace a rotation's credential writes and compositions, record the bindings a composition carries, drop SchemaReachOf, and re-pin mesh-sdk to e45bdc0891bb (issue 1945) --- cmd/mesh-controller/composed_live.go | 12 +++++ .../composed_served_integration_test.go | 27 +++++++++- cmd/mesh-controller/composed_trace.go | 51 +++++++++++++++++++ go.mod | 2 +- go.sum | 2 + internal/inventory/schema.go | 13 ----- internal/inventory/secrets.go | 30 +++++++++++ testdata/beside/CAPTURED | 2 +- 8 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 cmd/mesh-controller/composed_trace.go diff --git a/cmd/mesh-controller/composed_live.go b/cmd/mesh-controller/composed_live.go index 45724876..39b9924c 100644 --- a/cmd/mesh-controller/composed_live.go +++ b/cmd/mesh-controller/composed_live.go @@ -70,6 +70,9 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri continue } at.sign, at.policy = sign, policy + if composeTrace != nil { + traceInput(ctx, open, node) + } declared, err := composeSend(ctx, open, node) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", node, err)) @@ -93,11 +96,20 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n", node, strings.Join(moved, ", ")) } + if composeTrace != nil { + traceComposed(ctx, at, node, comps) + } wrote, err := writeNodeCompositions(ctx, at, comps) if len(wrote) > 0 { out[node] = wrote fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", ")) } + // And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232). + if len(declared.Bindings) > 0 { + if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil { + errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err)) + } + } if err != nil { errs = append(errs, fmt.Errorf("%s: %w", node, err)) } diff --git a/cmd/mesh-controller/composed_served_integration_test.go b/cmd/mesh-controller/composed_served_integration_test.go index 70528468..94d1aaed 100644 --- a/cmd/mesh-controller/composed_served_integration_test.go +++ b/cmd/mesh-controller/composed_served_integration_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -434,6 +435,28 @@ func TestServedADoneKeyIsComposedAndGranted(t *testing.T) { // composition keeps its own, and nothing is sent. func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) { m := aServedMesh(t) + // Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the + // callers: one failure shows the cause (issue 1945). + var traceMu sync.Mutex + var trace []string + note := func(line string) { + traceMu.Lock() + defer traceMu.Unlock() + trace = append(trace, time.Now().Format("15:04:05.000")+" "+line) + } + inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) } + composeTrace = func(line string) { note("compose " + line) } + t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil }) + traced := func() string { + traceMu.Lock() + defer traceMu.Unlock() + return strings.Join(trace, "\n ") + } + defer func() { + if t.Failed() { + t.Logf("TRACE\n %s", traced()) + } + }() before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")} holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "") var waited uint64 @@ -485,8 +508,8 @@ func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) { } } diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr) - t.Fatalf("the board's composition did not move within a minute after the rotation; LINES %s; holders before %+v, after %+v; DIAG %s; %s", - strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop")) + t.Fatalf("the board's composition did not move within a minute after the rotation; TRACE\n %s\nLINES %s; holders before %+v, after %+v; DIAG %s; %s", + traced(), strings.Join(strings.Fields(said), " "), holders, after, diag, m.whatIsThere(t, "laptop")) } moved := map[string]bool{} for node, was := range before { diff --git a/cmd/mesh-controller/composed_trace.go b/cmd/mesh-controller/composed_trace.go new file mode 100644 index 00000000..9ab7782a --- /dev/null +++ b/cmd/mesh-controller/composed_trace.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "sort" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// composeTrace, when set, is told what composing a node saw and decided: the Input's credential state before it composes, and for +// each module the digest it composed against the digest already written. A test sets it to read why a composition did or did not +// move; the controller never does. +var composeTrace func(string) + +// traceInput says what the node's composition will read of the pair credentials: those no row holds (would-make) and those held. +func traceInput(ctx context.Context, open *stores, node string) { + in, err := inputFor(ctx, open) + if err != nil { + composeTrace(fmt.Sprintf("%s: the Input cannot be read: %v", node, err)) + return + } + composeTrace(fmt.Sprintf("%s: before composing, would make %+v; credentials held %+v", node, in.wouldMake, in.Credentials)) +} + +// traceComposed says, per composition of a node, the digest composed now against the one written. +func traceComposed(ctx context.Context, at composedPlace, node string, comps map[string]sdkdelivery.Composed) { + keys := make([]string, 0, len(comps)) + for k := range comps { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + now, err := sdkdelivery.ComposedDigestOf(comps[key]) + if err != nil { + composeTrace(fmt.Sprintf("%s: %s: no digest: %v", node, key, err)) + continue + } + written := "none" + if e, err := at.digests.Get(ctx, key); err == nil { + if v, err := sdkdelivery.DecodeComposedDigest(e.Value()); err == nil { + written = v.Digest + } + } + same := "CHANGED" + if written == now { + same = "same" + } + composeTrace(fmt.Sprintf("%s: %s: composed %.19s, written %.19s: %s", node, key, now, written, same)) + } +} diff --git a/go.mod b/go.mod index 2b438ce7..e57269b0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 9c11bf62..b4be1d4c 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024150354-5da87a0dda89 h1:9FENYqhy git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024150354-5da87a0dda89/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8 h1:aFfcEXmd24GpY5na6Xaic0A9xmAAa/jbrrX5wErPDdU= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024161532-a523a8cbb0c8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb h1:uwU95N4tmDypH4B4ZcUOtg18PChDR3xODaZxdG0Ssl4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024182048-e45bdc0891bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/inventory/schema.go b/internal/inventory/schema.go index fa971828..40aaee29 100644 --- a/internal/inventory/schema.go +++ b/internal/inventory/schema.go @@ -3,8 +3,6 @@ package inventory import ( "context" "errors" - - "github.com/jackc/pgx/v5" ) // What a controller build knows of the store's schema (novox/hq issue 352): the highest migration it @@ -23,17 +21,6 @@ func (i *Inventory) RecordSchemaReach(ctx context.Context, build string, reach i return err } -// SchemaReachOf is the highest migration a build carries, as it recorded when it served; false for a -// build that never did. -func (i *Inventory) SchemaReachOf(ctx context.Context, build string) (int, bool, error) { - var reach int - err := i.store.Pool().QueryRow(ctx, `select reach from controller_schema where build = $1`, build).Scan(&reach) - if errors.Is(err, pgx.ErrNoRows) { - return 0, false, nil - } - return reach, err == nil, err -} - // SchemaApplied is the highest migration the store has applied. func (i *Inventory) SchemaApplied(ctx context.Context) (int, error) { var n *int diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 7af5ea93..1df09e40 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "runtime" "slices" "sort" "strings" @@ -21,6 +22,31 @@ import ( // The table holds nothing usable — see the migration and internal/secrets for why that is the // design rather than an inconvenience. +// SecretTrace, when set, is told of every write of a pair credential's row — a made or accepted one, a rotated one discarded — +// with the callers that led to it. A test sets it to see who wrote what; the controller never does. +var SecretTrace func(op, what string) + +func traceSecret(op, name, consumer, module, provider string) { + if SecretTrace == nil { + return + } + pcs := make([]uintptr, 8) + n := runtime.Callers(3, pcs) + frames := runtime.CallersFrames(pcs[:n]) + var callers []string + for { + f, more := frames.Next() + if i := strings.LastIndex(f.Function, "/"); i >= 0 { + f.Function = f.Function[i+1:] + } + callers = append(callers, f.Function) + if !more || len(callers) >= 5 { + break + } + } + SecretTrace(op, fmt.Sprintf("%s for %s/%s from %s, by %s", name, consumer, module, provider, strings.Join(callers, " < "))) +} + // Secret is one provision's credential, sealed to each end. type Secret struct { Name string @@ -146,6 +172,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul if err != nil { return Secret{}, err } + traceSecret("made", name, consumer, consumerModule, provider) return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule, Local: local, Provider: provider, ForConsumer: made.ForConsumer, ForProvider: made.ForProvider, @@ -276,6 +303,9 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerMo `delete from secret where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, name, consumerNode.ID, consumerModule, providerNode.ID, local) + if err == nil { + traceSecret("discarded", name, consumer, consumerModule, provider) + } return err } diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index f60d43be..5efcab5d 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog 2246ccc492e59579a140752e6f574b2121d296af modules/*/module.json, modules/*/files, modules/*/templates, predefined.json mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk a523a8cbb0c8b1a83fcccd1790224e1d94380785 conformance/events/module-event.json +mesh-sdk e45bdc0891bb7c90be639c4903d51bb5dc61a2ac conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: