From b01cf6323726e2029d2e19d581cd9f9f7223bbe5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:05:14 +0100 Subject: [PATCH 01/35] Revert "Merge pull request 'Revert L3: its node-engine removed every held module on novox, the bus server among them (issue 774)' (#109) from revert/774-l3 into main" This reverts commit c1b4533201c85a5a2a71f47e7d9496d2a2f8f313, reversing changes made to d8274204fbedb6d9062007632f5b75f239774068. --- modules/assignments-readers | 7 + .../integration_test.go | 4 +- .../cmd/mesh-delivery-assigner/main.go | 63 +- .../cmd/mesh-delivery-assigner/main_test.go | 31 +- .../cmd/mesh-delivery-assigner/pass.go | 8 +- .../cmd/mesh-delivery-assigner/pass_test.go | 22 + .../cmd/mesh-delivery-assigner/puts.go | 5 +- .../cmd/mesh-delivery-assigner/seat.go | 6 +- .../cmd/mesh-delivery-assigner/seat_tools.go | 16 +- .../mesh-delivery-assigner/seat_tools_test.go | 10 + .../cmd/mesh-delivery-assigner/world.go | 37 +- .../cmd/mesh-delivery-assigner/world_test.go | 38 ++ .../internal/assign/placement.go | 17 +- .../internal/assign/seat.go | 10 +- .../internal/assign/seat_test.go | 6 +- .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 2 + .../cmd/mesh-delivery-builder/main.go | 10 +- .../cmd/mesh-delivery-builder/refused_test.go | 73 +++ .../cmd/mesh-delivery-builder/standing.go | 211 ------- .../mesh-delivery-builder/standing_test.go | 228 ------- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 +- .../mesh-delivery-builder/manifest_test.go | 4 +- modules/mesh-delivery-builder/module.json | 2 +- .../cmd/mesh-delivery-checker-tools/main.go | 17 +- .../mesh-delivery-checker-tools/main_test.go | 18 + .../cmd/mesh-delivery-checker/main.go | 10 +- .../cmd/mesh-delivery-checker/refused_test.go | 62 ++ .../cmd/mesh-delivery-checker/standing.go | 211 ------- .../mesh-delivery-checker/standing_test.go | 218 ------- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 +- .../mesh-delivery-checker/manifest_test.go | 16 +- modules/mesh-delivery-checker/module.json | 13 +- .../mesh-delivery-indexer/integration_test.go | 64 +- .../cmd/mesh-delivery-indexer/main.go | 15 +- .../cmd/mesh-delivery-indexer/standing.go | 211 ------- .../mesh-delivery-indexer/standing_test.go | 165 ------ modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 +- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 2 +- .../cmd/mesh-delivery-cli/main_test.go | 6 +- .../cmd/mesh-delivery-view-tools/main.go | 14 +- .../cmd/mesh-delivery-view-tools/main_test.go | 39 ++ .../mesh-delivery-view/integration_test.go | 23 +- .../cmd/mesh-delivery-view/main.go | 4 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/read.go | 88 +++ .../mesh-delivery-view/internal/view/view.go | 83 +-- .../internal/view/view_test.go | 95 ++- .../internal/view/wants_test.go | 14 +- modules/mesh-delivery-view/manifest_test.go | 4 +- modules/mesh-delivery-view/module.json | 6 +- modules/mesh-grants/cmd/mesh-grants/bus.go | 4 + modules/mesh-grants/cmd/mesh-grants/create.go | 35 ++ modules/mesh-grants/cmd/mesh-grants/main.go | 21 +- modules/mesh-grants/cmd/mesh-grants/tools.go | 42 ++ modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 +- modules/mesh-grants/grants_test.go | 557 +++++++++++++++++- .../mesh-grants/internal/grants/accounts.go | 217 +++++++ .../mesh-grants/internal/grants/consumers.go | 242 ++++++++ modules/mesh-grants/internal/grants/follow.go | 22 +- modules/mesh-grants/internal/grants/grants.go | 268 +++++++++ modules/mesh-grants/internal/grants/inputs.go | 79 ++- modules/mesh-grants/internal/grants/part.go | 74 ++- .../mesh-grants/internal/grants/records.go | 5 + .../mesh-grants/internal/grants/rootfree.go | 11 +- modules/mesh-grants/internal/grants/sealed.go | 178 +++++- modules/mesh-grants/internal/grants/tools.go | 104 +++- modules/mesh-grants/module.json | 21 +- 74 files changed, 2550 insertions(+), 1574 deletions(-) create mode 100644 modules/assignments-readers create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go delete mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go delete mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go delete mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go delete mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go create mode 100644 modules/mesh-grants/internal/grants/accounts.go create mode 100644 modules/mesh-grants/internal/grants/consumers.go create mode 100644 modules/mesh-grants/internal/grants/grants.go diff --git a/modules/assignments-readers b/modules/assignments-readers new file mode 100644 index 0000000..436863a --- /dev/null +++ b/modules/assignments-readers @@ -0,0 +1,7 @@ +# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with +# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry +# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's +# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here +# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it. +mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after +mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1323409..234b56a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -177,8 +177,8 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) { // 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket. func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { for _, missing := range readBuckets { - if missing == delivery.PlacementBucket { - continue // it starts without it, below + if missing == delivery.PlacementBucket || sourceOfGrants(missing) { + continue // it starts without it, below (the grants' source: main_test.go) } b := startBed(t, missing) select { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 54791b2..34f00a4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -8,6 +8,7 @@ package main import ( "context" + "errors" "fmt" "io" "log" @@ -54,6 +55,8 @@ var readBuckets = []string{ provider.AssignmentsBucket, delivery.BuildsBucket, delivery.ControllerGrantsBucket, + delivery.PartGrantsBucket, + buscompose.LeaseBucket, delivery.ControllerComposedBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, @@ -83,7 +86,8 @@ var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposals // keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view // (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view // and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and -// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9). +// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by +// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3). var keptStates = []announce.State{ {Name: delivery.PlacementState, Readable: true}, {Name: delivery.PlacementProposalsState, Readable: true}, @@ -105,8 +109,33 @@ type buckets map[string]jetstream.KeyValue // without it and takes no put further until it starts again (ADR 0546). func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) { out := buckets{} + // The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names + // where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the + // controller holds it, as before. + away, why := false, "" + if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil { + logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err) + } else { + out[buscompose.LeaseBucket] = kv + e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey) + var raw []byte + if gerr == nil { + raw = e.Value() + } + if errors.Is(gerr, jetstream.ErrKeyNotFound) { + gerr = nil + } + away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr) + } for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { + if name == buscompose.LeaseBucket { + continue + } kv, err := js.KeyValue(ctx, name) + if name == provider.AssignmentsBucket && err != nil && away { + logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why) + continue + } if name == delivery.AssignerPutsBucket && err != nil { logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err) continue @@ -117,6 +146,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if sourceOfGrants(name) && err != nil { + // The grants part's grants and the controller's lease choose where grants are read from (issue 774): without + // them the controller's grants are read, as before the lease is handed away. + logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err) + continue + } if err != nil { return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) } @@ -125,6 +160,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any return out, nil } +// sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the +// controller's lease: neither stops the assigner when it is not there (delivery.GrantSource). +func sourceOfGrants(name string) bool { + return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket +} + // run is the program: it returns when ctx ends or the bus cannot be used. func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags) @@ -170,6 +211,8 @@ var watched = []string{ buscompose.ResolvedBucket, provider.AssignmentsBucket, delivery.ControllerGrantsBucket, + delivery.PartGrantsBucket, + buscompose.LeaseBucket, delivery.ControllerComposedBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, @@ -233,7 +276,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if e.Operation() == jetstream.KeyValuePut { value = e.Value() } + before := w.grantsBucket() w.putEntry(name, e.Key(), value, e.Revision(), e.Created()) + if name == buscompose.LeaseBucket && w.grantsBucket() == before { + // The controller renews its lease every few seconds: only a change of where grants are read from + // makes a pass (delivery.GrantSource). + continue + } wake() } } @@ -275,7 +324,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && slices.Contains(seatBuckets, name) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket) { continue // not on the bus at this start (open said so) } started++ @@ -297,13 +346,21 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, name string kv jetstream.KeyValue } - placementAppeared := make(chan appeared, len(seatBuckets)) + placementAppeared := make(chan appeared, len(seatBuckets)+2) for _, name := range seatBuckets { if kvs[name] == nil && js != nil { process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it", logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} }) } } + // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket} { + if kvs[name] == nil && js != nil { + name := name + process.AwaitBucket(ctx, js, nc, name, "the controller's grants are read", + logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} }) + } + } takePlacement := func(a appeared) error { if kvs[a.name] != nil { return nil // bound already, by the other of its two events diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 70572f5..f7a91d4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -17,8 +17,10 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provider" ) // 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them @@ -62,20 +64,30 @@ func TestOpensWhatItsManifestNames(t *testing.T) { } // ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens // the rest and starts, writing wants; any other bucket it cannot open stops the start, named. + // Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the + // assignments refused while the controller holds it stop the start, and once it is handed away they are not needed. + handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`) for _, c := range []struct { missing string err error starts bool + lease []byte + said string }{ - {delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true}, - {delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true}, - {delivery.AssignerPutsBucket, context.DeadlineExceeded, true}, - {delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false}, + {delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"}, + {delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"}, + {delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"}, + {delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""}, + // Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start. + {delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"}, + {buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"}, + {provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""}, + {provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"}, } { var logged string - kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) }) + kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) }) switch { - case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")): + case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)): t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged) case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err) @@ -114,13 +126,18 @@ type refusingJS struct { jetstream.JetStream missing string err error + lease []byte // the controller's lease, when it holds one } func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { if name == r.missing { return nil, r.err } - return &memKV{values: map[string][]byte{}}, nil + kv := &memKV{values: map[string][]byte{}} + if name == buscompose.LeaseBucket && r.lease != nil { + kv.values[buscompose.ControllerLeaseKey] = r.lease + } + return kv, nil } // watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index ad767b0..24d22c5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -149,6 +149,8 @@ func (p *passer) wants(ctx context.Context) { stale := map[string]neitherOne{} left := map[string]string{} defer func() { p.sayNeither(neither, stale, left) }() + // Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774). + grants := p.w.grantsBucket() for name := range names { fp := trunk[name].FP2 name := name @@ -164,7 +166,7 @@ func (p *passer) wants(ctx context.Context) { }, GrantedAt: func(fp string) bool { key, err := delivery.ControllerGrantKey(name, fp) - return err == nil && p.w.has(delivery.ControllerGrantsBucket, key) + return err == nil && p.w.has(grants, key) }, Restored: func(node string) bool { key, err := delivery.RestoreAskKey(node, name, fp) @@ -180,8 +182,8 @@ func (p *passer) wants(ctx context.Context) { hold, _ := delivery.HoldAskKey(name, fp) m.Released = p.answeredWith(hold, "release") m.Kept = p.answeredWith(hold, "keep") - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant) - if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil { + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant) + if raw := p.w.value(grants, grant); raw != nil { if g, err := delivery.DecodeControllerGrant(raw); err == nil { m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index e584c94..35dc5af 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provider" @@ -135,6 +136,9 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin type memStore map[string]*memKV func (s memStore) get(state, key string) ([]byte, uint64, bool, error) { + if s[state] == nil { + return nil, 0, false, nil + } v, ok := s[state].values[key] return v, uint64(len(v)), ok, nil } @@ -873,6 +877,24 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) { if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" { t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread) } + + // Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules + // run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of + // the node. A lease that goes back to a controller brings the comparison back. + place("n2", "d") + assigned("g14", "e") + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)) + got, all := p.w.assigned() + if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 { + t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all) + } + if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 { + t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:]) + } + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`)) + if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] { + t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen) + } } // 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 7549bb9..10e48f9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -213,8 +213,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo // composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one // place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and -// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them -// from their new owner and nothing else changes. +// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket). func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) { var out *delivery.ComposedDigest if d, ok := p.w.composed(node, module, fp); ok { @@ -222,7 +221,7 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, } granted := false if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil { - if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil { + if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil { if g, err := delivery.DecodeControllerGrant(raw); err == nil { granted = g.NamesNode(node, module) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 85d363d..42aa0bc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -524,7 +524,7 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, held, recorded := p.lays(ctx) again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, - Seed: prop.Seed, ExportDigest: prop.ExportDigest}) + Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()}) if len(findings) > 0 { p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; ")) return @@ -661,7 +661,7 @@ func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.An } p.seat.record(answer.Warrant) p.sayChanged(prop, answer, changes) - if p.seat != nil && !assign.Leased && !prop.Seed { + if p.seat != nil && !p.w.handedAway() && !prop.Seed { p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant)) } } @@ -729,7 +729,7 @@ func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal now := p.clock() seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now, Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{}, - Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)}) + Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()}) if len(findings) > 0 { p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; ")) return diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index b599301..ebbf4fb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -9,6 +9,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -214,7 +215,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { } now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()}) if len(findings) > 0 { return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; ")) } @@ -298,7 +299,7 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) { now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{}, - Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)}) + Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()}) if len(findings) > 0 { return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; ")) } @@ -308,6 +309,15 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) { return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil } +// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease +// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or +// unreadable, is not handed away. +func (s *seatTools) leased() bool { + raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err) + return away +} + // handActCall records a hand act in the controller's hand-act log. const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act" @@ -315,7 +325,7 @@ const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act" // proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys, // records the hand act and says placement-changed; every node returns to the controller's compositions. func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) { - if assign.Leased { + if s.leased() { return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to") } why := arg(args, "why") diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index fb22c73..89c25dd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -9,6 +9,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -142,6 +143,15 @@ func TestTheSeatsVerbs(t *testing.T) { if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 { t.Errorf("hand-back = %+v, %v", got, err) } + // Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and + // there is no controller to hand placement back to. + st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}} + if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil { + t.Errorf("a hold after the lease write: %v", err) + } + if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") { + t.Errorf("a hand-back after the lease write: %v", err) + } if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil { t.Errorf("the seat's verbs as served: %+v", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 3dba2bc..258e526 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -195,21 +195,38 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed // assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order // (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module. +// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6). func (w *world) assigned() (map[string][]string, map[string]assignment) { - byAssignment, all := w.assignedByController() + // A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh: + // the lease write's own check refuses that order (ADR 0547 §11). placed, seeded := w.placed() - both, _ := assign.Placed(byAssignment, placed, seeded) + byAssignment, all := w.assignedByController() + if away := w.handedAway(); away && seeded { + both, _ := assign.Placed(byAssignment, placed, seeded, away) + return both, map[string]assignment{} + } + both, _ := assign.Placed(byAssignment, placed, seeded, false) return both, all } -// placementDiffers is every pair the controller's assignments and the seat's placement do not both name. +// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once +// the controller's lease is handed away, since the placement is then the one source. func (w *world) placementDiffers() []assign.Pair { - byAssignment, _ := w.assignedByController() placed, seeded := w.placed() - _, differ := assign.Placed(byAssignment, placed, seeded) + byAssignment, _ := w.assignedByController() + _, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway()) return differ } +// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway, +// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its +// bucket not granted or not there), a controller's, expired or unreadable, is not handed away. +func (w *world) handedAway() bool { + raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey] + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil) + return away +} + // placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key // that does not read places nothing on its node and is said by the indexer's own read tool. func (w *world) placed() (map[string][]string, bool) { @@ -546,6 +563,16 @@ func (w *world) value(bucket, key string) []byte { return w.values[bucket][key] } +// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's +// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other +// case, a lease not held here included. +func (w *world) grantsBucket() string { + raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil) + bucket, _ := delivery.GrantSource(away) + return bucket +} + // has says whether a bucket holds a key now. func (w *world) has(bucket, key string) bool { w.mu.Lock() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index eb4fa5b..7ac1d63 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -6,7 +6,9 @@ import ( "slices" "strings" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-assigner/internal/assign" @@ -125,4 +127,40 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) { if got := w.changed(delivery.HaveBucket, "gone"); got != 0 { t.Errorf("changed = %d for a key never seen", got) } + // Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant + // only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it + // holds it, a grant only the controller holds being the one that counts instead. + fp := delivery.SourcePrefix2 + strings.Repeat("d", 64) + gkey, _ := delivery.ControllerGrantKey("web", fp) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}}) + for _, row := range []struct { + name, lease string + inPart, inController bool + bucket string + granted bool + }{ + {"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true}, + {"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false}, + {"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false}, + {"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true}, + {"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false}, + {"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false}, + } { + p := newPasser(buckets{}, time.Now()) + if row.lease != "" { + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease)) + } + if row.inPart { + p.w.put(delivery.PartGrantsBucket, gkey, grant) + } + if row.inController { + p.w.put(delivery.ControllerGrantsBucket, gkey, grant) + } + if got := p.w.grantsBucket(); got != row.bucket { + t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket) + } + if _, granted := p.composedFor("ace", "web", fp); granted != row.granted { + t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted) + } + } } diff --git a/modules/mesh-delivery-assigner/internal/assign/placement.go b/modules/mesh-delivery-assigner/internal/assign/placement.go index 9f31c35..131512d 100644 --- a/modules/mesh-delivery-assigner/internal/assign/placement.go +++ b/modules/mesh-delivery-assigner/internal/assign/placement.go @@ -2,6 +2,7 @@ package assign import ( "fmt" + "slices" "sort" "strings" @@ -15,7 +16,8 @@ import ( // removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal // the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not // yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key -// is written the controller's assignments are the one source, as before. +// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease +// handed away) the placement is the one source. // Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or // OnlyPlacement. @@ -32,8 +34,17 @@ const ( ) // Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then -// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. -func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) { +// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's +// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the +// assignments are then a source of nothing. +func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) { + if handedAway && seeded { + out := make(map[string][]string, len(placed)) + for node, modules := range placed { + out[node] = slices.Sorted(slices.Values(modules)) + } + return out, nil + } if !seeded { return assigned, nil } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat.go b/modules/mesh-delivery-assigner/internal/assign/seat.go index 0874e81..05a02c6 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat.go @@ -15,10 +15,6 @@ import ( // `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the // operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do. -// Leased says whether the lease write of ADR 0547 has happened, after which the controller's assignments are read no -// more (§8). Not before L3: until then the holder keeps the controller in step and refuses what its verbs cannot take. -var Leased = false - // ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the // revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's // digest. @@ -32,6 +28,10 @@ type ProposeIn struct { Recorded func(node string) bool Seed bool ExportDigest string + // Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk + // buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then + // the holder keeps the controller in step and refuses what its verbs cannot take. + Leased bool } // Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision), @@ -58,7 +58,7 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1)) continue } - if !Leased { + if !in.Leased { if why := c.BeforeLease(); why != "" { findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why)) } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat_test.go b/modules/mesh-delivery-assigner/internal/assign/seat_test.go index 1f9a8fd..257da34 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat_test.go @@ -60,18 +60,18 @@ func TestThePlacementSeatDecides(t *testing.T) { if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 { t.Errorf("config values before the lease write: %v", f) } - Leased = true for _, c := range []delivery.PlacementProposalChange{ {Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}, {Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}, {Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)}, {Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"}, } { - if _, _, f := Propose(in(c)); len(f) != 0 { + leased := in(c) + leased.Leased = true + if _, _, f := Propose(leased); len(f) != 0 { t.Errorf("after the lease write %s is refused: %v", c.Op, f) } } - Leased = false // A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again. if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" || diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1803f4b..c4d473c 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -33,9 +33,9 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", + want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"} + "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 91b6bca..191d590 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -24,12 +24,14 @@ "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", + "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", + "mesh-grants.grants", "node-engine.have" ], "uses": [ diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f003aae..7fb207c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -153,11 +153,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The toolchains first, so a build that starts from the first modules value finds the nodes. watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger) - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf) if err != nil { return err } @@ -254,7 +254,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil case err := <-errs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-watch.errs: return err diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go new file mode 100644 index 0000000..407c6ba --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the +// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and +// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, +// issue 774). +func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { + set := &buildrun.ToolchainSet{} + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } + // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk + // head is told why and opens it again then. + src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} + src.start() + if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { + t.Errorf("a trunk head with the source refused: %v, want why", err) + } + if src.opened { + t.Error("a refused source counts as opened, so it is never opened again") + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go deleted file mode 100644 index 4a0e7ca..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/process" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } - - // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every - // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without - // is an error naming the bucket and its grant. - var logs bytes.Buffer - logger := log.New(&logs, "", 0) - refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} - if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { - t.Errorf("a refused watch of the assignments stopped the start: %v", err) - } - if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { - t.Errorf("a refused toolchains bucket stopped the start: %v", err) - } - if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { - t.Errorf("the refusals were not said: %q", logs.String()) - } - if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || - !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { - t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) - } - // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk - // head is told why and opens it again then. - src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} - src.start() - if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { - t.Errorf("a trunk head with the source refused: %v, want why", err) - } - if src.opened { - t.Error("a refused source counts as opened, so it is never opened again") - } -} - -// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. -type refusingBus struct { - refuseOpen bool - err error -} - -func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { - if r.refuseOpen { - return nil, r.err - } - return refusedWatch{err: r.err}, nil -} - -type refusedWatch struct { - jetstream.KeyValue - err error -} - -func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} -func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index b8b03ce..6dd30b0 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index cbc4905..c82187f 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 44c3002..cf0cac7 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -92,8 +92,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). - if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments", - "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 2929850..faf5694 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -33,7 +33,7 @@ "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.assignments", + "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 660b6d5..33045f3 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -170,6 +170,21 @@ func TryTools(tr Trying) []stdio.Tool { } } +// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named .. +const Seat = "mesh-delivery-checker" + +// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try, +// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them. +func SeatTryTools(tr Trying) []stdio.Tool { + verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"} + var out []stdio.Tool + for _, t := range TryTools(tr) { + t.Name = Seat + "." + verbs[t.Name] + out = append(out, t) + } + return out +} + // Tools are the read tools over the checker's states. func Tools(r check.Reader) []stdio.Tool { return []stdio.Tool{ @@ -308,7 +323,7 @@ func main() { _, err := verdicts.Update(key, json.RawMessage(value), revision) return err }} - tools := append(append(Tools(r), TryTools(trying)...), RecoveryTools(r, written)...) + tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 282db23..779922b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -88,6 +88,24 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) { t.Errorf("read = %s, %v", raw, err) } + // The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike. + verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) }, + Trunks: func() (map[string]string, error) { + return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil + }}) + if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" { + t.Fatalf("verbs = %+v", verbs) + } + again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"}) + if err != nil || again.(map[string]any)["id"] != id { + t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id) + } + if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) { + t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw) + } + if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil { + t.Error("the verb tried a repository nothing checks") + } // Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew. ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"}) if err != nil || ended.(map[string]any)["state"] != "error" { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 5f3018c..732bdfb 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -208,11 +208,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The toolchains first, so a check that starts from the first candidate finds the nodes. watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf) if err != nil { return err } @@ -362,7 +362,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil case err := <-errs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-watch.errs: return err diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go new file mode 100644 index 0000000..3c20242 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the +// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and +// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, +// issue 774). +func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { + set := &buildrun.ToolchainSet{} + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go deleted file mode 100644 index 0715aea..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/process" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } - - // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every - // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without - // is an error naming the bucket and its grant. - var logs bytes.Buffer - logger := log.New(&logs, "", 0) - refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} - if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { - t.Errorf("a refused watch of the assignments stopped the start: %v", err) - } - if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { - t.Errorf("a refused toolchains bucket stopped the start: %v", err) - } - if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { - t.Errorf("the refusals were not said: %q", logs.String()) - } - if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || - !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { - t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) - } -} - -// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. -type refusingBus struct { - refuseOpen bool - err error -} - -func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { - if r.refuseOpen { - return nil, r.err - } - return refusedWatch{err: r.err}, nil -} - -type refusedWatch struct { - jetstream.KeyValue - err error -} - -func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} -func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 0eb617e..e4af0f0 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index cbc4905..c82187f 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 404988d..218d358 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -38,6 +38,11 @@ type stream struct { } type manifest struct { + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` Streams []stream `json:"streams"` Tools []string `json:"tools"` Module string `json:"module"` @@ -96,8 +101,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments", - "mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", + if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-delivery-assigner.compositions", "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } @@ -129,7 +134,12 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code. + if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-delivery-checker" || m.Claims[0].Scope != "mesh" || + !reflect.DeepEqual(m.Claims[0].Serves, []string{"try", "try-read", "try-end"}) { + t.Errorf("claims = %+v, want the mesh seat mesh-delivery-checker serving try, try-read and try-end", m.Claims) + } + for _, k := range []string{"logs", "emits", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the checker has none", k) } diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 6d12e3b..f020663 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -2,6 +2,17 @@ "module": "mesh-delivery-checker", "version": "1", "slug": "checker", + "claims": [ + { + "name": "mesh-delivery-checker", + "scope": "mesh", + "serves": [ + "try", + "try-read", + "try-end" + ] + } + ], "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported" @@ -23,8 +34,8 @@ "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.assignments", "mesh-delivery-assigner.compositions", + "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index edecd2f..e27b51f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -578,23 +578,39 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC } } -// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where -// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the -// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain -// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard -// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch. -func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { +// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of +// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside +// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's +// toolchain alone; when the placement puts mesh-build-go on n2 again, its report written since counts and the builders +// disagree (the guard stays); when it is taken off again, the fingerprint comes back. No restart, no timer: the +// placement's watch (ADR 0549 §3), not the controller's assignments. +func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - var assignments jetstream.KeyValue + var placement, toolchains jetstream.KeyValue + place := func(ctx context.Context, t *testing.T, node string, modules ...string) { + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + key, _ := delivery.PlacementKey(node) + value, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + if _, err := placement.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + } js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { t.Fatal(err) } + toolchains = kv for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} { key, _ := delivery.ToolchainKey(node, "go") value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()}) @@ -605,18 +621,12 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } } - // The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment). - if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil { + // The seat mesh-placement's placement, as the assigner writes it (ADR 0567): n1 places mesh-build-go, n2 nothing. + if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil { t.Fatal(err) } - for key, value := range map[string]string{ - "n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`, - "n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`, - } { - if _, err := assignments.Put(ctx, key, []byte(value)); err != nil { - t.Fatal(err) - } - } + place(ctx, t, "n1", "mesh-build-go") + place(ctx, t, "n2") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -647,16 +657,22 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { if !agreed(held) { t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains) } - if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil { + place(ctx, t, "n2", "mesh-build-go") + key, _ := delivery.ToolchainKey("n2", "go") + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: time.Now().Add(time.Second)}) + if err != nil { t.Fatal(err) } - // n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept + if _, err := toolchains.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + // n2's report, written since its placement, counts: the builders disagree. A fingerprint already computed is kept // (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree. time.Sleep(500 * time.Millisecond) if e, err := modules.Get(ctx, "avahi"); err != nil { t.Fatal(err) } else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 { - t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2) + t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2) } moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1) if moved == string(data) { @@ -669,12 +685,10 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2") } if m := read(disagree); !disagree(m) { - t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2) - } - if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil { - t.Fatal(err) + t.Fatalf("within 5 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2) } + place(ctx, t, "n2") if m := read(agreed); !agreed(m) { - t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2) + t.Fatalf("within 5 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.FP2, m.None2) } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index aa732ef..e55b265 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -24,6 +24,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" @@ -321,14 +322,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { sourceErrs <- follow(ctx, sw, each) }() } } - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf) - assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) { + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, func(ctx context.Context) { if err := ix.Recompute(ctx); err != nil { - logger.Printf("writing the modules again after an assignment of a build module changed: %v", err) + logger.Printf("writing the modules again after the placement of a build module changed: %v", err) } - }, logger) + }, logger.Printf) if err != nil { return err } @@ -442,7 +443,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-sourceErrs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-tools.errs: return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go deleted file mode 100644 index 28fccc4..0000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 6523675..f4410c0 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 6245202..c82187f 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index e05aab4..900ac2c 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -89,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-assigner.placement", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 544e452..693205a 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -22,8 +22,8 @@ "reads": [ "mesh-build-go.toolchains", "mesh-build-node.toolchains", - "mesh-controller.assignments", "mesh-controller.builds", + "mesh-delivery-assigner.placement", "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go index 1e0c713..a9ae081 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go @@ -19,7 +19,11 @@ import ( func serve(t *testing.T) string { t.Helper() m := view.New() - m.Apply(view.Entry{Bucket: view.AssignmentsBucket, Key: "a.x", Value: []byte(`{"node":"a","module":"x","build":"b1"}`)}) + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "x"}}}) + if err != nil { + t.Fatal(err) + } + m.Apply(view.Entry{Bucket: delivery.PlacementBucket, Key: "placement.a", Value: placed}) dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c") if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index d76f73b..55721c7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -166,6 +166,15 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { node, module := nodeModuleOf(args) return r.Haves(node, module) }}, + {Name: view.TrustsReadTool, + Description: "What every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2): each node's record's trusts " + + "(from node-engine_have .node), whether it names the vault's signing key as mesh-vault publishes it in its " + + "state signers and whether it still names the controller's, the nodes that do not name the vault's yet (not_yet), " + + "and every: true once all do, the gate before the controller's lease is handed away. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return r.Trusts() + }}, } } @@ -223,8 +232,9 @@ func read(state stdio.KeptState) view.StateRead { func main() { r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")), - ReachState: read(stdio.State("reach")), - HeardState: read(stdio.State("mesh-delivery-indexer.heard"))} + ReachState: read(stdio.State("reach")), + HeardState: read(stdio.State("mesh-delivery-indexer.heard")), + SignersState: read(stdio.State("mesh-vault.signers"))} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 3cb27d2..b103e85 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -1,13 +1,17 @@ package main import ( + "crypto/ed25519" "encoding/json" "fmt" + "reflect" "strings" "sync/atomic" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-view/internal/view" ) @@ -438,6 +442,41 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) { t.Errorf("haves of ace (the node's own key too) = %v", got) } + // L3 s6b (ADR 0550 §2): node_trusts answers for every node in one call whether its record names the vault's key the + // vault publishes, and every is true only once all do. + vaultPub, _, _ := ed25519.GenerateKey(nil) + vaultKey, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now())) + vaultFP := delivery.SignerFingerprint(vaultPub) + record := func(node string, trusts ...string) string { + raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{V: delivery.Version, Record: &delivery.NodeRecord{Node: node, Trusts: trusts}}) + return string(raw) + } + for _, row := range []struct { + name string + haves fake + every bool + notYet []string + }{ + {"one node trusts the controller's alone", fake{"ace.node": record("ace", "controller key sha256:aa", "vault key "+vaultFP), + "g14.node": record("g14", "controller key sha256:aa"), "ace.zsh": `{"fp":"x"}`}, false, []string{"g14"}}, + {"every node trusts the vault's key", fake{"ace.node": record("ace", "vault key "+vaultFP), + "g14.node": record("g14", "controller key sha256:aa", "vault key "+vaultFP)}, true, []string{}}, + } { + tr := view.Reader{HavesState: row.haves.reader().State, SignersState: fake{"key": string(vaultKey)}.reader().State} + out, err := run(t, tr, view.TrustsReadTool, map[string]any{}) + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + a := out.(view.TrustsAnswer) + if a.Every != row.every || !reflect.DeepEqual(a.NotYet, row.notYet) || a.VaultKey != vaultFP || len(a.Nodes) != 2 { + t.Errorf("%s: %+v", row.name, a) + } + } + if out, err := run(t, view.Reader{HavesState: fake{"ace.node": record("ace", "vault key "+vaultFP)}.reader().State, + SignersState: fake{}.reader().State}, view.TrustsReadTool, map[string]any{}); err != nil || out.(view.TrustsAnswer).Every { + t.Errorf("with no key published: %+v, %v", out, err) + } + // More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list // nothing; the whole answer reads MaxStateReads values in parallel and says it was cut. slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) }) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 55eedf8..140707e 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -48,7 +48,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -68,9 +68,20 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 } return b } - put(view.AssignmentsBucket, "a.x", []byte(`{"node":"a","module":"x","build":"b1"}`)) - put(view.AssignmentsBucket, "a.y", []byte(`{"node":"a","module":"y","build":"b2"}`)) - put(view.AssignmentsBucket, "b.x", []byte(`{"node":"b","module":"x","build":"b1","ended":"2026-10-18T11:30:00Z"}`)) + // The seat's placement (issue 774, ADR 0549 §3): x and y on a, nothing on b. + placeValue := func(node string, modules ...string) []byte { + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + v, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + return v + } + put(delivery.PlacementBucket, "placement.a", placeValue("a", "x", "y")) + put(delivery.PlacementBucket, "placement.b", placeValue("b")) put(view.BuildsBucket, "x.build.b1", []byte(`{"id":"b1","module":"x"}`)) put(view.BuildsBucket, "x.newest", []byte("b1")) put(delivery.HaveBucket, "a.x", haveValue("b1", delivery.PhaseChecking, delivery.HealthWaits)) @@ -150,7 +161,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -236,7 +247,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index ba9c342..dfdd0c5 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -1,4 +1,4 @@ -// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, assignments +// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, the seat's placement // and builds buckets on its own bus credential, derives one row per module and node in memory, and serves them on a // local socket (for mesh-delivery-cli) and on a local web page. It starts, stops or retries nothing; the one thing it writes is the operator's answer to an ask that needs no proof of who // answered, under `.view` in its bucket answers (ADR 0528 clause 6). @@ -106,7 +106,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { + for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { go follow(ctx, js, bucket, false, model, logger) } // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 2d50f31..d405399 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 6245202..c82187f 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 05ee803..9fdbe2f 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -44,6 +44,8 @@ type Reader struct { ReachState StateRead // the view's own reach (ADR 0556) // HeardState is the FORGE messages the indexer handled (issue 1126 note 76); unset reads none. HeardState StateRead + // SignersState is the vault's public signing halves (ADR 0550 §2); unset reads none. + SignersState StateRead } // ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556). @@ -649,3 +651,89 @@ func (r Reader) Differences(node, module string) (KeptAnswer, error) { func (r Reader) Haves(node, module string) (KeptAnswer, error) { return r.kept(r.HavesState, "the haves", node, module) } + +// TrustsReadTool reads what every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2; design 47's L3 +// s6b): the gate before the lease write is that every node's record names the vault's key. +const TrustsReadTool = "node_trusts" + +// NodeTrusts is what one node's record says it trusts: its lines (delivery.Trusts) and whether they name the vault's key +// the vault publishes and the controller's key. +type NodeTrusts struct { + Trusts []string `json:"trusts"` + TrustsVault bool `json:"trusts_vault"` + TrustsController bool `json:"trusts_controller"` +} + +// TrustsAnswer is node_trusts' answer: the vault's key as it publishes it, each node's trusts, the nodes whose record does +// not name the vault's key yet, and whether every node does. +type TrustsAnswer struct { + VaultKey string `json:"vault_key,omitempty"` + Nodes map[string]NodeTrusts `json:"nodes"` + NotYet []string `json:"not_yet"` + Every bool `json:"every"` + Unreadable []string `json:"unreadable,omitempty"` + Note string `json:"note,omitempty"` +} + +// Trusts reads every node's record (`.node` of the haves) and the vault's `signers` key, and answers which nodes +// trust the vault's key. Read-only. +func (r Reader) Trusts() (TrustsAnswer, error) { + if r.HavesState.Get == nil || r.HavesState.Keys == nil { + return TrustsAnswer{}, fmt.Errorf("the haves cannot be read here") + } + out := TrustsAnswer{Nodes: map[string]NodeTrusts{}, NotYet: []string{}} + if r.SignersState.Get != nil { + raw, err := r.SignersState.Get(delivery.SignersKeyKey) + if err != nil { + return TrustsAnswer{}, err + } + if len(raw) > 0 { + k, err := delivery.DecodeSignerKey(raw) + if err != nil { + return TrustsAnswer{}, err + } + out.VaultKey = k.Fingerprint + } + } + keys, err := r.HavesState.Keys() + if err != nil { + return TrustsAnswer{}, err + } + var records []string + for _, k := range keys { + if _, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode { + records = append(records, k) + } + } + values, _, err := r.HavesState.read(records) + if err != nil { + return TrustsAnswer{}, err + } + for key, raw := range values { + node := strings.TrimSuffix(key, "."+delivery.HaveNodeWord) + h, err := delivery.DecodeHaveNode(raw) + if err != nil { + out.Unreadable = append(out.Unreadable, key) + continue + } + var t NodeTrusts + if h.Record != nil { + t.Trusts = h.Record.Trusts + } + for _, line := range t.Trusts { + t.TrustsController = t.TrustsController || strings.HasPrefix(line, delivery.SignerController+" key ") + } + t.TrustsVault = out.VaultKey != "" && delivery.TrustsKey(t.Trusts, delivery.SignerVault, out.VaultKey) + out.Nodes[node] = t + if !t.TrustsVault { + out.NotYet = append(out.NotYet, node) + } + } + sort.Strings(out.NotYet) + sort.Strings(out.Unreadable) + out.Every = out.VaultKey != "" && len(out.Nodes) > 0 && len(out.NotYet) == 0 && len(out.Unreadable) == 0 + if out.VaultKey == "" { + out.Note = "the vault publishes no signing key yet (" + delivery.SignersRead + "), so no node can trust it" + } + return out, nil +} diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index ccdeb3b..7f8acc1 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -4,6 +4,7 @@ package view import ( "encoding/json" "fmt" + "maps" "slices" "sort" "strings" @@ -215,7 +216,7 @@ type UnreadableRow struct { // Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values. type Model struct { mu sync.Mutex - assigns map[string]assignment + placed *delivery.Placed // the seat's placement (the assigner's state, ADR 0567, ADR 0549 §3) builds map[string]buildEntry // by .build. groups map[string]groupEntry // the indexer's candidates, by group modules map[string]moduleEntry // the indexer's modules, by module @@ -263,22 +264,20 @@ type stuckEntry struct { err string } -// assignment holds only the fields of the controller's Assignment the view reads. -type assignment struct { - Node string `json:"node"` - Module string `json:"module"` - Build string `json:"build"` - Commit string `json:"commit"` - Since time.Time `json:"since"` - Ended time.Time `json:"ended"` - State string `json:"state"` - PulledIn bool `json:"pulled-in"` - RequiredBy string `json:"required-by"` -} +// pair is a module the seat's placement puts on a node. +type pair struct{ Node, Module string } -// open says whether the value is an assignment that stands for the module on its node, as the controller's -// Assignment.Stands does. A module pulled in by another stands too. -func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" } +// placedPairs is every module the seat's placement puts on a node, by node then module. The caller holds mu. +func (m *Model) placedPairs() []pair { + byNode := m.placed.ByNode() + var out []pair + for _, node := range slices.Sorted(maps.Keys(byNode)) { + for _, module := range byNode[node] { + out = append(out, pair{Node: node, Module: module}) + } + } + return out +} // build holds only the fields of the controller's Build record the view reads. A record with no state and no failure // is a done build. @@ -356,7 +355,7 @@ type nodeEntry struct { // New is an empty model. func New() *Model { - return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{}, + return &Model{placed: delivery.NewPlaced(), builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}, modHist: map[string][]delivery.Module{}, srcHist: map[uint64]map[string]string{}, taken: map[string]delivery.Taken{}, @@ -529,16 +528,9 @@ func (m *Model) apply(e Entry) { } st, err := delivery.DecodeStuck(e.Value) m.stuck[e.Key] = stuckEntry{value: st, err: errText(err)} - case AssignmentsBucket: - if e.Deleted { - delete(m.assigns, e.Key) - return - } - var a assignment - if json.Unmarshal(e.Value, &a) != nil || a.Node == "" || a.Module == "" { - return - } - m.assigns[a.Node+"."+a.Module] = a + case delivery.PlacementBucket: + // A key that does not read places nothing on its node; the seat's verb placement says why. + _, _, _ = m.placed.Put(e.Key, e.Value, e.Deleted, e.At) case delivery.CandidatesBucket: if e.Deleted { delete(m.groups, e.Key) @@ -714,18 +706,12 @@ func (m *Model) snapshot() Snapshot { m.mu.Lock() defer m.mu.Unlock() rows := []Row{} - for key, a := range m.assigns { - if !a.open() { - continue - } - r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned, Assigned: a.Build} - if a.PulledIn { - r.RequiredBy = a.RequiredBy - } + for _, a := range m.placedPairs() { + r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned} var waits []string hk, herr := delivery.HaveKey(a.Node, a.Module) if herr != nil { - hk = key + continue } h, has := m.haves[hk] // The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node. @@ -745,17 +731,8 @@ func (m *Model) snapshot() Snapshot { r.Have = h.have.FP } r.Since, r.Detail = h.at, h.have.Detail - if wanted { - if h.have.FP != w.value.FP { - waits = append(waits, "the node has not applied "+w.value.FP) - } - } else if a.Build != "" && h.have.Build != a.Build { - waits = append(waits, "the node has not applied "+a.Build) - } - } - if id := m.newest[a.Module]; id != "" && a.Build != "" && id != a.Build { - if b, ok := m.builds[a.Module+".build."+id]; ok && b.done() { - waits = append(waits, a.Module+" has a newer build "+id+" not yet assigned") + if wanted && h.have.FP != w.value.FP { + waits = append(waits, "the node has not applied "+w.value.FP) } } r.Waits = strings.Join(waits, "; ") @@ -766,7 +743,7 @@ func (m *Model) snapshot() Snapshot { if perr != nil { continue } - if a, ok := m.assigns[node+"."+module]; ok && a.open() { + if _, ok := m.placed.Since(node, module); ok { continue } r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err} @@ -984,8 +961,7 @@ func (s Snapshot) Counted() int { // The buckets the view reads besides the have bucket (which mesh-sdk's delivery package names). const ( - AssignmentsBucket = "mesh-controller_assignments" - BuildsBucket = "mesh-controller_builds" + BuildsBucket = "mesh-controller_builds" ) // The statuses of a row. @@ -1176,13 +1152,10 @@ func (m *Model) sourceRow() *SourceRow { waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP))) } } - for key, a := range m.assigns { - if !a.open() { - continue - } + for _, a := range m.placedPairs() { wk, werr := delivery.HaveKey(a.Node, a.Module) if werr != nil { - wk = key + continue } if _, ok := m.wants[wk]; ok { continue diff --git a/modules/mesh-delivery-view/internal/view/view_test.go b/modules/mesh-delivery-view/internal/view/view_test.go index d228cab..f4f8840 100644 --- a/modules/mesh-delivery-view/internal/view/view_test.go +++ b/modules/mesh-delivery-view/internal/view/view_test.go @@ -25,9 +25,22 @@ func haveE(t *testing.T, node, module string, v []byte, at time.Time) Entry { return Entry{Bucket: delivery.HaveBucket, Key: node + "." + module, Value: v, At: at} } -func assign(node, module, build, extra string) Entry { - v := `{"node":"` + node + `","module":"` + module + `","build":"` + build + `","commit":"c","since":"2026-10-18T11:00:00Z"` + extra + `}` - return Entry{Bucket: AssignmentsBucket, Key: node + "." + module, Value: []byte(v), At: t0} +// place is the seat's placement of node, as the assigner keeps it (ADR 0567): the modules placed there. +func place(t *testing.T, node string, modules ...string) Entry { + t.Helper() + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + key, err := delivery.PlacementKey(node) + if err != nil { + t.Fatal(err) + } + v, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.PlacementBucket, Key: key, Value: v, At: t0} } func buildRec(module, id, extra string) []Entry { @@ -58,7 +71,7 @@ func fill(m *Model, es ...Entry) { // base: two nodes (a, b), three modules (x, y, z). func base(t *testing.T) []Entry { es := []Entry{ - assign("a", "x", "b1", ""), assign("a", "y", "b2", ""), assign("b", "x", "b1", ""), assign("b", "z", "b3", ""), + place(t, "a", "x", "y"), place(t, "b", "x", "z"), haveE(t, "a", "x", have(t, "b1", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(1*time.Minute)), haveE(t, "b", "x", have(t, "b1", delivery.PhaseChecking, delivery.HealthWaits, "starting"), t0.Add(2*time.Minute)), haveE(t, "b", "z", have(t, "b3", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(3*time.Minute)), @@ -77,68 +90,35 @@ func TestRowsPerAssignedModuleAndNode(t *testing.T) { // 1134/AC1 t.Fatalf("want 4 rows, got %d counted %d: %+v", len(s.Rows), s.Counted(), s.Rows) } r := find(t, s, "b", "x") - want := Row{Node: "b", Module: "x", Status: StatusAssigned, Assigned: "b1", Have: "b1", Phase: "checking", Health: "waits", + want := Row{Node: "b", Module: "x", Status: StatusAssigned, Have: "b1", Phase: "checking", Health: "waits", Since: t0.Add(2 * time.Minute), Detail: "starting"} if !reflect.DeepEqual(r, want) { t.Fatalf("row = %+v, want %+v", r, want) } y := find(t, s, "a", "y") - if y.Waits != "no have yet" || y.Assigned != "b2" || y.Have != "" { + if y.Waits != "no have yet" || y.Have != "" { t.Fatalf("a.y = %+v, want 'no have yet'", y) } } -func TestWaitsForTheNodeAndForANewerBuild(t *testing.T) { // 1134/AC2 +func TestLeftBehindIsWhatThePlacementDoesNotPut(t *testing.T) { // 1134/AC3, issue 774 (the seat's placement) m := New() fill(m, base(t)...) - fill(m, haveE(t, "a", "x", have(t, "b0", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) - if w := find(t, m.Snapshot(), "a", "x").Waits; w != "the node has not applied b1" { - t.Fatalf("waits = %q", w) - } - fill(m, buildRec("z", "b4", "")...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "z has a newer build b4 not yet assigned" { - t.Fatalf("waits = %q", w) - } - // a newest build that is not done is not a newer build to assign - fill(m, buildRec("z", "b5", `,"failed":"boom"`)...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" { - t.Fatalf("a failed newest build must not be waited on, waits = %q", w) - } - fill(m, buildRec("z", "b6", `,"state":"asked"`)...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" { - t.Fatalf("an asked newest build must not be waited on, waits = %q", w) - } -} - -func TestLeftBehindIsNotRunningAndPulledInIs(t *testing.T) { // 1134/AC3 (both halves) - for name, extra := range map[string]string{"ended": `,"ended":"2026-10-18T11:30:00Z"`, "state": `,"state":"pending"`} { - m := New() - fill(m, base(t)...) - fill(m, assign("b", "z", "b3", extra)) - s := m.Snapshot() - r := find(t, s, "b", "z") - if r.Status != StatusLeftBehind { - t.Errorf("%s: status = %q, want %q", name, r.Status, StatusLeftBehind) - } - if s.Counted() != 3 { - t.Errorf("%s: counted = %d, want 3", name, s.Counted()) - } - } - m := New() - fill(m, base(t)...) - fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // no assignment at all + fill(m, place(t, "b", "x")) // z taken off b s := m.Snapshot() - if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 { - t.Fatalf("a have with no assignment: %+v counted %d", find(t, s, "a", "w"), s.Counted()) + if r := find(t, s, "b", "z"); r.Status != StatusLeftBehind || s.Counted() != 3 { + t.Errorf("status = %q counted %d, want %q and 3", r.Status, s.Counted(), StatusLeftBehind) } - // A module pulled in by another stands: shown and counted, with what required it. m = New() fill(m, base(t)...) - fill(m, assign("b", "z", "b3", `,"pulled-in":true,"required-by":"x"`)) + fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // placed nowhere s = m.Snapshot() - r := find(t, s, "b", "z") - if r.Status != StatusAssigned || r.RequiredBy != "x" || r.Have != "b3" || s.Counted() != 4 { - t.Fatalf("pulled-in row = %+v counted %d, want shown and counted with required-by x", r, s.Counted()) + if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 { + t.Fatalf("a have with no placement: %+v counted %d", find(t, s, "a", "w"), s.Counted()) + } + fill(m, Entry{Bucket: delivery.PlacementBucket, Key: "placement.b", Deleted: true}) + if s := m.Snapshot(); s.Counted() != 2 || find(t, s, "b", "x").Status != StatusLeftBehind { + t.Fatalf("b's placement deleted: counted %d, %+v", s.Counted(), s.Rows) } } @@ -181,18 +161,13 @@ func TestOnlyTheAffectedRowChanges(t *testing.T) { // 1134/AC5 if r := find(t, after, "b", "x"); r.Phase != "running" || r.Health != "ok" || !r.Since.Equal(t0.Add(9*time.Minute)) { t.Fatalf("changed row = %+v", r) } - // an assignment change and a build change touch only their rows - fill(m, assign("a", "y", "b7", "")) - if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != "b7" { - t.Fatalf("assignment change not seen: %+v", r) + // a want's change touches only its row + fill(m, wantE(t, "a", "y", delivery.Want{FP: fpNew})) + if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != fpNew { + t.Fatalf("want change not seen: %+v", r) } if !reflect.DeepEqual(find(t, m.Snapshot(), "b", "z"), find(t, after, "b", "z")) { - t.Fatal("an assignment of another row changed b.z") - } - fill(m, buildRec("x", "b8", "")...) - s := m.Snapshot() - if find(t, s, "a", "x").Waits == "" || find(t, s, "b", "x").Waits == "" || find(t, s, "b", "z").Waits != "" { - t.Fatalf("a build of x must touch x's rows only: %+v", s.Rows) + t.Fatal("a want of another row changed b.z") } // a deleted key (a bucket's expiry) removes its value m.Apply(Entry{Bucket: delivery.HaveBucket, Key: "a.x", Deleted: true}) diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go index 6019daa..80759a7 100644 --- a/modules/mesh-delivery-view/internal/view/wants_test.go +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -101,7 +101,7 @@ func TestEveryWantIsARowAgainstItsHave(t *testing.T) { func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { m := New() fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew), - assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""), + place(t, "ace", "web"), place(t, "g14", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK)) @@ -119,7 +119,7 @@ func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") { t.Errorf("a want that is not the module's fingerprint: %+v", s) } - fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", "")) + fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), place(t, "novox", "web")) s = m.Snapshot().Source if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") { t.Errorf("a node assigned the module with no want: %+v", s) @@ -177,7 +177,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { said = append(said, e) return nil }) - fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""), + fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), place(t, "ace", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew})) if len(said) != 0 { t.Fatalf("said %v before any have", said) @@ -213,7 +213,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { // have: the row shows the fingerprint wanted and waits while the node runs another. func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { m := New() - fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + fill(m, place(t, "ace", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) r := find(t, m.Snapshot(), "ace", "web") if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") { t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r) @@ -228,10 +228,12 @@ func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { // and a have nothing is assigned to keeps the module's name. func TestADottedModuleNameIsFoundAndKept(t *testing.T) { m := New() - fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK), + // The seat's placement refuses a dotted module name (mesh-sdk delivery.Placement), so such a module is never placed: + // its have is a row left behind under its own name. + fill(m, place(t, "ace"), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK), haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK)) s := m.Snapshot() - if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" { + if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Status != StatusLeftBehind { t.Errorf("the dotted module's row = %+v", r) } if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind { diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 50985b4..47727fe 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-vault.signers", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 475f6ca..895fa21 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -14,6 +14,7 @@ "assigner_want", "assigner_differences", "node_haves", + "node_trusts", "build_usage", "view_reach", "indexer_module_history", @@ -30,8 +31,6 @@ { "name": "reach", "history": 1 } ], "reads": [ - "mesh-controller.assignments", - "mesh-controller.builds", "mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", @@ -39,9 +38,11 @@ "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", + "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", + "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", @@ -54,6 +55,7 @@ "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", + "mesh-vault.signers", "node-engine.have" ], "own-secrets": { diff --git a/modules/mesh-grants/cmd/mesh-grants/bus.go b/modules/mesh-grants/cmd/mesh-grants/bus.go index e2ac265..7f839ea 100644 --- a/modules/mesh-grants/cmd/mesh-grants/bus.go +++ b/modules/mesh-grants/cmd/mesh-grants/bus.go @@ -214,6 +214,10 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) { return s.c.State(state).Put(key, json.RawMessage(value)) } +func (s ownStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() } + +func (s ownStore) Delete(state, key string) error { return s.c.State(state).Delete(key) } + // emitting is the process connection as the part's emitter: its events, with no options. type emitting struct{ c *process.Conn } diff --git a/modules/mesh-grants/cmd/mesh-grants/create.go b/modules/mesh-grants/cmd/mesh-grants/create.go index c155393..405a56b 100644 --- a/modules/mesh-grants/cmd/mesh-grants/create.go +++ b/modules/mesh-grants/cmd/mesh-grants/create.go @@ -18,6 +18,41 @@ type creating struct { js jetstream.JetStream } +// CreateConsumer creates one durable consumer an account needs, by its stream and name (ADR 0561 §6): the create alone; +// one the bus holds answers ErrHeld and is left as it is. +func (c creating) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error { + ctx, cancel := context.WithTimeout(c.ctx, requestWithin) + defer cancel() + if _, err := c.js.Consumer(ctx, stream, cfg.Durable); err == nil { + return grants.ErrHeld + } + _, err := c.js.CreateConsumer(ctx, stream, cfg) + switch { + case err == nil: + return nil + case errors.Is(err, jetstream.ErrConsumerExists): + return grants.ErrHeld + case errors.Is(err, context.DeadlineExceeded): + return fmt.Errorf("no answer in %s: a create the bus refuses is answered by none (%w)", requestWithin, err) + } + return err +} + +// MatchConsumer brings a held consumer to cfg on a person's ask (grants_consumer_match), keeping where it delivers from: +// the server refuses a changed deliver policy, and a consumer moved would replay or lose what it holds. +func (c creating) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error { + ctx, cancel := context.WithTimeout(c.ctx, requestWithin) + defer cancel() + held, err := c.js.Consumer(ctx, stream, cfg.Durable) + if err != nil { + return err + } + have := held.CachedInfo().Config + cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = have.DeliverPolicy, have.OptStartSeq, have.OptStartTime + _, err = c.js.UpdateConsumer(ctx, stream, cfg) + return err +} + func (c creating) CreateDeclared(cfg jetstream.StreamConfig) error { ctx, cancel := context.WithTimeout(c.ctx, requestWithin) defer cancel() diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index d971edc..6ef39b6 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -62,14 +62,9 @@ func main() { } } -// keptStates is what the part keeps, each readable (ADR 0495 §2-§3): the view and a session read them. -func keptStates() []announce.State { - var out []announce.State - for _, s := range grants.States { - out = append(out, announce.State{Name: s, Readable: true}) - } - return out -} +// keptStates is what the part keeps (ADR 0495 §2-§3): each readable, for the view and a session, but the wanted accounts, +// by the vault alone (ADR 0561 §2). +func keptStates() []announce.State { return grants.Announced() } // run is the process: it returns when ctx ends, or when it has no credential or cannot dial the bus at all. A bucket or // read the bus refuses never ends it. @@ -117,8 +112,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } bus := newBusReader(ctx, js) f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, - Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, - Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf} + Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, + Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf, + Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}} f.Watch = bus // The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on // the warrant once (mesh-sdk asker), after its refusals. @@ -138,8 +134,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) } // The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two - // condition subjects); it is pulled and every message acknowledged, so its pending never grows. Where it is not there - // yet the pass says so and hears again at the next change. + // condition subjects and the vault's account-issued; ADR 0561 §6 gives that to the part from the lease write); it is + // pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and + // hears again at the next change. The vault's bus-accounts is watched beside it, so a hash is composed either way. f.Hear = func(handle func(string, []byte)) error { cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module) if err != nil { diff --git a/modules/mesh-grants/cmd/mesh-grants/tools.go b/modules/mesh-grants/cmd/mesh-grants/tools.go index 822fb2b..beefe2b 100644 --- a/modules/mesh-grants/cmd/mesh-grants/tools.go +++ b/modules/mesh-grants/cmd/mesh-grants/tools.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "strings" stdio "git.novox.be/novox/mesh-sdk/go" @@ -39,6 +40,10 @@ func (runtimeStore) Put(state, key string, value []byte) (uint64, error) { return stdio.State(state).Put(key, json.RawMessage(value)) } +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + // runtimeBus calls tools and says events through the runtime. type runtimeBus struct{} @@ -113,6 +118,43 @@ func toolsOf(p *grants.Part) []stdio.Tool { Run: func(a map[string]any) (any, error) { return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why")) }}, + {Name: grants.ToolAccounts, Description: "The bus accounts the mesh wants (novox/hq ADR 0561 §2, §9): every account the vault " + + "mints that placement composes (a module's, a process's, a machine's provisioner client's), as the grants part last " + + "wrote it to its state accounts, which the vault alone reads: user, kind, node, module, process, the bus's address and " + + "fingerprint, a module's claims with the verbs each seat promises, and when the key was written; one user's or all; " + + "what the last pass wrote, deleted or kept; and the durable consumers the accounts need as the last pass found them " + + "(missing, differing and why, created; whether the lease is handed away). Never a password. Read-only: " + + "grants_compose_now writes them again.", + Input: schema(map[string]string{"user": "one bus user, e.g. ace.mesh-build-go"}), + Run: func(a map[string]any) (any, error) { return p.Accounts(grants.Answer(a, "user")) }}, + {Name: grants.ToolConsumerMatch, Description: "Bring one durable consumer an account needs (a module's on EVENTS, a process's on " + + "its work queue; novox/hq ADR 0561 §6) to the configuration the bus composer composes, by hand: the recovery when " + + "the condition mesh-grants.consumer-differs names it. Asks the part's process, which alone may, and keeps where the " + + "consumer delivers from. Refused while the controller holds its lease (the consumers are the controller's then) and " + + "for a consumer the records do not compose. grants_user_list's last pass says what became of it.", + Input: schema(map[string]string{"stream": "the stream, e.g. EVENTS", "consumer": "the consumer, e.g. ace_mesh-build-go"}, "stream", "consumer"), + Run: func(a map[string]any) (any, error) { + return p.MatchConsumer(grants.Answer(a, "stream"), grants.Answer(a, "consumer")) + }}, + {Name: grants.ToolAcceptHashes, Description: "Release the hold of a user list that changes several loaded users' hashes at " + + "once (novox/hq ADR 0561 §7; the condition mesh-grants.user-list-held names them), by a person's act with why: names " + + "exactly the users held, bound to the held list's digest, so the part's process writes that list once at its next pass " + + "and says hashes-accepted. It never releases another hold (a failed read, a user dropped for want of a hash, a lost " + + "right), and a list composed otherwise since is held again for a new acceptance.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "users": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "the users held, as the condition names them"}, + "why": map[string]any{"type": "string", "description": "why their hashes change"}}, "required": []string{"users", "why"}}, + Run: func(a map[string]any) (any, error) { + var users []string + if list, ok := a["users"].([]any); ok { + for _, u := range list { + if s, ok := u.(string); ok { + users = append(users, strings.TrimSpace(s)) + } + } + } + return p.AcceptHashes(stdio.CallerOf(a), users, grants.Answer(a, "why")) + }}, {Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " + "every act with what became of it (asked, switched, declined, expired, refused and why).", Input: schema(nil), diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index d164683..a07cd50 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 3cedaa9..61e5c63 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c h1:ngcItoLuh2uP24toZvkCXGA/3+TBrLTwTbE28XpYDWY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 85b58fd..694a03e 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "path/filepath" "slices" @@ -57,6 +58,45 @@ type fakeBus struct { refuseUsers bool // rootFree answers the controller's root-free verb: nil is no answerer at all (no responders). rootFree func(machines []string) (any, error) + // consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and + // consumerCreates each consumer the part asked to create, `.`. + consumers map[string][]map[string]any + consumerCreates []string +} + +// MatchConsumer is the bus's update of a held durable consumer to the configuration given. +func (b *fakeBus) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error { + b.mu.Lock() + defer b.mu.Unlock() + b.consumerCreates = append(b.consumerCreates, "match "+stream+"."+cfg.Durable) + for _, c := range b.consumers[stream] { + if c["name"] == cfg.Durable { + c["max_deliver"] = cfg.MaxDeliver + return nil + } + } + return errors.New("consumer not found") +} + +// CreateConsumer is the bus's create of a durable consumer: held when the bus holds one of that name, else held from now. +func (b *fakeBus) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error { + b.mu.Lock() + defer b.mu.Unlock() + b.consumerCreates = append(b.consumerCreates, stream+"."+cfg.Durable) + for _, c := range b.consumers[stream] { + if c["name"] == cfg.Durable { + return grants.ErrHeld + } + } + filters := append([]string{}, cfg.FilterSubjects...) + if cfg.FilterSubject != "" { + filters = []string{cfg.FilterSubject} + } + if b.consumers == nil { + b.consumers = map[string][]map[string]any{} + } + b.consumers[stream] = append(b.consumers[stream], map[string]any{"name": cfg.Durable, "filters": filters, "max_deliver": cfg.MaxDeliver}) + return nil } func (b *fakeBus) LastMembership(node, module string) ([]byte, uint64, bool, error) { @@ -177,6 +217,9 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) { } return json.Marshal(map[string]any{"users": out}) case grants.CallStreams: + if stream, _ := args["stream"].(string); stream != "" { + return json.Marshal(map[string]any{"detail": []map[string]any{{"name": stream, "consumer_detail": b.consumers[stream]}}}) + } return json.Marshal(map[string]any{"detail": b.streams}) case grants.CallRootFree: if b.rootFree == nil { @@ -217,6 +260,27 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) { return b.revs[state+"/"+key], nil } +func (s ownStore) Keys(state string) ([]string, error) { + b := s.b + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for k := range b.own[state] { + out = append(out, k) + } + slices.Sort(out) + return out, nil +} + +func (s ownStore) Delete(state, key string) error { + b := s.b + b.mu.Lock() + defer b.mu.Unlock() + b.writes = append(b.writes, state+" delete") + delete(b.own[state], key) + return nil +} + func (b *fakeBus) Emit(event string, _ any) error { b.mu.Lock() defer b.mu.Unlock() @@ -1144,7 +1208,7 @@ func TestGrants(t *testing.T) { t.Errorf("membership = %v", ms) } gr, _ := p.Grants("mod-x") - if g, _ := gr.(map[string]any); g["writes"] != grants.WritesNoGrant || g["count"] != 1 { + if g, _ := gr.(map[string]any); g["writes"] != grants.WritesGrants || g["controller_count"] != 1 { t.Errorf("grants = %v", gr) } @@ -1371,7 +1435,12 @@ func TestGrants(t *testing.T) { var fresh []asks.Condition fp := partOn(b) fp.Objects = p.Objects - fp.Asker, fp.Say = fa, func(c asks.Condition) error { fresh = append(fresh, c); return nil } + fp.Asker, fp.Say = fa, func(c asks.Condition) error { + if c.Key != grants.GrantMissingCondition { // its own row below + fresh = append(fresh, c) + } + return nil + } fp.Compose() if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != asks.StateCleared { t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh) @@ -1504,7 +1573,7 @@ func TestGrants(t *testing.T) { } } for _, w := range b.writes { - if !slices.Contains(grants.States, w) { + if !slices.Contains(grants.States, strings.TrimSuffix(w, " delete")) { t.Errorf("wrote %s, which is not the part's own state", w) } } @@ -1714,6 +1783,31 @@ func TestGrants(t *testing.T) { } }) + // 774 (ADR 0563 §2, §3): the node-engine's manifest (mesh-host) is placed on every node though no node is assigned it, + // so its declared state acts is created as a placed module's, and composed for the user list's create right. + t.Run("assertions: the node-engine's declared state is created though no node is assigned it", func(t *testing.T) { + b := fixture() + b.manifestOf(buscompose.EngineModule, `{"module":"mesh-host","version":"1","state":[{"name":"acts"}]}`) + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Create = b + p.Compose() + if !slices.Contains(b.creates, "KV_"+buscompose.EngineActsBucket) { + t.Fatalf("the node-engine's acts were not created: %v", b.creates) + } + recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{})) + if recs.Engine == nil || len(recs.Engine.State) != 1 || recs.Engine.State[0].Bucket() != buscompose.EngineActsBucket { + t.Fatalf("the records carry the node-engine as %+v", recs.Engine) + } + for node, mods := range recs.Assigned { + for _, d := range mods { + if d.Module == buscompose.EngineModule { + t.Errorf("the node-engine is a module of %s, which would give it a module's account", node) + } + } + } + }) + t.Run("assertions: a refused create is said, never stops it, and is tried at the next event", func(t *testing.T) { b := fixture() b.loaded = shadowOf(t, b) @@ -1823,6 +1917,463 @@ func TestGrants(t *testing.T) { t.Errorf("the refusal was not said: %v", logged) } }) + + // 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.. in shadow, once each and only for a + // module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no + // account with a hash and every read was made, and the tool says every difference from the controller's. + t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + fp := "src2:" + strings.Repeat("ab", 32) + key := func(module string) string { k, _ := delivery.PartGrantKey(module, fp); return k } + hash := func(user string) { + b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1}) + } + held := func() []string { return slices.Sorted(maps.Keys(b.own[grants.StateGrants])) } + // The controller's grant for this fingerprint, beside the stale one the fixture holds for another. + delete(b.states[grants.ReadControllerGrant], "mod-x.0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123") + b.put(grants.ReadControllerGrant, key("mod-x"), delivery.ControllerGrant{V: delivery.Version, Accounts: []string{"a.mod-x"}}) + hash("a.mod-x") + hash("a.nats") + res := p.Compose() + if res.Grants == nil || !slices.Equal(held(), []string{key("mod-x"), key("nats")}) || len(res.Grants.Written) != 2 { + t.Fatalf("grants of the hashed modules: held %v, %+v", held(), res.Grants) + } + if g, err := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key("mod-x")]); err != nil || !slices.Equal(g.Accounts, []string{"a.mod-x"}) { + t.Errorf("the grant of mod-x is %+v, %v", g, err) + } + said := func() int { + n := 0 + for _, e := range b.events { + if e == grants.EventGrantWritten { + n++ + } + } + return n + } + if said() != 2 { + t.Errorf("grant-written said %d times for two grants: %v", said(), b.events) + } + if res := p.Compose(); len(res.Grants.Written) != 0 || said() != 2 { + t.Errorf("a second pass wrote %v and said %d", res.Grants.Written, said()) + } + // The tool: equal on mod-x, and nats is the part's alone (the controller's fixture holds none). + got, err := p.Grants("") + gm, _ := got.(map[string]any) + diff, _ := gm["differences"].([]grants.GrantDifference) + if err != nil || gm["equal"] != false || len(diff) != 1 || diff[0].Key != key("nats") || diff[0].Kind != "only the part's" { + t.Errorf("grants = %+v, %v", got, err) + } + // A user with a hash later: written once. Without it again: deleted. Unread hashes: nothing deleted. + hash("b.mod-y") + if res := p.Compose(); !slices.Equal(res.Grants.Written, []string{key("mod-y")}) { + t.Errorf("mod-y was granted %v when its account got a hash", res.Grants.Written) + } + b.refused[grants.ReadAccounts] = true + if res := p.Compose(); res.Grants == nil || len(res.Grants.Deleted)+len(res.Grants.Written) != 0 || len(res.Grants.Kept) != 1 || len(held()) != 3 { + t.Errorf("a pass that could not read the hashes changed grants: %+v, held %v", res.Grants, held()) + } + delete(b.refused, grants.ReadAccounts) + delete(b.states[grants.ReadAccounts], "b.mod-y") + if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 { + t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held()) + } + }) + + // Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the + // new form at that commit (link2pay, snake, f1-circuits, txt-game), is granted at the indexer's fingerprint, never + // judged by its commit; and a placed module left without a grant is said, never passed over: in the pass, and as the + // condition mesh-grants.grant-missing once the part's grants are the ones read, cleared when it has one. + t.Run("grants: at the indexer's fingerprint whatever the commit, and a placed module without one is said", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + var said []asks.Condition + p.Say = func(c asks.Condition) error { + if c.Key == grants.GrantMissingCondition { + said = append(said, c) + } + return nil + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c" + b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)), + Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"}) + b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, None2: "its builder's version is not read yet", + Manifest: delivery.ManifestDigest([]byte(yManifest)), Repository: "novox/y", Commit: "c"}) + for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} { + b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1}) + } + res := p.Compose() + key, _ := delivery.PartGrantKey("mod-x", fp) + if _, held := b.own[grants.StateGrants][key]; !held { + t.Errorf("mod-x has no grant at the indexer's fingerprint %s: %+v", fp, res.Grants) + } + if res.Grants == nil || len(res.Grants.Missing) != 1 || !strings.HasPrefix(res.Grants.Missing[0], + "mod-y: no fingerprint of the new form: its builder's version is not read yet") { + t.Fatalf("missing = %+v", res.Grants) + } + if len(said) != 1 || said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "mod-y") { + t.Fatalf("said %+v", said) + } + if p.Compose(); len(said) != 1 { + t.Errorf("the same missing grant was said again: %+v", said) + } + b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(yManifest)), + Repository: "novox/y", Commit: "c"}) + if res := p.Compose(); len(res.Grants.Missing) != 0 || len(said) != 2 || said[1].State != asks.StateCleared { + t.Errorf("once mod-y has a grant: missing %v, said %+v", res.Grants.Missing, said) + } + }) + + // The guard of ADR 0549 §6 for this repository, as mesh-catalog's checks/placement is for the catalogue: no module.json + // here reads the controller's assignments unless ../assignments-readers names it with why, and that list only shrinks. + t.Run("assignments readers: only the named modules read the controller's assignments, and the list only shrinks", func(t *testing.T) { + raw, err := os.ReadFile("../assignments-readers") + if err != nil { + t.Fatal(err) + } + allowed := map[string]bool{} + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 1 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]] = true + } else if len(f) == 1 { + t.Errorf("../assignments-readers names %s with no why", f[0]) + } + } + paths, _ := filepath.Glob("../*/module.json") + reads := map[string]bool{} + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Reads []string `json:"reads"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", path, err) + } + if slices.Contains(m.Reads, grants.ReadAssignments) { + reads[m.Module] = true + if !allowed[m.Module] { + t.Errorf("%s reads %s: read where modules run from the seat mesh-placement's placement (ADR 0549 §3); the lease write "+ + "is refused while a module reads the assignments (ADR 0549 §6)", m.Module, grants.ReadAssignments) + } + } + } + for m := range allowed { + if !reads[m] { + t.Errorf("../assignments-readers names %s, which no longer reads %s: take it off the list", m, grants.ReadAssignments) + } + } + }) + + // 774 (the L3 review, ADR 0549 §3, §6): once the controller's lease is handed away the assignments are frozen and place + // nothing; the seat's placement alone does. A module left only in the stale assignments loses its account and its + // grant names no account of it; before the lease write the two are still unioned. + t.Run("placement alone once the lease is handed away: a module only in the stale assignments has no account or grant of it", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"} + pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-x"}}}) + b.states[grants.ReadPlacement] = map[string][]byte{"placement.a": pl} + for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} { + b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1}) + } + fp := "src2:" + strings.Repeat("ab", 32) + key, _ := delivery.PartGrantKey("mod-y", fp) + grantOf := func() []string { + g, _ := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key]) + return g.Accounts + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-28", "epoch": 60}) + p.Compose() + if b.own[grants.StateAccounts]["b.mod-y"] == nil || !slices.Equal(grantOf(), []string{"b.mod-y"}) { + t.Fatalf("under the controller's lease the assignments no longer count: account %v, grant %v", + b.own[grants.StateAccounts]["b.mod-y"] != nil, grantOf()) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + in := grants.ReadInputs(b, &grants.Manifests{}) + if len(in.Assigned["b"]) != 0 || !slices.Equal(in.Assigned["a"], []string{"mod-x", "nats"}) || in.BusNode != "a" { + t.Errorf("after the lease write the stale assignments still place: %v, bus %q", in.Assigned, in.BusNode) + } + res := p.Compose() + if res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) || slices.Contains(grantOf(), "b.mod-y") { + t.Fatalf("a module only in the stale assignments kept its account or grant: %+v, grant %v", res.Accounts, grantOf()) + } + }) + + // 774/AC4 (novox/hq ADR 0561 §2): every account the vault mints that the records compose is written to the state + // accounts once, with the bus's address and a module's claims, read by the vault alone; a key is deleted when placement + // no longer names its user, and never in a pass whose reads failed. + t.Run("wanted accounts: written once each, read by the vault alone, retired only when placement drops them", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"} + p.Compose() + p.Compose() + wrote := 0 + for _, w := range b.writes { + if w == grants.StateAccounts { + wrote++ + } + } + held := slices.Sorted(maps.Keys(b.own[grants.StateAccounts])) + if !slices.Equal(held, []string{"a.mod-x", "a.nats", "b.mod-y"}) || wrote != 3 { + t.Fatalf("accounts %v written %d times in two passes, want the three modules' once each", held, wrote) + } + nats, err := buscompose.ReadWantedAccount("a.nats", b.own[grants.StateAccounts]["a.nats"]) + if err != nil || nats.URL != "nats://a.mesh:4222" || nats.Fingerprint != "ab" || len(nats.Claims) != 1 || + nats.Claims[0].Seat != catalogue.BrokerSeat || nats.At.IsZero() { + t.Errorf("a.nats is wanted as %+v (%v)", nats, err) + } + var readers []string + for _, s := range grants.Announced() { + if s.Name == grants.StateAccounts { + if s.Readable { + t.Error("the wanted accounts are announced readable by anyone") + } + readers = s.Readers + } + } + if !slices.Equal(readers, []string{"mesh-vault"}) || buscompose.ReaderMay("mesh-delivery-view", buscompose.GrantsAccounts) { + t.Errorf("the wanted accounts are read by %v", readers) + } + // A read that fails retires nothing; once read, the user placement dropped is retired. + delete(b.states[grants.ReadAssignments], "b.mod-y") + b.refused[grants.ReadResolved] = true + if res := p.Compose(); res.Accounts == nil || len(res.Accounts.Deleted) != 0 || b.own[grants.StateAccounts]["b.mod-y"] == nil { + t.Fatalf("a pass whose read failed deleted %+v", res.Accounts) + } + delete(b.refused, grants.ReadResolved) + if res := p.Compose(); res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) { + t.Fatalf("the user placement dropped is not retired: %+v", res.Accounts) + } + // The tool reads them, one and all, each with when it was written. + one, err := p.Accounts("a.nats") + all, err2 := p.Accounts("") + if err != nil || err2 != nil || one.(map[string]any)["count"] != 1 || all.(map[string]any)["count"] != 2 || + one.(map[string]any)["accounts"].([]buscompose.WantedAccount)[0].At.IsZero() { + t.Errorf("the tool reads %v and %v (%v %v)", one, all, err, err2) + } + if _, err := p.Accounts("b.mod-y"); err == nil { + t.Error("the tool reads an account no longer wanted") + } + p.Pass() + if all, _ := p.Accounts(""); all.(map[string]any)["consumers"] == nil { + t.Errorf("the tool does not say the account consumers in shadow: %v", all) + } + // With no bus address no account is written, and the pass says why. + bare := partOn(fixture()) + if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") { + t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors) + } + }) + + // 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash + // within one pass; an event of another kind does not. + t.Run("the vault's account-issued composes the list again", func(t *testing.T) { + b := fixture() + var handle func(string, []byte) + p := partOn(b) + f := &grants.Follower{Part: p, Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {}, + Hear: func(h func(string, []byte)) error { handle = h; return nil }} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { f.Run(ctx); close(done) }() + defer func() { cancel(); <-done }() + waitFor(t, func() bool { return b.count(grants.CallUsers) > 0 && handle != nil }) + held := func() bool { + for _, c := range p.Last().UserList.Credentials.Users { + if c.User == "a.mod-x" { + return c.HashHeld + } + } + return false + } + if held() { + t.Fatal("a.mod-x has a hash before the vault made one") + } + b.mu.Lock() + b.states[grants.ReadAccounts] = map[string][]byte{"a.mod-x": []byte(`{"user":"a.mod-x","hash":"$2a$10$x","generation":1}`)} + b.mu.Unlock() + handle(buscompose.AccountIssuedSubject, []byte(`{"user":"a.mod-x","kind":"module","node":"a","module":"mod-x","generation":1}`)) + waitFor(t, held) + if !slices.Contains(grants.Consumes, grants.EventAccountIssued) || !grants.IsAccountIssued(grants.EventAccountIssued) || + grants.IsAccountIssued("mesh.mod.mesh-vault.event.secret-changed") { + t.Errorf("the part hears %v", grants.Consumes) + } + }) + + // 774/AC6 (novox/hq ADR 0561 §6): the durable consumers an account needs, with the controller's configuration, are + // compared in shadow while the controller holds the lease, and created, only where the bus lacks them, from the write + // that hands the lease away; one the bus holds is never changed. + t.Run("account consumers: compared while the controller holds the lease, created once it is handed away", func(t *testing.T) { + const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done"]}` + b := fixture() + b.manifestOf("mod-c", cManifest) + b.assign("b", "mod-c") + b.loaded = shadowOf(t, b) + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57}) + p := partOn(b) + p.Consumers = b + res := p.Compose() + if res.Consumers == nil || res.Consumers.On || !slices.Contains(res.Consumers.Missing, "EVENTS.b_mod-c") || len(b.consumerCreates) != 0 { + t.Fatalf("while the controller holds the lease: %+v, created %v", res.Consumers, b.consumerCreates) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}} + res = p.Compose() + if !res.Consumers.On || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) { + t.Fatalf("once handed away: %+v", res.Consumers) + } + var made map[string]any + for _, c := range b.consumers["EVENTS"] { + if c["name"] == "b_mod-c" { + made = c + } + } + if made == nil || made["max_deliver"] != 5 || !slices.Contains(made["filters"].([]string), "mesh.mod.mod-y.event.done") { + t.Errorf("b_mod-c was made as %v, not as the controller makes it", made) + } + res = p.Compose() + if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 { + t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates) + } + // A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to + // the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease. + var said []asks.Condition + p.Say = func(c asks.Condition) error { + if c.Key != grants.GrantMissingCondition { // its own row + said = append(said, c) + } + return nil + } + made["max_deliver"] = 9 + res = p.Compose() + if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") || + len(said) != 1 || said[0].Key != grants.ConsumerDiffersCondition || said[0].State != asks.StateRaised || + !strings.Contains(said[0].Explanation, "EVENTS.b_mod-c") { + t.Fatalf("a differing consumer: %+v, said %+v", res.Consumers, said) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/2/2026-10-27", "epoch": 58}) + if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") { + t.Errorf("matched by hand under the controller's lease: %v", err) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + if _, err := p.MatchConsumer("EVENTS", "b_nothing"); err == nil { + t.Error("a consumer the records do not compose was asked to match") + } + if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err != nil { + t.Fatal(err) + } + res = p.Compose() + if !slices.Contains(b.consumerCreates, "match EVENTS.b_mod-c") || made["max_deliver"] != 5 || len(res.Consumers.Matched) != 1 { + t.Errorf("the ask to match: %+v, %v", res.Consumers, b.consumerCreates) + } + res = p.Compose() + if len(res.Consumers.Differ) != 0 || said[len(said)-1].State != asks.StateCleared { + t.Errorf("once matched: %+v, said %+v", res.Consumers, said) + } + }) + + // 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no + // rotation by the vault naming each, is held: the list in force stays, user-list-held is raised naming those users. One + // user's change, or changes the vault's rotation names, are written. + t.Run("a list changing several loaded users' hashes is held unless the vault's rotation names each", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + hash := func(user, h string, generation int, rotating bool) { + v := map[string]any{"user": user, "hash": h, "generation": generation, "fingerprint": fmt.Sprintf("sha256:%d", generation)} + if rotating { + v["rotating"] = map[string]any{"login": user + "--b", "generation": generation, "revision": 7} + } + b.put(grants.ReadAccounts, user, v) + } + for _, u := range shadowOf(t, b) { + hash(u.User, "$2a$10$first", 1, false) + } + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}}) + p := partOn(b) + p.Objects = &fakeObjects{} + var said []asks.Condition + p.Say = func(c asks.Condition) error { said = append(said, c); return nil } + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + p.Compose() + inForce, _, _ := p.Sealed() + if inForce.Digest == "" { + t.Fatal("no list written after take-over") + } + // One user's hash changes: written. + hash("a.mod-x", "$2a$10$second", 2, false) + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest { + t.Fatal("a list changing one loaded user's hash was not written") + } + inForce, _, _ = p.Sealed() + // Two change at once, no rotation names them: held, said naming both. + hash("a.mod-x", "$2a$10$third", 3, false) + hash("a.nats", "$2a$10$third", 2, false) + said = nil + res := p.Compose() + if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x, a.nats") { + t.Fatalf("a list changing two loaded users' hashes: %+v", res.Sealed) + } + if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "a.mod-x, a.nats") { + t.Errorf("the hold was said as %+v", said) + } + // The hold is its own guard, beside take-over's refusals (ADR 0562): the switch stays taken over. + if p.SwitchState() != grants.TookOver { + t.Errorf("the hold moved the switch to %s", p.SwitchState()) + } + // A person accepts exactly the named users' changes of the held list, with why: refused for another set, with no + // why, or once the list changed since; accepted, the next pass writes it once, said as a hand act. + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x"}, "rotated by hand"); err == nil || !strings.Contains(err.Error(), "a.mod-x, a.nats") { + t.Errorf("an acceptance of another set than the held one: %v", err) + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, ""); err == nil { + t.Error("an acceptance with no why") + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.nats", "a.mod-x"}, "the vault's store was restored"); err != nil { + t.Fatal(err) + } + // The list changed since the acceptance (a.mod-x's hash again): the acceptance is bound to the held list, refused. + hash("a.mod-x", "$2a$10$fourth", 4, false) + res = p.Compose() + if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") { + t.Fatalf("an acceptance of a list that changed since released it: %+v", res.Sealed) + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, "the vault's store was restored"); err != nil { + t.Fatal(err) + } + b.events = nil + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared || + !slices.Contains(b.events, grants.EventHashesAccepted) { + t.Fatalf("an accepted hold was not released once and said: %+v, events %v", said, b.events) + } + inForce, _, _ = p.Sealed() + // The vault's rotation names each: written, the warning cleared. + hash("a.mod-x", "$2a$10$fifth", 5, true) + hash("a.nats", "$2a$10$fifth", 3, true) + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared { + t.Errorf("changes the vault's rotation names were held: %+v", said) + } + }) } type fakeAsker struct { diff --git a/modules/mesh-grants/internal/grants/accounts.go b/modules/mesh-grants/internal/grants/accounts.go new file mode 100644 index 0000000..bf806a8 --- /dev/null +++ b/modules/mesh-grants/internal/grants/accounts.go @@ -0,0 +1,217 @@ +package grants + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// **The accounts the mesh wants, said to the vault** (novox/hq ADR 0561 §2, issue 774). The bus composer derives every +// user from placement, the manifests and the wants; for each account the vault mints (a module's, a process's, a +// machine's provisioner client's), the part writes one key of its state `accounts`, keyed by the bus user, with the +// fields the controller's credential carries beside the password. A key is written only when it differs, and deleted +// only when placement no longer names its user and every input of the pass was read: a refused read never retires an +// account. The vault alone reads the state (buscompose.NamedReaders); it mints each user it holds no value for. + +// StateAccounts is the part's state of wanted accounts. +const StateAccounts = buscompose.AccountsState + +// AccountsReaders are the only modules that may read StateAccounts. +var AccountsReaders = buscompose.NamedReaders[buscompose.GrantsAccounts] + +// Announced is what the part says it keeps: every state readable by any reader, but the wanted accounts, by the vault +// alone (ADR 0495 §2-§3, ADR 0561 §2). +func Announced() []announce.State { + var out []announce.State + for _, s := range States { + if s == StateAccounts { + out = append(out, announce.State{Name: s, Readers: slices.Clone(AccountsReaders)}) + continue + } + out = append(out, announce.State{Name: s, Readable: true}) + } + return out +} + +// BusAddress is where every node on the private network reaches the bus, and its certificate's fingerprint: the part's +// own credential says both, since the controller gave every node's account the bus holder's overlay name (its +// brokerReachableAt). +type BusAddress struct { + URL string + Fingerprint string +} + +// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address. A +// node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure. +func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) { + principals, err := buscompose.Users(recs) + if err != nil { + return nil, err + } + out := map[string]buscompose.WantedAccount{} + for _, p := range principals { + w := buscompose.WantedAccount{User: p.Username(), Kind: p.Kind, Node: p.Node, Module: p.Module, Process: p.Process, + URL: bus.URL, Fingerprint: bus.Fingerprint} + switch p.Kind { + case buscompose.KindModule, buscompose.KindNodeTools: + if m, ok := manifestOn(in, p.Node, p.Module); ok { + w.Claims = ClaimsOf(m) + } + case buscompose.KindProcess, buscompose.KindProvisioner: + default: + continue + } + out[w.User] = w + } + return out, nil +} + +// manifestOn is the manifest a node runs of module: the build its want names, else the trunk's. +func manifestOn(in Inputs, node, module string) (catalogue.Manifest, bool) { + if m, ok := in.Running[node+"."+module]; ok { + return m, true + } + m, ok := in.Manifests[module] + return m, ok +} + +// ClaimsOf is a module's claims as its credential carries them: the controller's claimsFor, over the mesh's own seats +// (its seat table is seeded from them): each claim's seat and scope, and the verbs the runtime serves for it, the claim's +// own or every verb the seat promises; a seat the mesh does not define keeps the claim's scope and no verbs. +func ClaimsOf(m catalogue.Manifest) []buscompose.ClaimedSeat { + if len(m.Claims) == 0 { + return nil + } + byName := map[string]catalogue.Seat{} + for _, s := range catalogue.DefaultSeats() { + byName[s.Name] = s + } + var out []buscompose.ClaimedSeat + for _, c := range m.Claims { + claimed := buscompose.ClaimedSeat{Seat: c.Name, Scope: c.At()} + if s, known := byName[c.Name]; known { + claimed.Scope = s.Scope + claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)}) + } + out = append(out, claimed) + } + return out +} + +// Accounts answers the tool accounts (ADR 0561 §9): the wanted accounts as the part last wrote them, one user's or all, +// each with when it was written; and what the last pass did to them. Never a password: the state holds none. +func (p *Part) Accounts(user string) (any, error) { + users := []string{user} + if user == "" { + keys, err := p.Store.Keys(StateAccounts) + if err != nil { + return nil, fmt.Errorf("its own %s cannot be listed: %w", StateAccounts, err) + } + users = keys + } + out := []buscompose.WantedAccount{} + for _, u := range users { + raw, _, found, err := p.Store.Get(StateAccounts, u) + if err != nil { + return nil, fmt.Errorf("its own %s %s cannot be read: %w", StateAccounts, u, err) + } + if !found { + if user != "" { + return nil, fmt.Errorf("no account %s is wanted: placement composes no module, process or provisioner client of that user", user) + } + continue + } + w, err := buscompose.ReadWantedAccount(u, raw) + if err != nil { + return nil, err + } + out = append(out, w) + } + answer := map[string]any{"accounts": out, "count": len(out), "read_by": AccountsReaders} + if raw, _, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found { + var res Result + if json.Unmarshal(raw, &res) == nil && res.Accounts != nil { + answer["last_pass"] = res.Accounts + } + // And the durable consumers the accounts need, as the last pass found them: missing and differing ones are read + // here in shadow too, where no condition is raised (ADR 0561 §6, §9). + if res.Consumers != nil { + answer["consumers"] = res.Consumers + } + } + return answer, nil +} + +// AccountsWritten is what one pass did to the state of wanted accounts. +type AccountsWritten struct { + Wanted int `json:"wanted"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept are keys placement no longer names that this pass did not delete, with why. + Kept []string `json:"kept,omitempty"` +} + +// writeAccounts writes each wanted account whose key differs, and deletes the key of a user no longer placed, but only +// when every input of the pass was read. +func (p *Part) writeAccounts(res *Result, in Inputs, recs buscompose.Records) { + if p.Bus.URL == "" { + res.Errors = append(res.Errors, "the wanted accounts are not written: the part knows no bus address to give them (its own credential names none)") + return + } + wanted, err := WantedAccounts(in, recs, p.Bus) + if err != nil { + res.Errors = append(res.Errors, "the wanted accounts could not be composed: "+err.Error()) + return + } + done := &AccountsWritten{Wanted: len(wanted)} + res.Accounts = done + for _, user := range sortedKeys(wanted) { + w := wanted[user] + if raw, _, found, err := p.Store.Get(StateAccounts, user); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateAccounts, user, err)) + continue + } else if found { + if was, err := buscompose.ReadWantedAccount(user, raw); err == nil && was.Same(w) { + continue + } + } + w.At = p.now() + data, _ := json.Marshal(w) + if _, err := p.Store.Put(StateAccounts, user, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateAccounts, user, err)) + continue + } + done.Written = append(done.Written, user) + } + held, err := p.Store.Keys(StateAccounts) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no key of a user gone is deleted: %v", StateAccounts, err)) + return + } + failed := in.ReadFailures() + placed := Placed(in) + for _, user := range held { + if _, still := wanted[user]; still { + continue + } + switch { + case placed(user): + done.Kept = append(done.Kept, user+": placement still names it (composed with no account this pass)") + continue + case len(failed) > 0: + done.Kept = append(done.Kept, user+": a read of this pass failed, so no account is retired") + continue + } + if err := p.Store.Delete(StateAccounts, user); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateAccounts, user, err)) + continue + } + done.Deleted = append(done.Deleted, user) + } + sort.Strings(done.Kept) +} diff --git a/modules/mesh-grants/internal/grants/consumers.go b/modules/mesh-grants/internal/grants/consumers.go new file mode 100644 index 0000000..90a5f62 --- /dev/null +++ b/modules/mesh-grants/internal/grants/consumers.go @@ -0,0 +1,242 @@ +package grants + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774). The controller creates a module's +// consumer on EVENTS and each process's on its work queue when it issues the account. From the write that hands its +// lease away the grants part creates them, with the configuration the bus composer derives (buscompose.AccountConsumers), +// only where the bus lacks one; one the bus holds is never changed or deleted, a difference is only said. Before that +// write they are compared in shadow alone, so two parts never create one consumer. + +// ReadLease is the controller's lease as the part reads it (buscompose.NamedReaders names the vault and the part), and +// LeaseKey its one key. +const ( + ReadLease = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey +) + +// ConsumerCreator creates one durable consumer the bus does not hold: the part's process alone, whose account is granted +// the create of each account's consumer by stream and name. One the bus holds answers ErrHeld. +type ConsumerCreator interface { + CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error + // MatchConsumer brings a held consumer to cfg, keeping where it delivers from: only on a person's ask + // (grants_consumer_match), never by a pass. + MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error +} + +// ConsumersState is what one pass found and did of the account consumers. +type ConsumersState struct { + // On says the lease is handed away, so the part creates what the bus lacks; Why says why it is on or off. + On bool `json:"on"` + Why string `json:"why"` + // Wanted is how many the records compose; Missing those the bus lacks (`.`), Differ those it holds + // otherwise, each with how. + Wanted int `json:"wanted"` + Missing []string `json:"missing,omitempty"` + Differ []string `json:"differ,omitempty"` + Created []string `json:"created,omitempty"` + // Matched is a consumer brought to the composed configuration on a person's ask, or why not. + Matched []string `json:"matched,omitempty"` + NotCreated []string `json:"not_created,omitempty"` + // Unread is a stream whose consumers could not be read: nothing on it is compared or created. + Unread []string `json:"unread,omitempty"` +} + +// heldConsumer is one consumer as the bus module's nats_streams reads it for one stream. +type heldConsumer struct { + Name string `json:"name"` + Filters []string `json:"filters"` + MaxDeliver int `json:"max_deliver"` +} + +// heldConsumers reads the consumers the bus holds on stream, by name. +func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) { + raw, err := c.Call(CallStreams, map[string]any{"stream": stream}) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", CallStreams, stream, err) + } + var got struct { + Detail []struct { + Name string `json:"name"` + Consumers []heldConsumer `json:"consumer_detail"` + } `json:"detail"` + } + if err := json.Unmarshal(unwrap(raw), &got); err != nil { + return nil, fmt.Errorf("%s %s answered no consumers: %w", CallStreams, stream, err) + } + out := map[string]heldConsumer{} + for _, d := range got.Detail { + if d.Name != stream { + continue + } + for _, c := range d.Consumers { + out[c.Name] = c + } + } + return out, nil +} + +// leaseHandedAway reads the lease as the part's reads give it. +func leaseHandedAway(r Reader) (bool, string) { + raw, found, err := r.Get(ReadLease, LeaseKey) + return buscompose.LeaseHandedAway(raw, found, err) +} + +// consumers compares the account consumers the records compose with those the bus holds, and creates the missing ones +// once the lease is handed away and the part holds a creator. +func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { + users, err := buscompose.Users(recs) + if err != nil { + res.Errors = append(res.Errors, "the account consumers could not be composed: "+err.Error()) + return + } + st := &ConsumersState{} + st.On, st.Why = leaseHandedAway(p.Read) + res.Consumers = st + wanted := buscompose.AccountConsumers(users) + st.Wanted = len(wanted) + byStream := map[string][]buscompose.Consumer{} + for _, mc := range wanted { + byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer) + } + for _, stream := range sortedKeys(byStream) { + held, err := heldConsumers(p.Call, stream) + if err != nil { + st.Unread = append(st.Unread, stream+": "+err.Error()) + continue + } + for _, c := range byStream[stream] { + key := stream + "." + c.Name + h, ok := held[c.Name] + if ok { + why := consumerDiffers(c, h) + if why != "" && key == match && st.On && p.Consumers != nil { + if err := p.Consumers.MatchConsumer(stream, buscompose.ConsumerConfig(c)); err != nil { + st.Matched = append(st.Matched, key+": not matched: "+err.Error()) + } else { + st.Matched = append(st.Matched, key+": matched, asked by hand") + continue + } + } + if why != "" { + st.Differ = append(st.Differ, key+": "+why) + } + continue + } + if !st.On || p.Consumers == nil { + st.Missing = append(st.Missing, key) + continue + } + switch err := p.Consumers.CreateConsumer(stream, buscompose.ConsumerConfig(c)); { + case err == nil: + st.Created = append(st.Created, key) + p.say(res, EventAsserted, map[string]any{"consumer": c.Name, "stream": stream}) + case errors.Is(err, ErrHeld): + default: + st.Missing = append(st.Missing, key) + st.NotCreated = append(st.NotCreated, key+": "+err.Error()) + } + } + } + p.sayDiffer(st) +} + +// sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the +// lease write) differs from its composed configuration, naming each with the fields that differ, and clears it when they +// match. Said once per set of differences, again when it changes. Before the lease write the consumers are the +// controller's, and a difference is only said in the pass. +func (p *Part) sayDiffer(st *ConsumersState) { + if p.Say == nil { + return + } + why := "" + if st.On && len(st.Differ) > 0 { + why = strings.Join(st.Differ, "; ") + } + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.differKnown && why == p.differSaid { + return + } + c := asks.Condition{Key: ConsumerDiffersCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = asks.StateCleared + } else { + c.Severity, c.State = asks.SeverityWarning, asks.StateRaised + c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ)) + c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each" + c.Explanation = asks.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " + + "and the grants part never changes a held one by itself (its delivery position would be lost): " + why + c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise" + } + if p.Say(c) == nil { + p.differSaid, p.differKnown = why, true + } +} + +// ConsumerDiffersCondition is the key of the warning raised while an account consumer differs (ADR 0561 §6). +const ConsumerDiffersCondition = "mesh-grants.consumer-differs" + +// MatchConsumer is the tool consumer-match (ADR 0561 §6, §9): it asks the part's process to bring one account consumer +// the records compose to its composed configuration, keeping where it delivers from. Refused while the controller holds +// the lease (the consumers are the controller's then), and for a consumer the records do not compose. +func (p *Part) MatchConsumer(stream, name string) (any, error) { + if stream == "" || name == "" { + return nil, errors.New("consumer-match names a stream and a consumer") + } + if on, why := leaseHandedAway(p.Read); !on { + return nil, fmt.Errorf("not asked: %s; the controller makes and matches the consumers while it holds the lease", why) + } + in := ReadInputs(p.Read, &p.cache) + recs, _ := Records(in) + users, err := buscompose.Users(recs) + if err != nil { + return nil, err + } + found := false + for _, mc := range buscompose.AccountConsumers(users) { + if mc.Consumer.Stream == stream && mc.Consumer.Name == name { + found = true + } + } + if !found { + return nil, fmt.Errorf("the records compose no account consumer %s on %s: the part brings only those to their configuration", name, stream) + } + ask, _ := json.Marshal(Ask{By: "grants_consumer_match", At: p.now(), Consumer: stream + "." + name}) + rev, err := p.Store.Put(StateUserList, KeyAsked, ask) + if err != nil { + return nil, fmt.Errorf("the ask could not be written: %w", err) + } + return map[string]any{"asked": rev, "says": "the part's process brings " + stream + "." + name + " to its composed " + + "configuration at this ask, keeping where it delivers from; the last pass's consumers.matched says what became of it"}, nil +} + +// consumerDiffers says how a held consumer differs from the composed one in what nats_streams reads back, or "". +func consumerDiffers(c buscompose.Consumer, h heldConsumer) string { + var why []string + want, have := slices.Sorted(slices.Values(c.Filters)), slices.Sorted(slices.Values(h.Filters)) + if !slices.Equal(want, have) { + why = append(why, fmt.Sprintf("filters %v, composed %v", have, want)) + } + maxDeliver := c.MaxDeliver + if maxDeliver == 0 { + maxDeliver = -1 + } + if h.MaxDeliver != maxDeliver { + why = append(why, fmt.Sprintf("max deliver %d, composed %d", h.MaxDeliver, maxDeliver)) + } + sort.Strings(why) + return strings.Join(why, "; ") +} diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 3d19217..98ba29b 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -18,7 +19,7 @@ type Watcher interface { // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. -var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement} +var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease} // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect). @@ -28,9 +29,9 @@ type Follower struct { Logf func(string, ...any) // Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none. Asked func(wake func()) error - // Hear hears the controller seat's condition events (Consumes) and hands each, by event name, with its body, to - // handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: a machine's - // root-free word changed. + // Hear hears the controller seat's condition events and the vault's account-issued (Consumes) and hands each, by event + // name, with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: + // a machine's root-free word changed; and so does an account the vault minted (ADR 0561 §1), whose hash the list carries. Hear func(handle func(event string, body []byte)) error wake chan struct{} @@ -46,6 +47,8 @@ type Follower struct { held string notAsserted string refusedMans string + // leaseOn is whether the lease was handed away at its last change heard, once leaseSeen. + leaseOn, leaseSeen bool } func (f *Follower) init() { @@ -85,6 +88,15 @@ func (f *Follower) logf(format string, args ...any) { // heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the // bus module's loaded list moving. func (f *Follower) heard(state, key string, value []byte) bool { + if state == ReadLease { + // Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6). + on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil) + f.mu.Lock() + defer f.mu.Unlock() + was, seen := f.leaseOn, f.leaseSeen + f.leaseOn, f.leaseSeen = on, true + return !seen || was != on + } if state != ReadHave { return true } @@ -124,7 +136,7 @@ func (f *Follower) open() { } if f.Hear != nil && !f.heard_ { err := f.Hear(func(event string, body []byte) { - if IsRootNotFree(body) { + if IsAccountIssued(event) || IsRootNotFree(body) { f.Wake() } }) diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go new file mode 100644 index 0000000..17c9e93 --- /dev/null +++ b/modules/mesh-grants/internal/grants/grants.go @@ -0,0 +1,268 @@ +package grants + +import ( + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record +// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `.` in +// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the +// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key. +// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's +// (delivery.GrantSource), and the tool grants says every difference, so the switch is made on a measured equality. +// +// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has +// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the +// assigner want it, and a grant naming no account names every node). Nothing else is ever deleted, and nothing is written +// or deleted in a pass that could not read the vault's hashes, the indexer's modules or a manifest of the module. + +// GrantsWritten is what one pass did to the state grants. +type GrantsWritten struct { + Wanted int `json:"wanted"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept says why a pass wrote and deleted nothing, or left a module out. + Kept []string `json:"kept,omitempty"` + // Missing is each module the placement or an assignment puts on a machine that has no grant after the pass, + // `: `: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once + // the part's grants are the ones read, it is the condition mesh-grants.grant-missing. + Missing []string `json:"missing,omitempty"` +} + +// GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126). +const GrantMissingCondition = "mesh-grants.grant-missing" + +// MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `: `, +// sorted: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash +// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits +// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it, +// and that is the fingerprint granted). Pure. +func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []string, skip map[string]bool) []string { + placed := map[string]bool{} + for _, ms := range in.Assigned { + for _, m := range ms { + placed[m] = true + } + } + var out []string + for _, module := range sortedKeys(placed) { + fp, has := in.Fingerprints[module] + key, err := delivery.PartGrantKey(module, fp) + if has && err == nil { + if _, granted := want[key]; granted { + continue + } + } + why := "" + switch { + case skip[module]: + why = "its manifest at the indexer's trunk is refused" + case !has: + why = "no fingerprint of the new form: " + orSaid(in.NoFingerprint[module], "the indexer holds no value of it") + case err != nil: + why = fmt.Sprintf("no grant key for %s: %v", fp, err) + case slices.Contains(drop, key): + why = "none of its accounts at " + fp + " has a hash in the vault" + default: + why = "no grant composed at " + fp + } + out = append(out, module+": "+why) + } + return out +} + +func orSaid(s, none string) string { + if s == "" { + return none + } + return s +} + +// sayMissing raises mesh-grants.grant-missing while a placed module has no grant, once the controller's lease is handed +// away (the part's grants are then the ones the assigner reads), and clears it when every one has one; in shadow the +// pass and the tool say them. Said once per set, again when it changes, and always at the first pass after a start. +func (p *Part) sayMissing(missing []string) { + if p.Say == nil { + return + } + why := "" + if on, _ := leaseHandedAway(p.Read); on && len(missing) > 0 { + why = strings.Join(missing, "; ") + } + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.missingKnown && why == p.missingSaid { + return + } + c := asks.Condition{Key: GrantMissingCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = asks.StateCleared + } else { + c.Severity, c.State = asks.SeverityWarning, asks.StateRaised + c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing)) + c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none" + c.Explanation = asks.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " + + "credential rule gives them no want: " + why + c.Resolved = "every placed module has a grant at the fingerprint the indexer names" + } + if p.Say(c) == nil { + p.missingSaid, p.missingKnown = why, true + } +} + +// GrantsOf is the grant of every module at the fingerprint the indexer's trunk names for it, by key, and the keys that +// have none (drop): see the comment above. A module in skip (its manifest was refused) is neither. Pure. +func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.ControllerGrant, []string, error) { + principals, err := buscompose.Users(recs) + if err != nil { + return nil, nil, err + } + accounts, missing := map[string][]string{}, map[string]bool{} + for _, p := range principals { + if p.Module == "" || (p.Kind != buscompose.KindModule && p.Kind != buscompose.KindProcess) { + continue + } + if in.Hashes[p.Username()].Hash == "" { + missing[p.Module] = true + continue + } + accounts[p.Module] = append(accounts[p.Module], p.Username()) + } + out := map[string]delivery.ControllerGrant{} + var drop []string + for module, fp := range in.Fingerprints { + if skip[module] || !strings.HasPrefix(fp, delivery.SourcePrefix2) { + continue + } + key, err := delivery.PartGrantKey(module, fp) + if err != nil { + continue + } + names := accounts[module] + if missing[module] && len(names) == 0 { + drop = append(drop, key) + continue + } + sort.Strings(names) + out[key] = delivery.ControllerGrant{Accounts: names} + } + sort.Strings(drop) + return out, drop, nil +} + +// grantsReadFailed names a read of this pass without which no grant is written: the hashes and the indexer's modules and +// manifests. A manifest refused for one module only leaves that module out (writeGrants). +func grantsReadFailed(in Inputs) string { + for _, g := range in.Gaps { + if !g.Failed { + continue + } + for _, input := range []string{ReadAccounts, ReadModules, ReadManifests} { + if strings.HasPrefix(g.Input, input) { + return g.Input + ": " + g.Why + } + } + } + return "" +} + +// writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a +// module that has none now. +func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { + if why := grantsReadFailed(in); why != "" { + res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}} + return + } + skip := map[string]bool{} + for _, g := range in.Refused { + skip[strings.TrimPrefix(g.Input, "manifest of ")] = true + } + want, drop, err := GrantsOf(in, recs, skip) + if err != nil { + res.Errors = append(res.Errors, "the grants could not be composed: "+err.Error()) + return + } + done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)} + res.Grants = done + p.sayMissing(done.Missing) + for _, key := range sortedKeys(want) { + g := want[key] + data, err := delivery.EncodeControllerGrant(g) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("the grant %s cannot be encoded: %v", key, err)) + continue + } + if raw, _, found, err := p.Store.Get(StateGrants, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateGrants, key, err)) + continue + } else if found { + if was, err := delivery.DecodeControllerGrant(raw); err == nil && delivery.SameControllerGrant(was, withVersion(g)) { + continue + } + } + if _, err := p.Store.Put(StateGrants, key, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err)) + continue + } + done.Written = append(done.Written, key) + module, fp, _ := delivery.ParsePartGrantKey(key) + p.say(res, EventGrantWritten, map[string]any{"module": module, "fingerprint": fp, "accounts": g.Accounts}) + } + for _, key := range drop { + if _, _, found, err := p.Store.Get(StateGrants, key); err != nil || !found { + continue + } + if err := p.Store.Delete(StateGrants, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateGrants, key, err)) + continue + } + done.Deleted = append(done.Deleted, key) + } +} + +func withVersion(g delivery.ControllerGrant) delivery.ControllerGrant { + g.V = delivery.Version + return g +} + +// GrantDifference is one key on which the part's grants and the controller's differ. +type GrantDifference struct { + Key string `json:"key"` + // Kind is `only the controller's`, `only the part's` or `accounts differ`. + Kind string `json:"kind"` + Controller []string `json:"controller_accounts,omitempty"` + Part []string `json:"part_accounts,omitempty"` +} + +// CompareGrants lists every key the two sets differ on, in key order: a key held by one alone, or held by both with other +// accounts. Pure. +func CompareGrants(controller, part map[string]delivery.ControllerGrant) []GrantDifference { + keys := map[string]bool{} + for k := range controller { + keys[k] = true + } + for k := range part { + keys[k] = true + } + out := []GrantDifference{} + for _, k := range sortedKeys(keys) { + c, inC := controller[k] + g, inP := part[k] + switch { + case inC && !inP: + out = append(out, GrantDifference{Key: k, Kind: "only the controller's", Controller: c.Accounts}) + case inP && !inC: + out = append(out, GrantDifference{Key: k, Kind: "only the part's", Part: g.Accounts}) + case !delivery.SameControllerGrant(withVersion(c), withVersion(g)): + out = append(out, GrantDifference{Key: k, Kind: "accounts differ", Controller: c.Accounts, Part: g.Accounts}) + } + } + return out +} diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 68d256e..0637541 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -38,9 +38,11 @@ const ( ReadPlacement = delivery.PlacementRead ) -// Reads is every state the part reads, in the manifest's order. +// Reads is every state the part reads, in the manifest's order. ReadAssignments stays (and in Followed) only while the +// controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write +// they place nothing, and the first change after it takes them out (../assignments-readers names the part until then). var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement} + ReadAccounts, ReadPlacement, ReadLease} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -105,7 +107,7 @@ type resolved struct { type Inputs struct { // Nodes is every machine seen: in a node agent's have or a standing assignment. Nodes []string - // Assigned is each machine's standing assignments and what the catalogue's placement puts there, sorted: while both + // Assigned is each machine's standing assignments and what the seat's placement puts there, sorted: while both // stand (ADR 0549 §5) a module either names keeps its membership. Assigned map[string][]string // Placed is each node's placement as the seat mesh-placement holds it (ADR 0567 §1); empty before any is read. @@ -115,6 +117,12 @@ type Inputs struct { Manifests map[string]catalogue.Manifest // Digests is each module's manifest digest, by module. Digests map[string]string + // Fingerprints is the fingerprint of the new form the indexer's trunk value names for each module, by module: what a + // grant is written for (grants.go). + Fingerprints map[string]string + // NoFingerprint is why the indexer's value of a module names no fingerprint of the new form, by module (its none2, or + // that it names none): a module here gets no grant, and that is said (grants.go MissingGrants). + NoFingerprint map[string]string // Running is the manifest a machine is wanted to run, by `.`: the one the builder built at the // fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest. Running map[string]catalogue.Manifest @@ -138,6 +146,22 @@ type HeldHash struct { Generation int `json:"generation,omitempty"` At time.Time `json:"at"` Adopted bool `json:"adopted,omitempty"` + // Fingerprint is the vault's fingerprint of the value the hash is of: with the generation, what says the password + // changed, never the hash itself (a hash of the same password differs by its salt). + Fingerprint string `json:"fingerprint,omitempty"` + // Rotating is the vault's rotation of the account under way: a change of its hash it announces (ADR 0561 §4, §7). + Rotating *struct { + Login string `json:"login"` + } `json:"rotating,omitempty"` +} + +// credentialOf is what says which password a held hash is of, never the hash: adopted, or the vault's generation and the +// fingerprint of its value. +func credentialOf(h HeldHash) string { + if h.Adopted { + return "adopted" + } + return fmt.Sprintf("generation %d %s", h.Generation, h.Fingerprint) } // readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held. @@ -221,19 +245,23 @@ func each(r Reader, state string, gaps *[]Gap) map[string][]byte { // ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap. func ReadInputs(r Reader, cache *Manifests) Inputs { - in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, - Running: map[string]catalogue.Manifest{}, - Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)} + in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, Fingerprints: map[string]string{}, + NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{}, + Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)} nodes := map[string]bool{} + // **After the lease write the assignments place nothing** (novox/hq ADR 0549 §3, §6): they are frozen then, so a + // module the catalogue takes off a node would keep its account, grant, membership and user-list entry. Once the lease + // is handed away and the placement is read, the placement alone places modules, as the assigner's Placed does with + // the same lease reading (buscompose.LeaseHandedAway); before, or with no placement on the bus, the two are unioned. + byAssignment, busByAssignment := map[string][]string{}, "" for _, raw := range each(r, ReadAssignments, &in.Gaps) { var a assignment if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" { continue } - nodes[a.Node] = true - in.Assigned[a.Node] = append(in.Assigned[a.Node], a.Module) + byAssignment[a.Node] = append(byAssignment[a.Node], a.Module) if a.Module == BusModule { - in.BusNode = a.Node + busByAssignment = a.Node } } // The seat's placement (ADR 0567 §9): what it places is composed beside the assignments, and the seats held once @@ -257,11 +285,27 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Placed = map[string]delivery.Placement{} } in.Placed[node] = p + } + } + away, _ := leaseHandedAway(r) + if away && len(in.Placed) > 0 { + in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "not used: the controller's lease is handed away, " + + "so the seat's placement alone places modules (ADR 0549 §3, §6)"}) + } else { + for node, ms := range byAssignment { nodes[node] = true - for _, m := range p.Assigned() { - if !slices.Contains(in.Assigned[node], m) { - in.Assigned[node] = append(in.Assigned[node], m) - } + in.Assigned[node] = append(in.Assigned[node], ms...) + } + in.BusNode = busByAssignment + } + for node, p := range in.Placed { + nodes[node] = true + for _, m := range p.Assigned() { + if !slices.Contains(in.Assigned[node], m) { + in.Assigned[node] = append(in.Assigned[node], m) + } + if m == BusModule && in.BusNode == "" { + in.BusNode = node } } } @@ -322,6 +366,15 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { if err == nil && v.Manifest == "" { err = fmt.Errorf("the indexer's value names no manifest digest") } + if err == nil && strings.HasPrefix(v.FP2, delivery.SourcePrefix2) { + in.Fingerprints[module] = v.FP2 + } else if err == nil { + why := v.None2 + if why == "" { + why = "the indexer's value names no fingerprint of the new form" + } + in.NoFingerprint[module] = why + } var m catalogue.Manifest if err == nil { m, err = cache.get(r, v.Manifest) diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 9692077..11deb15 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/membership" ) @@ -23,31 +24,38 @@ const ( StateMemberships = "memberships" StateAssertions = "assertions" StateTakeover = "takeover" - KeyList = "list" - KeySwitch = "switch" + // StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go). + StateGrants = delivery.PartGrantsState + KeyList = "list" + KeySwitch = "switch" // KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList. KeyPass = "pass" KeyAsked = "asked" ) // States is every state the part keeps. -var States = []string{StateAssertions, StateMemberships, StateTakeover, StateUserList} +var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList} // The events the part says. const ( EventUserList = "user-list-changed" EventMembership = "membership-changed" + // EventGrantWritten is said when the part wrote a grant of its state grants (ADR 0547 §5, ADR 0561 §2). + EventGrantWritten = "grant-written" // EventAsserted is said when the part created a bucket, log or stream a manifest declares (issue 1790). EventAsserted = "asserted" ) // Emits is every event the part says. -var Emits = []string{EventAsserted, EventMembership, EventUserList} +var Emits = []string{EventAsserted, EventGrantWritten, EventHashesAccepted, EventMembership, EventUserList} // Store is the part's own state, by local name: the only thing the part writes. type Store interface { Get(state, key string) ([]byte, uint64, bool, error) Put(state, key string, value []byte) (uint64, error) + // Keys and Delete are for the wanted accounts (accounts.go) and the grants (grants.go): every other state is only put. + Keys(state string) ([]string, error) + Delete(state, key string) error } // Emitter says one of the part's events. @@ -111,9 +119,15 @@ type Result struct { // free, `: a verified sender is composed as from no root-free machine there (issue 1945). RootNotFree []string `json:"root_not_free,omitempty"` // Sealed is the list sealed to the bus node as the key `user-list.sealed` names it (ADR 0547 clause 7). - Sealed *SealedState `json:"sealed,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + Sealed *SealedState `json:"sealed,omitempty"` + // Consumers is what the pass found and created of the durable consumers an account needs (ADR 0561 §6). + Consumers *ConsumersState `json:"consumers,omitempty"` + // Accounts is what the pass wrote of the wanted accounts the vault mints from (ADR 0561 §2). + Accounts *AccountsWritten `json:"accounts,omitempty"` + // Grants is what the pass wrote of its grants (grants.go). + Grants *GrantsWritten `json:"grants,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -136,6 +150,12 @@ type Part struct { Logf func(string, ...any) // Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. Objects Objects + // Consumers creates an account's durable consumer from the lease write (consumers.go): the process's connection; + // nil in the tools, which compare only. + Consumers ConsumerCreator + // Bus is where the nodes reach the bus, which each wanted account carries (accounts.go): the process's own credential's; + // empty in the tools, which write no account. + Bus BusAddress // Say says a condition to the operator: the process's; nil in the tools. Say func(asks.Condition) error // Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix. @@ -157,6 +177,14 @@ type Part struct { holdMu sync.Mutex heldSaid string heldKnown bool + // differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go). + differSaid string + differKnown bool + // missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go). + missingSaid string + missingKnown bool + // hashHold are the users whose hash changes held the pass's list, or nil (sealed.go). + hashHold []string } // Ask is the value of KeyAsked: a tool's ask to compose now, and, from grants_assert, to create one module's missing @@ -165,18 +193,21 @@ type Ask struct { By string `json:"by"` At time.Time `json:"at"` Assert string `json:"assert,omitempty"` + // Consumer is, from grants_consumer_match, one account consumer `.` to bring to the composed + // configuration by hand (ADR 0561 §6). + Consumer string `json:"consumer,omitempty"` } -// askedAssert is the module a new ask names to assert by hand, read once per ask; empty when none. -func (p *Part) askedAssert() string { +// asked is a new ask, read once per ask; the zero Ask when none. +func (p *Part) asked() Ask { raw, rev, found, err := p.Store.Get(StateUserList, KeyAsked) if err != nil || !found || rev == p.askSeen { - return "" + return Ask{} } p.askSeen = rev var a Ask _ = json.Unmarshal(raw, &a) - return a.Assert + return a } func (p *Part) now() time.Time { @@ -240,6 +271,14 @@ func (p *Part) Compose() Result { } } } + // The accounts the vault mints from, said in the state only it reads (ADR 0561 §2). + if p.Store != nil { + p.writeAccounts(&res, in, recs) + } + // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5). + if p.Store != nil { + p.writeGrants(&res, in, recs) + } // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) @@ -274,10 +313,11 @@ func (p *Part) Compose() Result { } } held, heldErr := HeldStreams(p.Call) - forced := "" - if p.Create != nil { - forced = p.askedAssert() + var ask Ask + if p.Create != nil || p.Consumers != nil { + ask = p.asked() } + forced := ask.Assert for _, module := range sortedKeys(in.Manifests) { if !assigned(in, module) { continue @@ -299,6 +339,7 @@ func (p *Part) Compose() Result { res.AssertionsWritten++ } } + p.consumers(&res, recs, ask.Consumer) if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found { data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}}) if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil { @@ -324,6 +365,11 @@ func (p *Part) Last() Result { } func assigned(in Inputs, module string) bool { + // The node-engine runs on every node though no node is assigned it (novox/hq ADR 0563 §2, §3): its manifest is + // placed everywhere, so its declared state is asserted and created as a placed module's. + if module == buscompose.EngineModule { + return true + } for _, ms := range in.Assigned { for _, m := range ms { if m == module { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index bf479b2..d07c7ae 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -78,6 +78,11 @@ func Records(in Inputs) (buscompose.Records, []Gap) { } } } + // The node-engine's own manifest, placed on every node (ADR 0563): its state, logs and streams alone, created by this + // part; it adds no principal, since each node's engine speaks as the node's account. + if m, ok := in.Manifests[buscompose.EngineModule]; ok { + out.Engine = &buscompose.Declared{Module: m.Module, State: bucketsOf(m), Logs: logsOf(m), Streams: streamsOf(m)} + } return out, gaps } diff --git a/modules/mesh-grants/internal/grants/rootfree.go b/modules/mesh-grants/internal/grants/rootfree.go index b4e1223..614bc4b 100644 --- a/modules/mesh-grants/internal/grants/rootfree.go +++ b/modules/mesh-grants/internal/grants/rootfree.go @@ -23,8 +23,17 @@ const ( KindRootNotFree = "root-not-free" ) +// EventAccountIssued is the vault's word that it minted an account (novox/hq ADR 0561 §1): its hash is in the vault's +// bus-accounts, so a pass composes the list with it. +const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent + // Consumes is every event the part hears, as its manifest's consumes name them. -var Consumes = []string{EventConditionRaised, EventConditionCleared} +var Consumes = []string{EventConditionRaised, EventConditionCleared, EventAccountIssued} + +// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject. +func IsAccountIssued(event string) bool { + return event == EventAccountIssued || event == buscompose.AccountIssuedSubject +} // EventSubject is the bus subject of a controller seat event named `mesh-controller.`. func EventSubject(event string) string { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index eb76a44..55ff811 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -2,10 +2,12 @@ package grants import ( "encoding/json" + "errors" "fmt" "regexp" "slices" "strings" + "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -91,11 +93,12 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string why = "no node runs the bus's module, so the list is sealed to nobody" case key == "": why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" - case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost) != "": + case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "": // After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user // of it for want of a hash: that user's module would be locked out of the bus (review of the switch). - why = p.holdsBack(in, leftOut, bus, lost) + why = p.holdsBack(in, leftOut, bus, lost, digest) held = why + p.keepHashHold(res, digest) case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: object, err := delivery.SealUserList(list, key) if err == nil { @@ -133,7 +136,8 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string return } if next.Digest == digest && cur.Digest != digest { - p.keepWrittenUsers(res, list) + p.keepWrittenUsers(res, list, in.Hashes) + p.acceptedWritten(res, digest) } // The object the key named before is taken away once the key names the new one: the bucket keeps the list in force. if had && cur.Object != "" && cur.Object != next.Object { @@ -288,11 +292,50 @@ func (p *Part) followReasons(res Result, in Inputs) []string { // composed after take-over must keep. const KeyWrittenUsers = "sealed-users" -func (p *Part) keepWrittenUsers(res *Result, list []byte) { - data, _ := json.Marshal(listUsers(list)) +// KeyWrittenCredentials is the key of the state user-list naming, per user of the list in force, which password its hash +// is of (credentialOf: adopted, or the vault's generation and fingerprint), never the hash: what tells a list that changes +// several loaded users' hashes at once (ADR 0561 §7). +const KeyWrittenCredentials = "sealed-credentials" + +func (p *Part) keepWrittenUsers(res *Result, list []byte, hashes map[string]HeldHash) { + users := listUsers(list) + data, _ := json.Marshal(users) if _, err := p.Store.Put(StateUserList, KeyWrittenUsers, data); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenUsers, err)) } + creds := map[string]string{} + for _, u := range users { + if h, ok := hashes[u]; ok { + creds[u] = credentialOf(h) + } + } + data, _ = json.Marshal(creds) + if _, err := p.Store.Put(StateUserList, KeyWrittenCredentials, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenCredentials, err)) + } +} + +// changedHashes are the users of the list in force whose hash the list composed now would change with no rotation by the +// vault naming them, sorted; none when the part kept no credentials of the list in force (a list written before). +func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { + raw, _, found, err := p.Store.Get(StateUserList, KeyWrittenCredentials) + if err != nil || !found { + return nil, err + } + var had map[string]string + if err := json.Unmarshal(raw, &had); err != nil { + return nil, err + } + var changed []string + for user, was := range had { + h, ok := hashes[user] + if !ok || credentialOf(h) == was || h.Rotating != nil { + continue + } + changed = append(changed, user) + } + slices.Sort(changed) + return changed, nil } var listUser = regexp.MustCompile(`\{\s*user:\s*"([^"]+)"`) @@ -312,7 +355,8 @@ func listUsers(list []byte) []string { // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when // they could not be read). Empty when it may. -func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string { +func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string { + p.hashHold = nil if f := in.ReadFailures(); len(f) > 0 { return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")" } @@ -342,5 +386,127 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string { if len(unhashed) > 0 { return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault" } + // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new + // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. + changed, err := p.changedHashes(in.Hashes) + if err != nil { + return "held: the credentials of the list in force cannot be read: " + err.Error() + } + if len(changed) > 1 && !p.accepted(digest, changed) { + p.hashHold = changed + return "held: the list in force is kept, since the hashes of " + fmt.Sprint(len(changed)) + " loaded users would change at " + + "once with no rotation by the vault naming them: " + strings.Join(changed, ", ") + } return "" } + +// **A person accepts a held list's hash changes** (ADR 0561 §7, §9; tools-are-how-a-module-is-spoken-to): the hold of +// several loaded users' hashes changing at once is released only by grants_accept_hashes, naming exactly the users held, +// with why, bound to the held list's digest. The acceptance releases this guard alone, once: the other holds (a failed +// read, a user dropped for want of a hash, a lost right) stand whatever it says; and a list composed otherwise since is +// held again, for a new acceptance. + +// KeyHashHold names the list held for its hash changes (its digest and the users), and KeyHashAccept a person's acceptance. +const ( + KeyHashHold = "hash-hold" + KeyHashAccept = "hash-accept" +) + +// EventHashesAccepted is said when a list held for its hash changes is written on a person's acceptance: a hand act. +const EventHashesAccepted = "hashes-accepted" + +// HashHold is the value of KeyHashHold, and HashAccept of KeyHashAccept. +type HashHold struct { + Digest string `json:"digest"` + Users []string `json:"users"` + At time.Time `json:"at"` +} + +type HashAccept struct { + Digest string `json:"digest"` + Users []string `json:"users"` + By string `json:"by"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// keepHashHold writes which list is held for its hash changes, when that is the hold. +func (p *Part) keepHashHold(res *Result, digest string) { + if p.hashHold == nil { + return + } + data, _ := json.Marshal(HashHold{Digest: digest, Users: p.hashHold, At: p.now()}) + if raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold); err == nil && found { + var was HashHold + if json.Unmarshal(raw, &was) == nil && was.Digest == digest && slices.Equal(was.Users, p.hashHold) { + return + } + } + if _, err := p.Store.Put(StateUserList, KeyHashHold, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyHashHold, err)) + } +} + +// accepted says a person accepted exactly these users' hash changes of the list of this digest. +func (p *Part) accepted(digest string, users []string) bool { + raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept) + if err != nil || !found { + return false + } + var a HashAccept + return json.Unmarshal(raw, &a) == nil && a.Digest == digest && slices.Equal(a.Users, users) +} + +// acceptedWritten says, once, that the list a person accepted is written, and forgets the acceptance and the hold. +func (p *Part) acceptedWritten(res *Result, digest string) { + raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept) + if err != nil || !found { + return + } + var a HashAccept + if json.Unmarshal(raw, &a) != nil || a.Digest != digest { + return + } + p.say(res, EventHashesAccepted, map[string]any{"digest": digest, "users": a.Users, "by": a.By, "why": a.Why}) + for _, k := range []string{KeyHashAccept, KeyHashHold} { + if err := p.Store.Delete(StateUserList, k); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, k, err)) + } + } +} + +// AcceptHashes is the tool accept-hashes: a person accepts the hash changes of the list held for them, naming exactly its +// users, with why. Refused when no list is held for its hashes, or for another set of users. Bound to the held list's +// digest: a list composed otherwise since is held again. +func (p *Part) AcceptHashes(caller string, users []string, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("accept-hashes says why (`why`)") + } + raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold) + if err != nil { + return nil, fmt.Errorf("the held list cannot be read: %w", err) + } + var h HashHold + if !found || json.Unmarshal(raw, &h) != nil || h.Digest == "" { + return nil, errors.New("no list is held for its users' hashes changing: there is nothing to accept") + } + named := slices.Sorted(slices.Values(users)) + if !slices.Equal(named, h.Users) { + return nil, fmt.Errorf("the list held (%s) changes the hashes of %s: an acceptance names exactly those, and %s is not that", + short(h.Digest), strings.Join(h.Users, ", "), strings.Join(named, ", ")) + } + data, _ := json.Marshal(HashAccept{Digest: h.Digest, Users: h.Users, By: someoneOr(caller), Why: strings.TrimSpace(why), At: p.now()}) + rev, err := p.Store.Put(StateUserList, KeyHashAccept, data) + if err != nil { + return nil, fmt.Errorf("the acceptance could not be written: %w", err) + } + return map[string]any{"accepted": rev, "digest": h.Digest, "users": h.Users, "says": "the part's process writes the held " + + "list at its next pass, once, unless another hold stands or the list composed then is another; it says hashes-accepted"}, nil +} + +func someoneOr(caller string) string { + if caller == "" { + return "a caller the runtime did not prove" + } + return caller +} diff --git a/modules/mesh-grants/internal/grants/tools.go b/modules/mesh-grants/internal/grants/tools.go index 09700dc..defb2f5 100644 --- a/modules/mesh-grants/internal/grants/tools.go +++ b/modules/mesh-grants/internal/grants/tools.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -21,43 +20,92 @@ const ( ToolTakeOver = "grants_take_over" ToolTakeOverState = "grants_take_over_state" ToolHandBack = "grants_hand_back" + ToolAccounts = "grants_accounts" + ToolConsumerMatch = "grants_consumer_match" + ToolAcceptHashes = "grants_accept_hashes" ) // Tools is every tool, in the manifest's order. -var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState} +var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState, + ToolAccounts, ToolConsumerMatch, ToolAcceptHashes} -// WritesNoGrant is what grants says of the part's own grants in this milestone. -const WritesNoGrant = "none: the grants part writes no grants.. before ADR 0547's third milestone (clause 5); the controller's are beside" +// WritesGrants is what grants says of the part's own grants in this milestone: written, and read by nobody until the +// controller's lease is handed away (delivery.GrantSource). +const WritesGrants = "in shadow: the grants part writes grants.. into its state grants, which the assigner reads only once the controller's lease is handed away (ADR 0547 clause 5); the controller's are beside, with every difference" -// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all. +// Grants answers the tool grants: the grants the part wrote, the controller's, and every difference between them, one +// module's or all. func (p *Part) Grants(module string) (any, error) { - keys, err := p.Read.Keys(ReadControllerGrant) - if err != nil { - return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil - } - type grant struct { - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Accounts []string `json:"accounts,omitempty"` - } - out := []grant{} - for _, k := range keys { - m, fp, err := delivery.ParseControllerGrantKey(k) - if err != nil || (module != "" && m != module) { - continue + out := map[string]any{"writes": WritesGrants} + read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.ControllerGrant, error) { + keys, one, err := get() + if err != nil { + return nil, err } - g := grant{Module: m, Fingerprint: fp} - if module != "" { - if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found { - if v, err := delivery.DecodeControllerGrant(raw); err == nil { - g.Accounts = v.Accounts - } + m := map[string]delivery.ControllerGrant{} + for _, k := range keys { + mod, _, err := delivery.ParseControllerGrantKey(k) + if err != nil || (module != "" && mod != module) { + continue } + raw, found, err := one(k) + if err != nil { + return nil, err + } + if !found { + continue + } + g, err := delivery.DecodeControllerGrant(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + m[k] = g } - out = append(out, g) + return m, nil } - sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint }) - return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil + controller, cerr := read(func() ([]string, func(string) ([]byte, bool, error), error) { + keys, err := p.Read.Keys(ReadControllerGrant) + return keys, func(k string) ([]byte, bool, error) { return p.Read.Get(ReadControllerGrant, k) }, err + }) + mine, perr := read(func() ([]string, func(string) ([]byte, bool, error), error) { + keys, err := p.Store.Keys(StateGrants) + return keys, func(k string) ([]byte, bool, error) { + raw, _, found, err := p.Store.Get(StateGrants, k) + return raw, found, err + }, err + }) + type grant struct { + Key string `json:"key"` + Module string `json:"module"` + Accounts []string `json:"accounts,omitempty"` + } + list := func(m map[string]delivery.ControllerGrant) []grant { + out := []grant{} + for _, k := range sortedKeys(m) { + mod, _, _ := delivery.ParseControllerGrantKey(k) + out = append(out, grant{Key: k, Module: mod, Accounts: m[k].Accounts}) + } + return out + } + if cerr != nil { + out["controller_unread"] = cerr.Error() + } else { + out["controller"], out["controller_count"] = list(controller), len(controller) + } + if perr != nil { + out["written_unread"] = perr.Error() + } else { + out["written"], out["count"] = list(mine), len(mine) + } + if cerr == nil && perr == nil { + diff := CompareGrants(controller, mine) + out["differences"], out["differing"] = diff, len(diff) + out["equal"] = len(diff) == 0 + } + if last := p.Last(); last.Grants != nil { + out["last_pass"] = last.Grants + } + return out, nil } // MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 785008e..4417453 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -12,11 +12,13 @@ "mesh-delivery-indexer.modules", "mesh-delivery-assigner.placement", "node-engine.have", - "mesh-vault.bus-accounts" + "mesh-vault.bus-accounts", + "mesh-controller.lease" ], "consumes": [ "mesh-controller.condition-raised", - "mesh-controller.condition-cleared" + "mesh-controller.condition-cleared", + "mesh-vault.account-issued" ], "uses": [ "operator-channel" @@ -29,6 +31,8 @@ ], "emits": [ "asserted", + "grant-written", + "hashes-accepted", "membership-changed", "user-list-changed" ], @@ -40,13 +44,24 @@ "grants_assert", "grants_take_over", "grants_take_over_state", - "grants_hand_back" + "grants_hand_back", + "grants_accounts", + "grants_consumer_match", + "grants_accept_hashes" ], "state": [ + { + "name": "accounts", + "history": 5 + }, { "name": "assertions", "history": 5 }, + { + "name": "grants", + "history": 5 + }, { "name": "memberships", "history": 5 From 98ad1aa252a22d4b26671e1677b4a8077802690a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:07:00 +0100 Subject: [PATCH 02/35] Pin mesh-sdk 22c9ddd in the modules L3 changes, the L3 re-land group's sdk head (issue 774) --- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 ++-- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 ++-- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 0f8f0e9..e050a8b 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028225633-76f7c4d2238f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 235c60c..043e05f 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028225633-76f7c4d2238f h1:jCd8WN+Olo4nlaB1cz1HYpEV4xc6Mzw7viXdwU+0fic= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028225633-76f7c4d2238f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 6dd30b0..1031a9e 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index c82187f..043e05f 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index e4af0f0..b431a85 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index c82187f..043e05f 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index f4410c0..96d5ca6 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index c82187f..043e05f 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index d405399..537359b 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index c82187f..043e05f 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index a07cd50..d73ed08 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 61e5c63..52700cd 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c h1:hcqOCd5q3M0ZiEzB6MhwxT/AOKkm9gaskKqPcpkAAg8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028205404-8eafc0c5ce3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From c7fbc530daaac9aa6d9ba851844609e9c5443060 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:15:55 +0100 Subject: [PATCH 03/35] mesh-grants: compose the user list from the running and the trunk manifest, and write a grant only once the loaded list carries it, so L3's parts never use a subject the bus has not loaded (issue 774) --- modules/mesh-grants/grants_test.go | 63 ++++++++++++-- modules/mesh-grants/internal/grants/grants.go | 35 +++++++- modules/mesh-grants/internal/grants/part.go | 32 +++++--- .../mesh-grants/internal/grants/records.go | 82 ++++++++++++++++--- 4 files changed, 185 insertions(+), 27 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 694a03e..81dfbe9 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -481,8 +481,8 @@ func TestGrants(t *testing.T) { []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"}, {"a manifest missing is a gap and its user is left out", func(b *fakeBus) { delete(b.states[grants.ReadModules], "mod-y") }, []string{"a.mod-x", "a.nats", "controller", "node.a", "node.b"}, "manifest of mod-y"}, - {"a want's built manifest is composed, not the trunk's", func(b *fakeBus) { - built := strings.Replace(xManifest, `"history":3}]`, `"history":3},{"name":"newer"}]`, 1) + {"a want's built manifest is composed, not the trunk's; the list's records carry both (issue 774)", func(b *fakeBus) { + built := strings.Replace(xManifest, `{"name":"kept","history":3}`, `{"name":"newer"}`, 1) digest := delivery.ManifestDigest([]byte(built)) b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "mod-x", JSON: built}) fp := "src2:" + strings.Repeat("cd", 32) @@ -510,8 +510,22 @@ func TestGrants(t *testing.T) { } if strings.HasPrefix(row.name, "a want's") { x := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.mod-x" })] - if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") { - t.Errorf("a.mod-x is not granted the state its wanted build declares: %v", x.Publish) + if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") || slices.Contains(x.Publish, "$KV.mod-x_kept.>") { + t.Errorf("a.mod-x is not granted the state its wanted build declares alone: %v", x.Publish) + } + // The list composes what runs and what the trunk names: a subject only the trunk's manifest names is in + // it before the want moves to the trunk's build (issue 774). + listRecs, _ := grants.ListRecords(in) + listed, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + if got := names(listed); !slices.Equal(got, row.users) { + t.Errorf("the list's users = %v, want %v", got, row.users) + } + lx := listed[slices.IndexFunc(listed, func(u grants.User) bool { return u.User == "a.mod-x" })] + if !slices.Contains(lx.Publish, "$KV.mod-x_newer.>") || !slices.Contains(lx.Publish, "$KV.mod-x_kept.>") { + t.Errorf("the list does not grant a.mod-x both its running and its trunk manifest's state: %v", lx.Publish) } } }) @@ -1921,7 +1935,7 @@ func TestGrants(t *testing.T) { // 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.. in shadow, once each and only for a // module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no // account with a hash and every read was made, and the tool says every difference from the controller's. - t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) { + t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's; after take-over once the loaded list carries them", func(t *testing.T) { b := fixture() b.loaded = shadowOf(t, b) p := partOn(b) @@ -1979,6 +1993,45 @@ func TestGrants(t *testing.T) { if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 { t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held()) } + + // Issue 774: after take-over a new grant is written only once the list the bus loaded carries it. mod-x moves to a + // new trunk fingerprint while the list is held (users of the loaded list have no hash): it waits, said per key. + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + loads := func(source, digest string) { + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey}}) + } + loads("a.nats.src2", "sha256:aa") + p.Objects = &fakeObjects{} + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + fp2 := "src2:" + strings.Repeat("cd", 32) + key2, _ := delivery.PartGrantKey("mod-x", fp2) + b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp2, Manifest: delivery.ManifestDigest([]byte(xManifest)), Repository: "novox/x", Commit: "c"}) + res = p.Compose() + if _, written := b.own[grants.StateGrants][key2]; written || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") || + len(res.Grants.Waits) != 1 || !strings.HasPrefix(res.Grants.Waits[0], key2+": waits for the bus to load") { + t.Fatalf("a held list let a new grant through: sealed %+v, grants %+v", res.Sealed, res.Grants) + } + // Every user hashed: the list is sealed, and the grants still wait while the bus has another list loaded. + for _, u := range shadowOf(t, b) { + hash(u.User) + } + res = p.Compose() + sealed, _, _ := p.Sealed() + if _, written := b.own[grants.StateGrants][key2]; written || sealed.Digest == "" || len(res.Grants.Waits) != 2 { + t.Fatalf("a grant was written before the bus loaded the list carrying it: sealed %+v, grants %+v", sealed, res.Grants) + } + // The bus loads the sealed list: both written, nothing waits. + loads(delivery.GrantsUserListSource, sealed.Digest) + if res := p.Compose(); len(res.Grants.Waits) != 0 || !slices.Equal(res.Grants.Written, []string{key2, key("mod-y")}) { + t.Errorf("once the bus loaded the list carrying them: %+v", res.Grants) + } }) // Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go index 17c9e93..18c6fc0 100644 --- a/modules/mesh-grants/internal/grants/grants.go +++ b/modules/mesh-grants/internal/grants/grants.go @@ -34,6 +34,34 @@ type GrantsWritten struct { // `: `: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once // the part's grants are the ones read, it is the condition mesh-grants.grant-missing. Missing []string `json:"missing,omitempty"` + // Waits is each grant composed and not written, `: `: after take-over a grant is new or changed only once the + // list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass). + Waits []string `json:"waits,omitempty"` +} + +// GrantWaits says why a grant composed now, new or changed, is not written yet, or "" when it may be (issue 774). The +// assigner moves a module's want to the fingerprint once its grant is there, so a grant is the word that the bus's user list +// in force already carries the module's subjects at that fingerprint. After take-over that is so when the list the bus +// node says it loaded is the grants part's own and is the list composed now (composed: its digest), which ListRecords +// composes from what runs and what the trunk names; a list held, not yet sealed, or not yet loaded is not it. In shadow +// nobody reads the part's grants and the controller's list is in force, so nothing waits. Pure. +func GrantWaits(tookOver bool, loaded *delivery.UserListWritten, composed string) string { + switch { + case !tookOver: + return "" + case composed == "": + return "waits: the user list could not be composed, so no list carries it" + case loaded == nil: + return "waits for the bus node to say which user list it loaded" + case loaded.Source != delivery.GrantsUserListSource: + return "waits for the bus to load the grants part's list " + short(composed) + ": it loaded " + loaded.Source + "'s" + case loaded.Error != "": + return "waits for the bus to load the list " + short(composed) + ": its node could not write it: " + loaded.Error + case loaded.Digest != composed: + return "waits for the bus to load the list " + short(composed) + ", which carries it: it has " + short(loaded.Digest) + + " loaded (a list held, or sealed and not yet loaded)" + } + return "" } // GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126). @@ -175,7 +203,7 @@ func grantsReadFailed(in Inputs) string { // writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a // module that has none now. -func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { +func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed string) { if why := grantsReadFailed(in); why != "" { res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}} return @@ -192,6 +220,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)} res.Grants = done p.sayMissing(done.Missing) + waits := GrantWaits(p.SwitchState() == TookOver, in.Loaded, composed) for _, key := range sortedKeys(want) { g := want[key] data, err := delivery.EncodeControllerGrant(g) @@ -207,6 +236,10 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { continue } } + if waits != "" { + done.Waits = append(done.Waits, key+": "+waits) + continue + } if _, err := p.Store.Put(StateGrants, key, data); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err)) continue diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 11deb15..1e306be 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -233,22 +233,26 @@ func (p *Part) Compose() Result { for _, g := range in.Refused { res.Refused = append(res.Refused, g.Input+": "+g.Why) } - recs, left := Records(in) + // recs say what runs (memberships, consumers); listRecs what the user list and the grants carry: what runs and what + // the trunk names, so the list grants a build's subjects before the want moves to it (issue 774). + recs, _ := Records(in) + listRecs, left := ListRecords(in) holdings := HoldingsOf(in) - res.RootNotFree = WithRootFree(&recs, p.Call) + res.RootNotFree = WithRootFree(&listRecs, p.Call) + recs.RootFree = listRecs.RootFree st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode, Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()} if st.LeftOut == nil { st.LeftOut = []Gap{} } - users, err := ComposeUsers(recs) + users, err := ComposeUsers(listRecs) if err != nil { res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error()) } users = WithHashes(users, in.Hashes) st.Digest, st.Users = Digest(users), len(users) loaded, err := LoadedUsers(p.Call) - st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(recs))) + st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) if err != nil { st.Compare.Unread = err.Error() } @@ -273,14 +277,20 @@ func (p *Part) Compose() Result { } // The accounts the vault mints from, said in the state only it reads (ADR 0561 §2). if p.Store != nil { - p.writeAccounts(&res, in, recs) + p.writeAccounts(&res, in, listRecs) } - // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5). + list, n, leftOut, err := ComposeList(listRecs, in.Hashes) + composed := "" + if err == nil { + composed = delivery.UserListDigest(list) + } + // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over + // each new one only once the bus loaded a list that carries it (issue 774). if p.Store != nil { - p.writeGrants(&res, in, recs) + p.writeGrants(&res, in, listRecs, composed) } // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). - if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { + if err != nil { res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) } else { // The users the bus loaded: what a first list after take-over must keep where the part has not yet written one. @@ -293,11 +303,11 @@ func (p *Part) Compose() Result { } ms := Memberships(recs) res.Memberships = len(ms) - composed := make([]membership.Membership, 0, len(ms)) + members := make([]membership.Membership, 0, len(ms)) for _, m := range ms { - composed = append(composed, m.Membership) + members = append(members, m.Membership) } - seated := holdings.ByMemberships(composed...) + seated := holdings.ByMemberships(members...) for _, key := range sortedKeys(ms) { m, now := publish(p.Publish, ms[key], holdings, seated) switch { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index d07c7ae..a0e575f 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -2,6 +2,7 @@ package grants import ( "fmt" + "reflect" "slices" "strings" @@ -14,7 +15,15 @@ import ( // BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest // the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow // composes what it can and says what it could not. -func Records(in Inputs) (buscompose.Records, []Gap) { +func Records(in Inputs) (buscompose.Records, []Gap) { return records(in, false) } + +// ListRecords is Records for the user list and the grants (issue 774): a module whose wanted build's manifest is not the +// trunk's is declared with what both grant (unionDeclared), so the list carries the trunk manifest's subjects before the +// assigner moves the want to it, and a grant at the trunk's fingerprint waits for nothing the want's move would bring. +// Memberships and consumers stay with Records: they say what runs. Pure. +func ListRecords(in Inputs) (buscompose.Records, []Gap) { return records(in, true) } + +func records(in Inputs, union bool) (buscompose.Records, []Gap) { var gaps []Gap seats := map[string]catalogue.SeatDeclaration{} declarers := map[string]string{} @@ -59,21 +68,29 @@ func Records(in Inputs) (buscompose.Records, []Gap) { gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"}) continue } - d := declaredFor(m, seats, declarers) + declare := func(m catalogue.Manifest) buscompose.Declared { + d := declaredFor(m, seats, declarers) + if len(in.Placed) > 0 { + d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + } + for _, sr := range catalogue.ReadsGranted(m, here, node) { + d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) + } + d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) + return d + } + d := declare(m) + trunk, onTrunk := in.Manifests[module] // What a node composing a consumer reads of it is the trunk manifest's served keys (frombus reads the indexer's // manifest of another node's module), and what runs may still be another build: both are granted (issue 1126). - if trunk, ok := in.Manifests[module]; ok { + if onTrunk { d.Served = slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(d.Served), catalogue.ServedSettings(trunk)...)))) } - if len(in.Placed) > 0 { - d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + if union && onTrunk && !reflect.DeepEqual(m, trunk) { + d = unionDeclared(d, declare(trunk)) } - for _, sr := range catalogue.ReadsGranted(m, here, node) { - d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) - } - d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) out.Assigned[node] = append(out.Assigned[node], d) - if m.Instances == catalogue.InstancesInterchangeable { + if m.Instances == catalogue.InstancesInterchangeable || union && onTrunk && trunk.Instances == catalogue.InstancesInterchangeable { out.Interchangeable[m.Module] = true } } @@ -356,3 +373,48 @@ func sameSeat(a, b string) bool { sb, okb := catalogue.SeatNamed(b) return oka && okb && sa.Name == sb.Name } + +// unionDeclared is what either declaration grants (issue 774): every list of b's entries a lacks appended to a's, a +// process of both merged by its name, a flag that adds a grant set where either sets it, and an account kept where +// either has one (NoAccount only where both say none). a's names and RunsAs stand. Pure. +func unionDeclared(a, b buscompose.Declared) buscompose.Declared { + out := a + unionFields(reflect.ValueOf(&out).Elem(), reflect.ValueOf(b)) + out.NoAccount = a.NoAccount && b.NoAccount + procs := slices.Clone(a.Processes) + for _, bp := range b.Processes { + i := slices.IndexFunc(procs, func(ap buscompose.ProcessPrincipal) bool { return ap.Process == bp.Process }) + if i < 0 { + procs = append(procs, bp) + continue + } + merged := procs[i] + unionFields(reflect.ValueOf(&merged).Elem(), reflect.ValueOf(bp)) + procs[i] = merged + } + out.Processes = procs + return out +} + +// unionFields appends to each slice field of dst the elements of src's it lacks, and sets each bool field src sets. +func unionFields(dst, src reflect.Value) { + for i := 0; i < dst.NumField(); i++ { + f, g := dst.Field(i), src.Field(i) + switch f.Kind() { + case reflect.Slice: + merged := reflect.AppendSlice(reflect.MakeSlice(f.Type(), 0, f.Len()+g.Len()), f) + for j := 0; j < g.Len(); j++ { + held := false + for k := 0; k < merged.Len() && !held; k++ { + held = reflect.DeepEqual(merged.Index(k).Interface(), g.Index(j).Interface()) + } + if !held { + merged = reflect.Append(merged, g.Index(j)) + } + } + f.Set(merged) + case reflect.Bool: + f.SetBool(f.Bool() || g.Bool()) + } + } +} From 013b7420f6f0b94799cfdf6e94e67ec79ec774ba Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:18:21 +0100 Subject: [PATCH 04/35] mesh-grants: hold a grant only while the loaded list lacks its permissions, read from the part's own record of the lists it sealed, so a release whose bus tools are down still moves (issue 774) --- modules/mesh-grants/grants_test.go | 64 +++++--- modules/mesh-grants/internal/grants/grants.go | 144 +++++++++++++++--- modules/mesh-grants/internal/grants/part.go | 17 +-- modules/mesh-grants/internal/grants/sealed.go | 3 +- 4 files changed, 177 insertions(+), 51 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 81dfbe9..f6391bc 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -1994,15 +1994,17 @@ func TestGrants(t *testing.T) { t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held()) } - // Issue 774: after take-over a new grant is written only once the list the bus loaded carries it. mod-x moves to a - // new trunk fingerprint while the list is held (users of the loaded list have no hash): it waits, said per key. + // Issue 774: after take-over a grant is written at once when the list the bus loaded carries its accounts' permissions, + // held list or not, and waits only when it adds one. (1) mod-x moves to a new fingerprint with the same permissions + // while the list is held (users of the loaded list have no hash): written. busKey, err := seal.GenerateKeyPair() if err != nil { t.Fatal(err) } loads := func(source, digest string) { b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, - UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey}}) + UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey, + Follows: delivery.FollowGrants}}) } loads("a.nats.src2", "sha256:aa") p.Objects = &fakeObjects{} @@ -2010,27 +2012,55 @@ func TestGrants(t *testing.T) { if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { t.Fatal(err) } - fp2 := "src2:" + strings.Repeat("cd", 32) - key2, _ := delivery.PartGrantKey("mod-x", fp2) - b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp2, Manifest: delivery.ManifestDigest([]byte(xManifest)), Repository: "novox/x", Commit: "c"}) - res = p.Compose() - if _, written := b.own[grants.StateGrants][key2]; written || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") || - len(res.Grants.Waits) != 1 || !strings.HasPrefix(res.Grants.Waits[0], key2+": waits for the bus to load") { - t.Fatalf("a held list let a new grant through: sealed %+v, grants %+v", res.Sealed, res.Grants) + moves := func(module, manifest, fp string) string { + digest := delivery.ManifestDigest([]byte(manifest)) + b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: manifest}) + b.put(grants.ReadModules, module, delivery.Module{V: delivery.Version, FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + k, _ := delivery.PartGrantKey(module, fp) + return k } - // Every user hashed: the list is sealed, and the grants still wait while the bus has another list loaded. + written := func(k string) bool { _, ok := b.own[grants.StateGrants][k]; return ok } + keyX2 := moves("mod-x", xManifest, "src2:"+strings.Repeat("cd", 32)) + res = p.Compose() + if !written(keyX2) || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") || len(res.Grants.Waits) != 0 { + t.Fatalf("a new fingerprint with the same permissions under a held list: sealed %+v, grants %+v", res.Sealed, res.Grants) + } + // Every user hashed: the list is sealed, and the part keeps its permissions. The bus loads it, and its tools go quiet. for _, u := range shadowOf(t, b) { hash(u.User) } - res = p.Compose() + p.Compose() sealed, _, _ := p.Sealed() - if _, written := b.own[grants.StateGrants][key2]; written || sealed.Digest == "" || len(res.Grants.Waits) != 2 { - t.Fatalf("a grant was written before the bus loaded the list carrying it: sealed %+v, grants %+v", sealed, res.Grants) + if sealed.Digest == "" { + t.Fatal("no list sealed once every user has a hash") } - // The bus loads the sealed list: both written, nothing waits. loads(delivery.GrantsUserListSource, sealed.Digest) - if res := p.Compose(); len(res.Grants.Waits) != 0 || !slices.Equal(res.Grants.Written, []string{key2, key("mod-y")}) { - t.Errorf("once the bus loaded the list carrying them: %+v", res.Grants) + b.refuseUsers = true + // (2) mod-y's trunk adds a state, so its account adds permissions: it waits, judged from the part's own record of the + // loaded list; mod-x at another fingerprint with the same permissions is written. + fp3 := "src2:" + strings.Repeat("ef", 32) + keyY3 := moves("mod-y", strings.Replace(yManifest, `"reads"`, `"state":[{"name":"fresh"}],"reads"`, 1), fp3) + keyX3 := moves("mod-x", xManifest, fp3) + res = p.Compose() + if !written(keyX3) || written(keyY3) || len(res.Grants.Waits) != 1 || + !strings.HasPrefix(res.Grants.Waits[0], keyY3+": waits for the bus to load a list carrying what it adds: b.mod-y") { + t.Fatalf("an added permission against the loaded list: %+v", res.Grants) + } + // A loaded list the part keeps no permissions of: every grant naming an account waits, said. + loads(delivery.GrantsUserListSource, "sha256:bb") + keyX4 := moves("mod-x", xManifest, "src2:"+strings.Repeat("12", 32)) + if res := p.Compose(); written(keyX4) || len(res.Grants.Waits) != 2 || !strings.Contains(res.Grants.Waits[0], "unknown") { + t.Fatalf("an unknown loaded list let a grant through: %+v", res.Grants) + } + // The bus loads a list carrying mod-y's state (its tools answer it): both written, nothing waits. + listRecs, _ := grants.ListRecords(grants.ReadInputs(b, &grants.Manifests{})) + now, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + b.refuseUsers, b.loaded = false, now + if res := p.Compose(); len(res.Grants.Waits) != 0 || !written(keyY3) || !written(keyX4) { + t.Errorf("once the loaded list carries them: %+v", res.Grants) } }) diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go index 18c6fc0..92df498 100644 --- a/modules/mesh-grants/internal/grants/grants.go +++ b/modules/mesh-grants/internal/grants/grants.go @@ -1,6 +1,7 @@ package grants import ( + "encoding/json" "fmt" "slices" "sort" @@ -40,28 +41,121 @@ type GrantsWritten struct { } // GrantWaits says why a grant composed now, new or changed, is not written yet, or "" when it may be (issue 774). The -// assigner moves a module's want to the fingerprint once its grant is there, so a grant is the word that the bus's user list -// in force already carries the module's subjects at that fingerprint. After take-over that is so when the list the bus -// node says it loaded is the grants part's own and is the list composed now (composed: its digest), which ListRecords -// composes from what runs and what the trunk names; a list held, not yet sealed, or not yet loaded is not it. In shadow -// nobody reads the part's grants and the controller's list is in force, so nothing waits. Pure. -func GrantWaits(tookOver bool, loaded *delivery.UserListWritten, composed string) string { - switch { - case !tookOver: +// assigner moves a module's want to the fingerprint once its grant is there, so the grant is the word that the list the bus +// has loaded already carries the module's accounts as composed now (composed, from ListRecords: what runs and what the trunk +// names): every publish and subscribe allowed, the allowance of responses, and no deny the composed account lacks. A grant +// that adds no permission is written at once, whatever list is held or pending; one that adds any waits until a list that +// carries it is loaded. loaded is the loaded list by user, nil when its contents are unknown (unknown says why): then only +// a grant naming no account is written. In shadow nobody reads the part's grants and nothing waits. Pure. +func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]User, unknown string) string { + if !tookOver || len(accounts) == 0 { return "" - case composed == "": - return "waits: the user list could not be composed, so no list carries it" - case loaded == nil: - return "waits for the bus node to say which user list it loaded" - case loaded.Source != delivery.GrantsUserListSource: - return "waits for the bus to load the grants part's list " + short(composed) + ": it loaded " + loaded.Source + "'s" - case loaded.Error != "": - return "waits for the bus to load the list " + short(composed) + ": its node could not write it: " + loaded.Error - case loaded.Digest != composed: - return "waits for the bus to load the list " + short(composed) + ", which carries it: it has " + short(loaded.Digest) + - " loaded (a list held, or sealed and not yet loaded)" } - return "" + if loaded == nil { + return "waits: the contents of the list the bus loaded are unknown (" + unknown + "), so no grant adding an account is written" + } + var adds []string + for _, user := range accounts { + c, ok := composed[user] + if !ok { + continue + } + l, ok := loaded[user] + if !ok { + adds = append(adds, user+" (not in the loaded list)") + continue + } + n := len(minus(c.Publish, l.Publish)) + len(minus(c.Subscribe, l.Subscribe)) + if c.AllowResponses && !l.AllowResponses { + n++ + } + if l.PublishDeny != nil { + n += len(minus(l.PublishDeny, c.PublishDeny)) + } + if n > 0 { + adds = append(adds, fmt.Sprintf("%s (%d permissions)", user, n)) + } + } + if len(adds) == 0 { + return "" + } + return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ") +} + +// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each +// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus +// module's tools (issue 774). +const KeySealedGrants = "sealed-grants" + +// SealedGrants is the value of KeySealedGrants. +type SealedGrants struct { + Lists []SealedList `json:"lists"` +} + +// SealedList is one sealed list's users and their permissions. +type SealedList struct { + Digest string `json:"digest"` + Users []User `json:"users"` +} + +// keepSealedGrants records the permissions of the list sealed now, beside the one before. +func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) { + in := listUsers(list) + var users []User + for _, u := range composed { + if slices.Contains(in, u.User) { + u.Hash = "" + users = append(users, u) + } + } + var was SealedGrants + if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found { + _ = json.Unmarshal(raw, &was) + } + next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}} + for _, l := range was.Lists { + if l.Digest != digest && len(next.Lists) < 2 { + next.Lists = append(next.Lists, l) + } + } + data, _ := json.Marshal(next) + if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err)) + } +} + +// loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the +// list of the digest the bus node says it loaded from the part's key; nil and why when neither says. +func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string]User, string) { + byUser := func(us []User) map[string]User { + out := map[string]User{} + for _, u := range us { + out[u.User] = u + } + return out + } + if readErr == nil { + return byUser(read), "" + } + l := in.Loaded + switch { + case l == nil: + return nil, "the bus node says no loaded list, and nats_users: " + readErr.Error() + case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "": + return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error() + } + raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants) + if err == nil && found { + var sg SealedGrants + if json.Unmarshal(raw, &sg) == nil { + for _, sl := range sg.Lists { + if sl.Digest == l.Digest { + return byUser(sl.Users), "" + } + } + } + } + return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error() } // GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126). @@ -203,7 +297,7 @@ func grantsReadFailed(in Inputs) string { // writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a // module that has none now. -func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed string) { +func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed []User, loaded map[string]User, unknown string) { if why := grantsReadFailed(in); why != "" { res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}} return @@ -220,7 +314,11 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)} res.Grants = done p.sayMissing(done.Missing) - waits := GrantWaits(p.SwitchState() == TookOver, in.Loaded, composed) + tookOver := p.SwitchState() == TookOver + byUser := map[string]User{} + for _, u := range composed { + byUser[u.User] = u + } for _, key := range sortedKeys(want) { g := want[key] data, err := delivery.EncodeControllerGrant(g) @@ -236,7 +334,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, comp continue } } - if waits != "" { + if waits := GrantWaits(tookOver, g.Accounts, byUser, loaded, unknown); waits != "" { done.Waits = append(done.Waits, key+": "+waits) continue } diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 1e306be..ca23375 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -251,10 +251,10 @@ func (p *Part) Compose() Result { } users = WithHashes(users, in.Hashes) st.Digest, st.Users = Digest(users), len(users) - loaded, err := LoadedUsers(p.Call) + loaded, loadErr := LoadedUsers(p.Call) st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) - if err != nil { - st.Compare.Unread = err.Error() + if loadErr != nil { + st.Compare.Unread = loadErr.Error() } st.Credentials = CredentialsOf(users, loaded, in.Hashes) st.Inputs = st.inputs() @@ -280,14 +280,11 @@ func (p *Part) Compose() Result { p.writeAccounts(&res, in, listRecs) } list, n, leftOut, err := ComposeList(listRecs, in.Hashes) - composed := "" - if err == nil { - composed = delivery.UserListDigest(list) - } // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over - // each new one only once the bus loaded a list that carries it (issue 774). + // one that adds a permission only once the bus loaded a list that carries it (issue 774). if p.Store != nil { - p.writeGrants(&res, in, listRecs, composed) + inForce, unknown := p.loadedContents(in, loaded, loadErr) + p.writeGrants(&res, in, listRecs, users, inForce, unknown) } // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). if err != nil { @@ -299,7 +296,7 @@ func (p *Part) Compose() Result { bus = names(loaded) lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users)) } - p.writeSealed(&res, list, n, leftOut, in, bus, lost) + p.writeSealed(&res, list, n, leftOut, in, bus, lost, users) } ms := Memberships(recs) res.Memberships = len(ms) diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index 55ff811..097e09d 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -74,7 +74,7 @@ func followOf(state string) string { // writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in // res what it wrote, or why it wrote nothing. -func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string) { +func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) { cur, _, had := p.Sealed() next := cur next.Follow = followOf(p.SwitchState()) @@ -137,6 +137,7 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string } if next.Digest == digest && cur.Digest != digest { p.keepWrittenUsers(res, list, in.Hashes) + p.keepSealedGrants(res, digest, list, composed) p.acceptedWritten(res, digest) } // The object the key named before is taken away once the key names the new one: the bucket keeps the list in force. From da787ae36048f130a9c561e683a5a438aedb9e65 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:44:18 +0100 Subject: [PATCH 05/35] Watchman: hold mesh-watchdogs and write its watchdogs as findings, so health can leave the controller (issue 1952 WM1-WM5) The signals rows S1, S8, S9, S10, S11, S21 and the haves' unhealthy and needs-operator are pulses whose expiry is a finding; stuck keys and presence are its own findings; it keeps the dead letters from the lease write on its own consumer, lists and shows them, and refuses deliver and drop until the hand-act log's home (issue 1953). Tools read every state it keeps and start resync. Pins mesh-sdk 2a04bf0. --- .../cmd/mesh-delivery-watchman-tools/main.go | 170 ++++++ .../mesh-delivery-watchman-tools/main_test.go | 144 +++++ .../integration_test.go | 102 +++- .../cmd/mesh-delivery-watchman/main.go | 58 +- .../cmd/mesh-delivery-watchman/signals.go | 185 +++++++ modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 +- .../internal/deadletters/deadletters.go | 358 ++++++++++++ .../internal/deadletters/deadletters_test.go | 64 +++ .../internal/socket/socket.go | 125 +++++ .../internal/watch/advisories.go | 56 ++ .../internal/watch/dead.go | 78 +++ .../internal/watch/finding.go | 159 ++++++ .../internal/watch/findings_test.go | 200 +++++++ .../internal/watch/mirror.go | 2 + .../internal/watch/signals.go | 524 ++++++++++++++++++ .../internal/watch/watch.go | 61 +- .../internal/watch/watch_test.go | 50 +- .../mesh-delivery-watchman/manifest_test.go | 64 ++- modules/mesh-delivery-watchman/module.json | 39 +- 20 files changed, 2401 insertions(+), 44 deletions(-) create mode 100644 modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go create mode 100644 modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go create mode 100644 modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go create mode 100644 modules/mesh-delivery-watchman/internal/deadletters/deadletters.go create mode 100644 modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go create mode 100644 modules/mesh-delivery-watchman/internal/socket/socket.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/advisories.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/dead.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/finding.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/findings_test.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/signals.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go new file mode 100644 index 0000000..1ae2d78 --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go @@ -0,0 +1,170 @@ +// Command mesh-delivery-watchman-tools serves the seat mesh-watchdogs' verbs and the watchman's read tools (novox/hq ADR 0564 +// §4, §5, §12, issue 1952 WM1): the runtime launches it for a call. It reads every state the watchman keeps (presence, pulse, +// stuck, findings: its keys, one key, how many it holds) and asks the watchman's program, on its socket, for what only the +// program reads or does: DEAD_LETTERS, a stream, and resync. A process never serves (ADR 0436). +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" +) + +// Seat is the seat the watchman holds. +const Seat = "mesh-watchdogs" + +// The seat's verbs and the module's own tools, as the manifest names them. +const ( + VerbSignals = "signals" + VerbStuck = "stuck" + VerbPresence = "presence" + VerbDeadLetters = "dead-letters" + VerbResync = "resync" + ToolFindings = "watchman_findings" +) + +// Verbs are the seat's verbs, in the manifest's order. +var Verbs = []string{VerbSignals, VerbStuck, VerbPresence, VerbDeadLetters, VerbResync} + +// ModuleTools are the module's own tools, in the manifest's order. +var ModuleTools = []string{ToolFindings} + +// Reads names, per tool, the states it reads whole; Acts, per tool, the program's act it starts. The clause-12 test holds +// that every state the manifest keeps is read by one, and every act the program takes by hand is started by one. +var ( + Reads = map[string][]string{VerbSignals: {"pulse"}, VerbStuck: {"stuck"}, VerbPresence: {"presence"}, ToolFindings: {"findings"}} + Acts = map[string][]string{VerbDeadLetters: {socket.AskDeadLetters}, VerbResync: {socket.AskResync}} +) + +// state is what the tools need of a state: the runtime's, or a test's. +type state interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// keyed is one key as a read answers it. +type keyed struct { + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` +} + +// readAll answers a state whole, or one key, with how many keys it holds; prefix narrows the keys. +func readAll(st state, name, key, prefix string) (any, error) { + if key != "" { + e, err := st.Get(key) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", name, key, err) + } + if e == nil { + return nil, fmt.Errorf("%s holds no key %s", name, key) + } + return keyed{Key: key, Value: e.Value, Revision: e.Revision}, nil + } + keys, err := st.Keys() + if err != nil { + return nil, fmt.Errorf("%s cannot be listed: %w", name, err) + } + sort.Strings(keys) + out := []keyed{} + for _, k := range keys { + if prefix != "" && !strings.HasPrefix(k, prefix) { + continue + } + e, err := st.Get(k) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", name, k, err) + } + if e != nil { + out = append(out, keyed{Key: k, Value: e.Value, Revision: e.Revision}) + } + } + return map[string]any{"state": name, "holds": len(keys), "keys": out}, nil +} + +func str(args map[string]any, name string) string { + v, _ := args[name].(string) + return strings.TrimSpace(v) +} + +var keyArgs = map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "one key only"}, + "prefix": map[string]any{"type": "string", "description": "only the keys that begin with it"}}} + +// Tools are the seat's verbs and the module's tools, over the states states gives and the program asked through ask. +func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, error)) []stdio.Tool { + read := func(name string) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + return readAll(states(name), name, str(args, "key"), str(args, "prefix")) + } + } + signals := func(args map[string]any) (any, error) { + pulses, err := readAll(states("pulse"), "pulse", str(args, "key"), str(args, "prefix")) + if err != nil { + return nil, err + } + return map[string]any{"rows": watch.SignalRows, "pulses": pulses}, nil + } + dead := func(args map[string]any) (any, error) { + a := socket.Ask{Ask: socket.AskDeadLetters, Consumer: str(args, "consumer"), Act: str(args, "act"), Why: str(args, "why")} + if v, ok := args["limit"].(float64); ok { + a.Limit = int(v) + } + if v, ok := args["id"].(float64); ok && v > 0 { + a.ID = uint64(v) + } + return ask(a) + } + resync := func(map[string]any) (any, error) { return ask(socket.Ask{Ask: socket.AskResync}) } + verb := func(v string) string { return Seat + "." + v } + return []stdio.Tool{ + {Name: verb(VerbSignals), Input: keyArgs, Run: signals, + Description: "The signals table's rows the watchman holds (S1, S8, S9, S10, S11, S21 and the haves' health), each a pulse key " + + "whose time to live is its bound and whose expiry is its finding (novox/hq ADR 0564 §4), and every pulse the watchman " + + "keeps: the moves it mirrors (a build step, a want, a presence) and the signals (signal.., with what their " + + "expiry does). Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbStuck), Input: keyArgs, Run: read("stuck"), + Description: "What stopped moving: every key of the watchman's stuck bucket, . with what it waited for since " + + "when, and whether it moved since (never deleted). A key not moved is also the watchman's finding " + + "delivery..stuck, a node's presence machine..silent. Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbPresence), Input: keyArgs, Run: read("presence"), + Description: "Who is online on the bus as the watchman hears it: each node's agent () and each delivery part's instance " + + "(part..), online or not since the entry's time. Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbDeadLetters), Run: dead, + Description: "What a consumer gave up on, kept in DEAD_LETTERS (novox/hq ADR 0564 §5, ADR 0264, ADR 0376): newest first, " + + "for one consumer (its name or .), at most limit; one whole by id (a restricted one's body " + + "withheld). Who keeps them now: the controller until the lease names no controller, the watchman from then. " + + "Delivering again and dropping (act deliver or drop, with why) are refused here until the hand-act log's home " + + "(issue 1953): before the lease write use mesh-controller.dead-letters.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "consumer": map[string]any{"type": "string", "description": "only this consumer's, its name or ."}, + "limit": map[string]any{"type": "integer", "description": "at most this many, newest first"}, + "id": map[string]any{"type": "integer", "description": "one dead letter whole, by its id"}, + "act": map[string]any{"type": "string", "enum": []string{"deliver", "drop"}, "description": "deliver again or drop (refused until issue 1953)"}, + "why": map[string]any{"type": "string", "description": "why, with act"}}}}, + {Name: verb(VerbResync), Run: resync, + Description: "Rebuild what the watchman holds by hand (novox/hq ADR 0564 §12): presence from the bus's open connections, " + + "the dead letters' count from DEAD_LETTERS and whether to take the notices, and every finding it observes written " + + "again to its bucket. Answers how many findings it wrote."}, + {Name: ToolFindings, Input: keyArgs, Run: read("findings"), + Description: "The watchman's findings (novox/hq ADR 0564 §2): each thing it observes wrong now, .., with its " + + "severity, subject, evidence and the row that observed it; the condition keeper raises and clears the conditions from " + + "them. Give key for one, prefix to narrow. Read-only."}, + } +} + +func main() { + path := socket.PathOf(os.Getenv) + tools := Tools(func(name string) state { return stdio.State(name) }, func(a socket.Ask) (json.RawMessage, error) { return socket.On(path, a) }) + if err := stdio.Serve("", tools); err != nil { + fmt.Fprintln(os.Stderr, "mesh-delivery-watchman-tools:", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go new file mode 100644 index 0000000..97a9262 --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "slices" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-watchman/internal/socket" +) + +type fakeState map[string]string + +func (f fakeState) Keys() ([]string, error) { + out := []string{} + for k := range f { + out = append(out, k) + } + return out, nil +} + +func (f fakeState) Get(key string) (*stdio.StateEntry, error) { + v, ok := f[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: 3}, nil +} + +// 1952/WM1, ADR 0564 §12: every state the manifest keeps is read by a tool, every act the program starts by hand is +// started by one, and every verb the claim serves and every tool the manifest names is served; and each answers. +func TestEveryStateAndActHasATool(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Tools []string `json:"tools"` + State []struct { + Name string `json:"name"` + } `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + asked := []socket.Ask{} + states := map[string]fakeState{"findings": {"machine.ace.silent": `{"v":1}`, "core.doctor.self-check-silent": `{"v":1}`}} + tools := Tools(func(name string) state { return states[name] }, func(a socket.Ask) (json.RawMessage, error) { + asked = append(asked, a) + if a.Act != "" { + return nil, errors.New("refused") + } + return json.RawMessage(`{}`), nil + }) + served := map[string]stdio.Tool{} + for _, tool := range tools { + served[tool.Name] = tool + } + var names []string + for _, c := range m.Claims { + if c.Name != Seat || !slices.Equal(c.Serves, Verbs) { + t.Errorf("claim %+v, want %s serving %v", c, Seat, Verbs) + } + for _, v := range c.Serves { + names = append(names, Seat+"."+v) + } + } + if !slices.Equal(m.Tools, ModuleTools) { + t.Errorf("tools = %v, want %v", m.Tools, ModuleTools) + } + names = append(names, m.Tools...) + if len(served) != len(names) { + t.Errorf("served %d tools, the manifest names %v", len(served), names) + } + for _, n := range names { + if _, ok := served[n]; !ok { + t.Errorf("%s is named and not served", n) + } + } + read := map[string]bool{} + for _, r := range Reads { + for _, s := range r { + read[s] = true + } + } + for _, s := range m.State { + if !read[s.Name] { + t.Errorf("the state %s is read by no tool", s.Name) + } + } + acts := map[string]bool{} + for _, a := range Acts { + for _, s := range a { + acts[s] = true + } + } + for _, a := range []string{socket.AskDeadLetters, socket.AskResync} { + if !acts[a] { + t.Errorf("the program's act %s is started by no tool", a) + } + } + t.Run("a read answers every key and how many", func(t *testing.T) { + out, err := served[ToolFindings].Run(map[string]any{"prefix": "machine."}) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(out) + var got struct { + Holds int `json:"holds"` + Keys []struct { + Key string `json:"key"` + } `json:"keys"` + } + _ = json.Unmarshal(b, &got) + if got.Holds != 2 || len(got.Keys) != 1 || got.Keys[0].Key != "machine.ace.silent" { + t.Fatalf("answer = %s", b) + } + if _, err := served[ToolFindings].Run(map[string]any{"key": "no.such.key"}); err == nil { + t.Fatal("a key not held is answered") + } + }) + t.Run("dead-letters and resync ask the program", func(t *testing.T) { + if _, err := served[Seat+"."+VerbDeadLetters].Run(map[string]any{"consumer": "ace_web", "limit": float64(3), "id": float64(7)}); err != nil { + t.Fatal(err) + } + if _, err := served[Seat+"."+VerbDeadLetters].Run(map[string]any{"act": "drop", "why": "x"}); err == nil { + t.Fatal("a refused act answered") + } + if _, err := served[Seat+"."+VerbResync].Run(nil); err != nil { + t.Fatal(err) + } + want := []socket.Ask{{Ask: socket.AskDeadLetters, Consumer: "ace_web", Limit: 3, ID: 7}, {Ask: socket.AskDeadLetters, Act: "drop", Why: "x"}, {Ask: socket.AskResync}} + if !slices.Equal(asked, want) { + t.Fatalf("asked %+v", asked) + } + }) +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index 27cf64c..b32baa3 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -17,8 +17,13 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-watchman/internal/deadletters" + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" ) // busConf is the composed accounts file's shape (mesh-controller internal/broker): the account MESH imports the system @@ -103,6 +108,7 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -111,6 +117,31 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 kvs[c.Bucket] = kv } + // The controller's lease, its consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one + // message a consumer will give up on (1952/WM5). + lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket}) + if err != nil { + t.Fatal(err) + } + for _, c := range []jetstream.StreamConfig{ + {Name: buscompose.DeadLetterNoticesStream, Subjects: []string{buscompose.MaxDeliveriesAdvisories}, Retention: jetstream.WorkQueuePolicy}, + {Name: buscompose.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}, Duplicates: 24 * time.Hour, AllowMsgTTL: true}, + {Name: "SRC", Subjects: []string{"src.>"}}, + } { + if _, err := js.CreateStream(ctx, c); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateConsumer(ctx, buscompose.DeadLetterNoticesStream, jetstream.ConsumerConfig{Durable: buscompose.ControllerName, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + given := nats.NewMsg("src.ask") + given.Header.Set("X-Asker", "n1") + given.Data = []byte("do it") + if _, err := js.PublishMsg(ctx, given); err != nil { + t.Fatal(err) + } + // A node already connected when the watchman starts: its presence comes from the one connections request. connect("node.n0") @@ -119,7 +150,13 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_PULSE_BOUND": "3s"} + // A unix socket's path is at most 108 bytes, which a test's TMPDIR may pass. + sockDir, err := os.MkdirTemp("/tmp", "ws") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_PULSE_BOUND": "3s", "MESH_WATCHMAN_SOCKET": filepath.Join(sockDir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() @@ -224,6 +261,63 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 t.Errorf("a stuck key %s was written for a move that happened before the bound", k) } } + + // 1952/WM4: what stopped moving is the watchman's own finding: the ask no node ended, and node n1 silent. + finding := func(key string) bool { _, ok := has(watch.FindingsBucket, key); return ok } + until("the stuck ask's finding", func() bool { + return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck)) + }) + until("node n1's silence", func() bool { return finding("machine.n1.silent") }) + + // 1952/WM5: while the lease names a controller the watchman takes no notice; handed away while the controller's consumer + // stands, it says the notices wait; the consumer removed (the lease write's step), it makes its own and keeps a notice + // with its headers. + if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil { + t.Fatal("the watchman made its consumer while the lease names a controller") + } + if _, err := lease.Put(ctx, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator","why":"the lease write"}`)); err != nil { + t.Fatal(err) + } + waits := watch.FindingKey(watch.ScopeBus, "DEAD_LETTER_NOTICES.controller", watch.KindNoticesWaiting) + until("the notices said waiting behind the controller's consumer", func() bool { return finding(waits) }) + if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil { + t.Fatal("the watchman made its consumer while the controller's stands") + } + if err := js.DeleteConsumer(ctx, buscompose.DeadLetterNoticesStream, buscompose.ControllerName); err != nil { + t.Fatal(err) + } + until("the watchman's own consumer on the notices", func() bool { + _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer) + return err == nil + }) + until("the wait no longer said", func() bool { return !finding(waits) }) + notice, _ := json.Marshal(map[string]any{"stream": "SRC", "consumer": "c1", "stream_seq": 1, "deliveries": 5, "timestamp": time.Now().UTC()}) + if err := ctl.Publish(buscompose.MaxDeliveriesAdvisories[:len(buscompose.MaxDeliveriesAdvisories)-1]+"SRC.c1", notice); err != nil { + t.Fatal(err) + } + dl, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + t.Fatal(err) + } + var kept *jetstream.RawStreamMsg + until("the notice kept in DEAD_LETTERS", func() bool { + kept, err = dl.GetLastMsgForSubject(ctx, buscompose.DeadLetterSubject("SRC", "c1")) + return err == nil + }) + if kept.Header.Get(deadletters.HeaderStream) != "SRC" || kept.Header.Get(deadletters.HeaderSequence) != "1" || + kept.Header.Get(deadletters.HeaderSubject) != "src.ask" || kept.Header.Get("X-Asker") != "n1" || string(kept.Data) != "do it" { + t.Fatalf("kept = %v %q", kept.Header, kept.Data) + } + until("the consumer's dead letters said", func() bool { return finding("bus.SRC.c1.max-deliveries") }) + // The tools' socket lists it, and refuses to drop it. + answer, err := socket.On(env["MESH_WATCHMAN_SOCKET"], socket.Ask{Ask: socket.AskDeadLetters, Consumer: "c1"}) + if err != nil || !strings.Contains(string(answer), `"total":1`) || !strings.Contains(string(answer), "the watchman") { + t.Fatalf("dead-letters = %s, %v", answer, err) + } + if _, err := socket.On(env["MESH_WATCHMAN_SOCKET"], socket.Ask{Ask: socket.AskDeadLetters, ID: kept.Sequence, Act: "drop", Why: "test"}); err == nil || + !strings.Contains(err.Error(), "issue 1953") { + t.Fatalf("a drop was not refused naming the hand-act log's home: %v", err) + } } // TestAnAskRunningBeforeTheStartGetsItsPulse puts a running step in the build module's steps before the watchman starts, and @@ -271,6 +365,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -291,7 +386,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker} + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_WATCHMAN_SOCKET": filepath.Join(dir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() @@ -355,6 +450,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -393,7 +489,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker} + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_WATCHMAN_SOCKET": filepath.Join(dir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index c349da7..af4d80b 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,8 +22,10 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-watchman/internal/socket" "mesh-delivery-watchman/internal/watch" ) @@ -44,12 +46,19 @@ func main() { } } -// kvOut writes the watchman's three buckets. +// kvOut writes the watchman's four buckets. type kvOut struct { - ctx context.Context - presence, pulse, stuk jetstream.KeyValue + ctx context.Context + presence, pulse, stuk, findings jetstream.KeyValue } +func (o *kvOut) PutFinding(key string, value []byte) error { + _, err := o.findings.Put(o.ctx, key, value) + return err +} + +func (o *kvOut) DeleteFinding(key string) error { return o.findings.Delete(o.ctx, key) } + func (o *kvOut) PutPresence(key string, online bool) error { value, err := delivery.EncodePresence(delivery.Presence{Online: online}) if err != nil { @@ -161,11 +170,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants // and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build // seats' steps, and says that moves are not watched (issue 1945). - var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue } + var kvs struct{ presence, pulse, stuck, findings, wants, haves, lease jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue - }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { + }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}, {watch.FindingsBucket, &kvs.findings}} { if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil { return err } @@ -176,7 +185,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it } - w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now) + kvs.lease = process.OptionalBucket(ctx, js, buscompose.LeaseBucket, "the lease is read as naming a controller, so the dead letters stay the controller's", logger.Printf) + w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now) w.Logf(logger.Printf) // A test shortens how long a move may take; the mesh never sets it. if v := strings.TrimSpace(getenv("MESH_DELIVERY_PULSE_BOUND")); v != "" { @@ -197,6 +207,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro w.SeedPresence(e.Key(), p.Online, e.Created()) } }}, + {kvs.findings, func(e jetstream.KeyValueEntry) { + if opOf(e) == watch.Put { + w.SeedFinding(e.Key(), e.Value()) + } + }}, {kvs.stuck, func(e jetstream.KeyValueEntry) { if opOf(e) == watch.Put { w.SeedStuck(e.Key(), e.Value()) @@ -212,6 +227,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro pulses, err := settle(ctx, kvs.pulse, func(e jetstream.KeyValueEntry) { if opOf(e) == watch.Put { w.SeedPulse(e.Key()) + w.SeedSignal(e.Key(), e.Value()) } }) if err != nil { @@ -281,9 +297,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := w.Rebuild(ask); err != nil { return err } + dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} + dead.count() + dead.check() w.Reconcile() + if err := hear(conn, w, dead, logger.Printf); err != nil { + return err + } errs := make(chan error, 8) + if kvs.lease != nil { + lw, err := kvs.lease.Watch(ctx, buscompose.ControllerLeaseKey) + if err != nil { + process.NotWatched(logger.Printf, buscompose.LeaseBucket, err, "the lease is read as naming a controller, so the dead letters stay the controller's") + } else { + go func() { + errs <- follow(ctx, lw, func(e jetstream.KeyValueEntry) { + var raw []byte + if opOf(e) == watch.Put { + raw = e.Value() + } + on, _ := buscompose.LeaseHandedAway(raw, len(raw) > 0, nil) + dead.lease(on) + }) + }() + } + } + if l, err := socket.Listen(socket.PathOf(getenv)); err != nil { + logger.Printf("the tools' socket cannot be opened (%v): dead-letters and resync answer nothing until the watchman starts again", err) + } else { + go func() { + errs <- socket.Serve(ctx, l, func(a socket.Ask) (any, error) { return answer(a, w, dead, ask) }) + }() + } go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go new file mode 100644 index 0000000..6f7f742 --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go @@ -0,0 +1,185 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + + "mesh-delivery-watchman/internal/deadletters" + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" +) + +// The subjects the watchman hears for the signals table's rows (novox/hq ADR 0564 §4, Limits 3 and 11), each on the bus, +// none in the controller's store. Its grant names them (mesh-sdk buscompose, the watchman's principal). +const ( + PowerSubject = "mesh.mod.power.event.*" + FailingSubject = "mesh.mod.*.event.provisioner.failing" + RecoveredSubject = "mesh.mod.*.event.provisioner.recovered" + ToolsAliveSubject = "mesh.control.*.tools-alive" + CheckedSubject = "mesh.seat.mesh-self-check.event.checked" + DeadKeptSubject = "mesh.events.dead.>" +) + +// HeardSubjects is every subject the watchman subscribes for its rows, beside the bus's connect and disconnect. +var HeardSubjects = []string{PowerSubject, FailingSubject, RecoveredSubject, ToolsAliveSubject, CheckedSubject, + watch.ConsumerDeletedAdvisories, DeadKeptSubject} + +// heard hands one message of a row's subject to the watchman. +func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, ...any)) { + switch { + case strings.HasPrefix(m.Subject, "mesh.mod.power.event."): + w.Power(m.Header.Get("x-node"), strings.TrimPrefix(m.Subject, "mesh.mod.power.event.")) + case strings.HasSuffix(m.Subject, ".event.provisioner.failing") || strings.HasSuffix(m.Subject, ".event.provisioner.recovered"): + st, err := watch.ReadStanding(m.Subject, m.Data) + if err != nil { + logf("%v; left", err) + return + } + w.Stood(st) + case strings.HasSuffix(m.Subject, ".tools-alive"): + var body struct { + Every float64 `json:"every"` + } + _ = json.Unmarshal(m.Data, &body) + w.ToolsAlive(watch.ToolsNode(m.Subject), time.Duration(body.Every*float64(time.Second))) + case m.Subject == CheckedSubject: + w.Checked(m.Header.Get("x-node")) + case strings.HasPrefix(m.Subject, "$JS.EVENT.ADVISORY."): + a, ok := watch.ReadAdvisory(m.Subject, m.Data, deadletters.ConsumerInWords) + if !ok { + return + } + if dead.isTheControllersNotices(a.ID) { + dead.check() // the lease write's step removed it: the watchman may take the notices now + return + } + w.Advised(a) + case strings.HasPrefix(m.Subject, "mesh.events.dead."): + dead.count() + } +} + +// hear subscribes every row's subject; a subject the bus refuses stops the start, naming it. +func hear(conn *nats.Conn, w *watch.Watchman, dead *deadKeeper, logf func(string, ...any)) error { + for _, subject := range HeardSubjects { + if _, err := conn.Subscribe(subject, func(m *nats.Msg) { heard(w, dead, m, logf) }); err != nil { + return fmt.Errorf("the watchman cannot hear %s: %w", subject, err) + } + } + return conn.Flush() +} + +// deadKeeper keeps what consumers gave up on from the lease write (ADR 0564 §5): it counts DEAD_LETTERS into findings, +// and takes DEAD_LETTER_NOTICES on its own consumer only once the lease names no controller and `controller` is gone. +type deadKeeper struct { + ctx context.Context + js jetstream.JetStream + w *watch.Watchman + logf func(string, ...any) + + mu sync.Mutex + handedAway bool + taking jetstream.ConsumeContext +} + +// count reads DEAD_LETTERS' count per consumer into the findings. +func (d *deadKeeper) count() { + held, err := deadletters.Held(d.ctx, d.js) + if err != nil { + d.logf("the dead letters cannot be counted, their findings are left as they are: %v", err) + return + } + d.w.DeadLettersHeld(held, deadletters.ConsumerInWords) +} + +// lease is the lease's change heard: handed away or not. +func (d *deadKeeper) lease(handedAway bool) { + d.mu.Lock() + d.handedAway = handedAway + d.mu.Unlock() + d.check() +} + +func (d *deadKeeper) away() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.handedAway +} + +// isTheControllersNotices says an advisory is about the controller's consumer on DEAD_LETTER_NOTICES once the lease is +// handed away: its removal is the lease write's step, not a loss. +func (d *deadKeeper) isTheControllersNotices(id string) bool { + return d.away() && id == buscompose.DeadLetterNoticesStream+"."+buscompose.ControllerName +} + +// check takes the notices, or says they wait, as Gate has it. +func (d *deadKeeper) check() { + away := d.away() + stands := true + if away { + var err error + if stands, err = deadletters.ControllerStands(d.ctx, d.js); err != nil { + d.logf("whether the controller's consumer on %s stands cannot be read (%v): it is taken to stand", buscompose.DeadLetterNoticesStream, err) + stands = true + } + } + take, waiting := deadletters.Gate(away, stands) + d.w.NoticesWaiting(waiting) + d.mu.Lock() + defer d.mu.Unlock() + if !take || d.taking != nil { + return + } + cc, err := deadletters.TakeNotices(d.ctx, d.js, func(deadletters.DeadLetter) { go d.count() }, d.logf) + if err != nil { + d.logf("the dead-letter notices cannot be taken yet: %v", err) + return + } + d.taking = cc + d.logf("the lease names no controller and its consumer on %s is gone: the watchman keeps the dead letters", buscompose.DeadLetterNoticesStream) +} + +// answer is the program's answer to the tools' ask on its socket. +func answer(a socket.Ask, w *watch.Watchman, dead *deadKeeper, ask func([]byte) ([]byte, error)) (any, error) { + switch a.Ask { + case socket.AskResync: + if err := w.Rebuild(ask); err != nil { + return nil, err + } + dead.count() + dead.check() + return map[string]any{"findings-written": w.Rewrite(), "presence": "rebuilt from the bus's connections"}, nil + case socket.AskDeadLetters: + switch { + case a.Act == "deliver" || a.Act == "drop": + if strings.TrimSpace(a.Why) == "" { + return nil, errors.New("a dead letter is delivered again or dropped only with why") + } + return nil, deadletters.Refusal(dead.away(), a.Act) + case a.Act != "": + return nil, fmt.Errorf("%q is not deliver or drop", a.Act) + case a.ID > 0: + return deadletters.Named(dead.ctx, dead.js, a.ID) + } + list, total, err := deadletters.List(dead.ctx, dead.js, a.Consumer, a.Limit) + if err != nil { + return nil, err + } + keeper := "the controller (it holds its lease)" + if dead.away() { + keeper = "the watchman" + } + return map[string]any{"dead-letters": list, "total": total, "kept-by": keeper}, nil + } + return nil, fmt.Errorf("refused: the watchman's socket answers %q and %q", socket.AskDeadLetters, socket.AskResync) +} diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d42c521..1390d78 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index 084deeb..e796a10 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf h1:Yy51rUWqyk+cZPAcSX02dnK4JVSTUJIfLa9j+741+1Y= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go b/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go new file mode 100644 index 0000000..4e4a228 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go @@ -0,0 +1,358 @@ +// Package deadletters is what a consumer gave up on, kept by the watchman from the lease write (novox/hq ADR 0564 §5, +// ADR 0264, ADR 0376, issue 1952 WM5): ported from mesh-controller internal/link/deadletters.go at e04846e9 with nothing +// changed in what is kept, aged or withheld, on the jetstream API. +// +// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The controller's consumer `controller` never expires, so +// the watchman makes its own (Consumer) only when the lease names no controller and `controller` is gone (Gate); while +// `controller` still stands after the lease write it says so as a finding. Removing `controller` is a step of the lease +// write, with the bus module's tool. Before the lease write the watchman only lists and shows DEAD_LETTERS, which the +// controller still fills, and refuses to deliver or drop, naming the controller's verb; from it, delivering and dropping +// wait for the hand-act log's home (issue 1953), so they are refused naming that. +package deadletters + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// Consumer is the watchman's own consumer on DEAD_LETTER_NOTICES, made only by Gate's leave. +const Consumer = "mesh-delivery-watchman" + +// The headers a kept message carries beside its own (mesh-controller link.DeadHeader*: the same names, so the +// controller's copies and the watchman's are read alike). +const ( + HeaderStream = "Mesh-Dead-Stream" + HeaderConsumer = "Mesh-Dead-Consumer" + HeaderSequence = "Mesh-Dead-Sequence" + HeaderSubject = "Mesh-Dead-Subject" + HeaderDeliveries = "Mesh-Dead-Deliveries" + HeaderGaveUp = "Mesh-Dead-Gave-Up" + HeaderPublished = "Mesh-Dead-Published" + HeaderLost = "Mesh-Dead-Lost" + HeaderMsgID = "Mesh-Dead-Msg-Id" + HeaderRestricted = "Mesh-Dead-Restricted" + HeaderAgesOut = "Mesh-Dead-Ages-Out" + HeaderWorkQueue = "Mesh-Dead-Work-Queue" +) + +// DeadLetter is one message a consumer gave up on, as DEAD_LETTERS keeps it (the controller's JSON). +type DeadLetter struct { + ID uint64 `json:"id"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Who string `json:"who"` + Subject string `json:"subject"` + Sequence uint64 `json:"sequence"` + Deliveries uint64 `json:"deliveries"` + GaveUp time.Time `json:"gave_up"` + Published time.Time `json:"published,omitzero"` + Lost string `json:"lost,omitempty"` + Size int `json:"size"` + Restricted bool `json:"restricted,omitempty"` + AgesOut time.Time `json:"ages_out,omitzero"` + Withheld string `json:"withheld,omitempty"` + WorkQueue bool `json:"work-queue,omitempty"` + Body string `json:"body,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` +} + +// Now is the clock dead letters age by: a variable, so a test can move it. +var Now = time.Now + +// WithheldWords is what is said in place of a restricted dead letter's body. +const WithheldWords = "its body and its own headers are not shown: it came from a restricted stream, which only " + + "the modules its grant names may read (novox/hq ADR 0376)" + +// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what. +func ConsumerInWords(stream, name string) string { + switch { + case name == buscompose.ControllerName || name == buscompose.MergesConsumerName: + return "the controller's consumer on " + stream + case stream == "NODES": + return "how " + name + " hears what it should be (its declaration consumer)" + case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"): + seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-")) + return "the worker every holder of " + seat + " takes its asks from" + case stream == buscompose.EventsStream: + if node, module, ok := strings.Cut(name, "_"); ok { + return "how " + module + " on " + node + " hears what it consumes" + } + } + return "the consumer " + name + " on " + stream +} + +// sourceAge is whether a stream is restricted, a module's declared work queue, and its maximum age; a stream gone is +// neither. +func sourceAge(ctx context.Context, js jetstream.JetStream, stream string) (restricted, workQueue bool, maxAge time.Duration, err error) { + s, err := js.Stream(ctx, stream) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return false, false, 0, nil + } + if err != nil { + return false, false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err) + } + cfg := s.CachedInfo().Config + return cfg.Metadata[buscompose.RestrictedMetadata] == "true", cfg.Metadata[buscompose.WorkQueueMetadata] == "true", cfg.MaxAge, nil +} + +// notice is the part of the server's maximum-deliveries advisory a kept message is made from. +type notice struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + StreamSeq uint64 `json:"stream_seq"` + Deliveries uint64 `json:"deliveries"` + Timestamp time.Time `json:"timestamp"` +} + +// Keep keeps the message one maximum-deliveries notice is about. An error keeps nothing and the notice is to be taken +// again; a source that no longer holds the message is no error: the record is kept with why. +func Keep(ctx context.Context, js jetstream.JetStream, body []byte) (DeadLetter, error) { + var a notice + if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" || a.StreamSeq == 0 { + return DeadLetter{}, fmt.Errorf("not a maximum-deliveries notice the mesh can read: %.200s", body) + } + gaveUp := a.Timestamp + if gaveUp.IsZero() { + gaveUp = Now() + } + kept := &nats.Msg{Subject: buscompose.DeadLetterSubject(a.Stream, a.Consumer), Header: nats.Header{}} + restricted, workQueue, maxAge, err := sourceAge(ctx, js, a.Stream) + if err != nil { + return DeadLetter{}, err + } + if restricted { + kept.Header.Set(HeaderRestricted, "true") + } + if workQueue { + kept.Header.Set(HeaderWorkQueue, "true") + } + var raw *jetstream.RawStreamMsg + src, err := js.Stream(ctx, a.Stream) + if err == nil { + raw, err = src.GetMsg(ctx, a.StreamSeq) + } + agesOut := time.Time{} + if err == nil && restricted && maxAge > 0 { + agesOut = raw.Time.Add(maxAge) + kept.Header.Set(HeaderAgesOut, agesOut.UTC().Format(time.RFC3339Nano)) + } + switch { + case err == nil && !agesOut.IsZero() && agesOut.Sub(Now()) < time.Second: + kept.Header.Set(HeaderSubject, raw.Subject) + kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano)) + kept.Header.Set(HeaderLost, fmt.Sprintf("message %d of %s had passed its stream's maximum age of %s when the notice "+ + "was taken, so it is kept without its body (novox/hq ADR 0376)", a.StreamSeq, a.Stream, maxAge)) + case err == nil: + for k, v := range raw.Header { + if !strings.HasPrefix(k, "Nats-") { + kept.Header[k] = v + } + } + if id := raw.Header.Get(nats.MsgIdHdr); id != "" { + kept.Header.Set(HeaderMsgID, id) + } + kept.Header.Set(HeaderSubject, raw.Subject) + kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano)) + kept.Data = raw.Data + if !agesOut.IsZero() { + kept.Header.Set(nats.MsgTTLHdr, strconv.FormatInt(int64(agesOut.Sub(Now())/time.Second), 10)+"s") + } + case errors.Is(err, jetstream.ErrMsgNotFound) || errors.Is(err, jetstream.ErrStreamNotFound): + kept.Header.Set(HeaderLost, fmt.Sprintf("%s no longer held message %d when the notice was taken: %v", a.Stream, a.StreamSeq, err)) + default: + return DeadLetter{}, fmt.Errorf("message %d of %s could not be read: %w", a.StreamSeq, a.Stream, err) + } + kept.Header.Set(HeaderStream, a.Stream) + kept.Header.Set(HeaderConsumer, a.Consumer) + kept.Header.Set(HeaderSequence, strconv.FormatUint(a.StreamSeq, 10)) + kept.Header.Set(HeaderDeliveries, strconv.FormatUint(a.Deliveries, 10)) + kept.Header.Set(HeaderGaveUp, gaveUp.UTC().Format(time.RFC3339Nano)) + // One copy per message given up, however often its notice is taken, by whichever keeper takes it. + kept.Header.Set(nats.MsgIdHdr, fmt.Sprintf("%s.%s.%d", a.Stream, a.Consumer, a.StreamSeq)) + ack, err := js.PublishMsg(ctx, kept) + if err != nil { + return DeadLetter{}, fmt.Errorf("message %d of %s could not be kept in %s: %w", a.StreamSeq, a.Stream, buscompose.DeadLettersStream, err) + } + return Of(ack.Sequence, kept.Subject, kept.Header, kept.Data, false), nil +} + +// Of reads a kept message; whole gives its body and its own headers. +func Of(id uint64, subject string, h nats.Header, data []byte, whole bool) DeadLetter { + d := DeadLetter{ID: id, Stream: h.Get(HeaderStream), Consumer: h.Get(HeaderConsumer), Subject: h.Get(HeaderSubject), + Lost: h.Get(HeaderLost), Size: len(data)} + if d.Stream == "" || d.Consumer == "" { + d.Stream, d.Consumer, _ = buscompose.DeadLetterOf(subject) + } + d.Who = ConsumerInWords(d.Stream, d.Consumer) + d.Sequence, _ = strconv.ParseUint(h.Get(HeaderSequence), 10, 64) + d.Deliveries, _ = strconv.ParseUint(h.Get(HeaderDeliveries), 10, 64) + d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderGaveUp)) + d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderPublished)) + d.Restricted = h.Get(HeaderRestricted) == "true" + d.WorkQueue = h.Get(HeaderWorkQueue) == "true" + d.AgesOut, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderAgesOut)) + if whole { + d.Body = string(data) + d.Headers = map[string][]string{} + for k, v := range h { + if !strings.HasPrefix(k, "Mesh-Dead-") && !strings.HasPrefix(k, "Nats-") { + d.Headers[k] = append([]string(nil), v...) + } + } + } + return d +} + +// Held is how many messages DEAD_LETTERS holds for each consumer, by `.`. +func Held(ctx context.Context, js jetstream.JetStream) (map[string]int, error) { + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + info, err := s.Info(ctx, jetstream.WithSubjectFilter(">")) + if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + held := map[string]int{} + for subject, n := range info.State.Subjects { + if stream, consumer, ok := buscompose.DeadLetterOf(subject); ok && n > 0 { + held[stream+"."+consumer] += int(n) + } + } + return held, nil +} + +// List is what DEAD_LETTERS holds, newest first, at most most (all when most is not positive); for one consumer when +// consumer names one (its name, or `.`). A restricted one's body is withheld. The total is the stream's +// own count. +func List(ctx context.Context, js jetstream.JetStream, consumer string, most int) ([]DeadLetter, int, error) { + held, err := Held(ctx, js) + if err != nil { + return nil, 0, err + } + total := 0 + for key, n := range held { + if consumer == "" || key == consumer || strings.HasSuffix(key, "."+consumer) { + total += n + } + } + if total == 0 { + return nil, 0, nil + } + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return nil, 0, err + } + info, err := s.Info(ctx) + if err != nil { + return nil, 0, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + var out []DeadLetter + for seq := info.State.LastSeq; seq >= info.State.FirstSeq && seq > 0; seq-- { + if most > 0 && len(out) >= most || len(out) >= total { + break + } + raw, err := s.GetMsg(ctx, seq) + if errors.Is(err, jetstream.ErrMsgNotFound) { + continue + } + if err != nil { + return out, total, fmt.Errorf("dead letter %d cannot be read: %w", seq, err) + } + d := Of(seq, raw.Subject, raw.Header, raw.Data, false) + if consumer != "" && consumer != d.Consumer && consumer != d.Stream+"."+d.Consumer { + continue + } + if d.Restricted { + d.Withheld = WithheldWords + } + out = append(out, d) + } + return out, total, nil +} + +// ErrNoDeadLetter is an id DEAD_LETTERS does not hold. +var ErrNoDeadLetter = errors.New("no such dead letter") + +// Named is one kept message whole, but for a restricted one, whose body and own headers are never shown. +func Named(ctx context.Context, js jetstream.JetStream, id uint64) (DeadLetter, error) { + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return DeadLetter{}, err + } + raw, err := s.GetMsg(ctx, id) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return DeadLetter{}, fmt.Errorf("%w: %s holds no message %d — delivered again or dropped already, or never kept; "+ + "dead-letters lists what it holds", ErrNoDeadLetter, buscompose.DeadLettersStream, id) + } + if err != nil { + return DeadLetter{}, fmt.Errorf("dead letter %d cannot be read: %w", id, err) + } + d := Of(id, raw.Subject, raw.Header, raw.Data, true) + if d.Restricted { + d.Body, d.Headers, d.Withheld = "", nil, WithheldWords + } + return d, nil +} + +// Refusal is why the watchman delivers nothing again and drops nothing: before the lease write the controller keeps them +// and its verb acts; from it the act waits for the hand-act log's home (issue 1953), where a person's act is recorded. +// act is deliver or drop. +func Refusal(handedAway bool, act string) error { + done := map[string]string{"deliver": "delivered again", "drop": "dropped"}[act] + if done == "" { + return fmt.Errorf("%q is not deliver or drop", act) + } + if !handedAway { + return fmt.Errorf("the controller still holds its lease and keeps the dead letters: %s it with "+ + "mesh-controller.dead-letters (novox/hq ADR 0564 §5); nothing is %s here", act, done) + } + return fmt.Errorf("a dead letter is %s only as a person's act recorded in the hand-act log, whose home is not "+ + "delivered yet (issue 1953): nothing is %s", done, done) +} + +// Gate is whether the watchman takes the notices itself: only when the lease names no controller (handedAway) and the +// controller's consumer is gone. waiting says the lease is handed away while `controller` still stands, which is the +// watchman's finding: each notice waits in the queue until the lease write's step removes it. +func Gate(handedAway, controllerStands bool) (take, waiting bool) { + return handedAway && !controllerStands, handedAway && controllerStands +} + +// ControllerStands asks the bus whether the controller's consumer on DEAD_LETTER_NOTICES stands. +func ControllerStands(ctx context.Context, js jetstream.JetStream) (bool, error) { + _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, buscompose.ControllerName) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + return false, nil + } + return err == nil, err +} + +// TakeNotices makes the watchman's own durable consumer on DEAD_LETTER_NOTICES (never touching `controller`) and keeps +// each notice it takes; a notice not kept is handed back to be taken again. kept hears each one kept. +func TakeNotices(ctx context.Context, js jetstream.JetStream, kept func(DeadLetter), logf func(string, ...any)) (jetstream.ConsumeContext, error) { + c, err := js.CreateOrUpdateConsumer(ctx, buscompose.DeadLetterNoticesStream, jetstream.ConsumerConfig{ + Durable: Consumer, AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Minute}) + if err != nil { + return nil, fmt.Errorf("the consumer %s on %s cannot be made: %w", Consumer, buscompose.DeadLetterNoticesStream, err) + } + return c.Consume(func(m jetstream.Msg) { + d, err := Keep(ctx, js, m.Data()) + if err != nil { + logf("a dead-letter notice was not kept, offered again in a minute: %v", err) + _ = m.NakWithDelay(time.Minute) + return + } + _ = m.Ack() + kept(d) + }) +} diff --git a/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go b/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go new file mode 100644 index 0000000..2926eca --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go @@ -0,0 +1,64 @@ +package deadletters + +import ( + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// 1952/WM5, ADR 0564 §5: the watchman makes no consumer while `controller` exists or the lease names a controller, makes +// one once both are false, and says the wait meanwhile; it delivers and drops nothing, naming who does; and a kept message +// reads as the controller's copies do, a restricted one's age and work queue kept. +func TestTheWatchmanKeepsTheDeadLettersOnlyFromTheLeaseWrite(t *testing.T) { + for _, c := range []struct { + handedAway, controllerStands, take, waiting bool + }{ + {false, true, false, false}, + {false, false, false, false}, + {true, true, false, true}, + {true, false, true, false}, + } { + take, waiting := Gate(c.handedAway, c.controllerStands) + if take != c.take || waiting != c.waiting { + t.Errorf("Gate(%v, %v) = %v, %v", c.handedAway, c.controllerStands, take, waiting) + } + } + for _, c := range []struct { + away bool + act string + says string + }{ + {false, "deliver", "mesh-controller.dead-letters"}, + {false, "drop", "nothing is dropped"}, + {true, "drop", "issue 1953"}, + {true, "deliver", "delivered again"}, + {true, "send", "not deliver or drop"}, + } { + if err := Refusal(c.away, c.act); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("Refusal(%v, %s) = %v, want it to say %q", c.away, c.act, err, c.says) + } + } + at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + h := nats.Header{} + h.Set(HeaderStream, "EVENTS") + h.Set(HeaderConsumer, "ace_web") + h.Set(HeaderSequence, "41") + h.Set(HeaderDeliveries, "5") + h.Set(HeaderGaveUp, at.Format(time.RFC3339Nano)) + h.Set(HeaderRestricted, "true") + h.Set(HeaderWorkQueue, "true") + h.Set(HeaderAgesOut, at.Add(time.Hour).Format(time.RFC3339Nano)) + h.Set("X-Own", "kept") + h.Set("Nats-Msg-Id", "EVENTS.ace_web.41") + d := Of(9, "mesh.events.dead.EVENTS.ace_web", h, []byte("body"), true) + if d.ID != 9 || d.Sequence != 41 || d.Deliveries != 5 || !d.GaveUp.Equal(at) || !d.Restricted || !d.WorkQueue || + !d.AgesOut.Equal(at.Add(time.Hour)) || d.Who != "how web on ace hears what it consumes" || + d.Body != "body" || len(d.Headers) != 1 || d.Headers["X-Own"][0] != "kept" { + t.Fatalf("dead letter = %+v", d) + } + if d := Of(10, "mesh.events.dead.SEAT_X.y_worker", nats.Header{}, nil, false); d.Stream != "SEAT_X" || d.Consumer != "y_worker" || d.Body != "" { + t.Fatalf("a copy without headers reads its stream and consumer from its subject: %+v", d) + } +} diff --git a/modules/mesh-delivery-watchman/internal/socket/socket.go b/modules/mesh-delivery-watchman/internal/socket/socket.go new file mode 100644 index 0000000..42cf799 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/socket/socket.go @@ -0,0 +1,125 @@ +// Package socket is how the watchman's tools ask its program what only the program reads (novox/hq ADR 0564 §12, as the +// view's tools ask the view, mesh-delivery-view internal/view/serve.go): DEAD_LETTERS is a stream, which the tool runtime +// grants no bundle, and resync is an act of the program. One line of JSON each way, on a unix socket on the program's +// machine; the socket keeps nothing. +package socket + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +// Path is where the program listens unless MESH_WATCHMAN_SOCKET says otherwise. +const Path = "/var/lib/mesh-delivery-watchman-tools/tools.sock" + +// PathOf is the socket's path as getenv says it. +func PathOf(getenv func(string) string) string { + if p := strings.TrimSpace(getenv("MESH_WATCHMAN_SOCKET")); p != "" { + return p + } + return Path +} + +// The asks the program answers. +const ( + AskDeadLetters = "dead-letters" + AskResync = "resync" +) + +// Ask is one ask of the tools: dead-letters (consumer, limit, id; deliver or drop with why) or resync. +type Ask struct { + Ask string `json:"ask"` + Consumer string `json:"consumer,omitempty"` + Limit int `json:"limit,omitempty"` + ID uint64 `json:"id,omitempty"` + Act string `json:"act,omitempty"` + Why string `json:"why,omitempty"` +} + +// Listen listens on the unix socket at path, in a directory of mode 0755, the socket 0666. +func Listen(path string) (net.Listener, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + l, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + if err := os.Chmod(path, 0o666); err != nil { + l.Close() + return nil, err + } + return l, nil +} + +// Serve answers each ask on l with answer until ctx ends. +func Serve(ctx context.Context, l net.Listener, answer func(Ask) (any, error)) error { + go func() { <-ctx.Done(); l.Close() }() + for { + c, err := l.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + go func() { + defer c.Close() + _ = c.SetDeadline(time.Now().Add(30 * time.Second)) + line, err := bufio.NewReader(io.LimitReader(c, 4096)).ReadBytes('\n') + if err != nil && len(line) == 0 { + return + } + var a Ask + enc := json.NewEncoder(c) + if json.Unmarshal(line, &a) != nil { + _ = enc.Encode(map[string]any{"error": "refused: an ask is one line of JSON"}) + return + } + out, err := answer(a) + if err != nil { + _ = enc.Encode(map[string]any{"error": err.Error()}) + return + } + _ = enc.Encode(map[string]any{"answer": out}) + }() + } +} + +// On asks the program at path and returns its answer. +func On(path string, a Ask) (json.RawMessage, error) { + c, err := net.DialTimeout("unix", path, 5*time.Second) + if err != nil { + return nil, fmt.Errorf("cannot reach the watchman at %s: %w", path, err) + } + defer c.Close() + _ = c.SetDeadline(time.Now().Add(30 * time.Second)) + line, _ := json.Marshal(a) + if _, err := c.Write(append(line, '\n')); err != nil { + return nil, err + } + var reply struct { + Answer json.RawMessage `json:"answer"` + Error string `json:"error"` + } + if err := json.NewDecoder(c).Decode(&reply); err != nil { + return nil, fmt.Errorf("the watchman's answer cannot be read: %w", err) + } + if reply.Error != "" { + return nil, errors.New(reply.Error) + } + return reply.Answer, nil +} diff --git a/modules/mesh-delivery-watchman/internal/watch/advisories.go b/modules/mesh-delivery-watchman/internal/watch/advisories.go new file mode 100644 index 0000000..d4d0057 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/advisories.go @@ -0,0 +1,56 @@ +package watch + +import ( + "encoding/json" + "fmt" + "strings" +) + +// The bus's advisories the watchman hears (S9, mesh-controller internal/link/advisories.go at e04846e9): a durable consumer +// the mesh defines deleted is `consumer-lost`. A message given up on is said from DEAD_LETTERS, where it is kept, for as +// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects end +// with it (ADR 0564 §4). + +// Advisory subjects the watchman subscribes. +const ( + ConsumerDeletedAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>" + MaxDeliveriesAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>" +) + +// KindConsumerLost is a durable consumer of the mesh deleted. +const KindConsumerLost = "consumer-lost" + +// ReadAdvisory is one JetStream advisory as the mesh says it; false for one the watchman does not say. who words a +// consumer. +func ReadAdvisory(subject string, body []byte, who func(stream, consumer string) string) (Advisory, bool) { + var a struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + } + if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" { + return Advisory{}, false + } + if !strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.DELETED.") || !MeshNamed(a.Stream, a.Consumer) { + return Advisory{}, false + } + return Advisory{Kind: KindConsumerLost, ID: a.Stream + "." + a.Consumer, Machine: consumerMachine(a.Stream, a.Consumer), + Said: fmt.Sprintf("%s was deleted from the bus", who(a.Stream, a.Consumer))}, true +} + +// MeshNamed says a consumer is one of the durable consumers the mesh defines, by its name's shape; every other is a +// reader's own, named at random and deleted when its read is done. +func MeshNamed(stream, name string) bool { + switch { + case strings.HasPrefix(stream, "KV_") || stream == "ASSIGNMENTS": + return false + case stream == "NODES": + return true + case strings.HasPrefix(stream, "SEAT_"): + return strings.HasSuffix(name, "_worker") + case name == "controller" || name == "controller-merges": + return true + case stream == "EVENTS": + return strings.Contains(name, "_") + } + return false +} diff --git a/modules/mesh-delivery-watchman/internal/watch/dead.go b/modules/mesh-delivery-watchman/internal/watch/dead.go new file mode 100644 index 0000000..8919dd6 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/dead.go @@ -0,0 +1,78 @@ +package watch + +import ( + "fmt" + "sort" + "strings" +) + +// The dead letters' findings (novox/hq ADR 0564 §5, issue 330): a consumer DEAD_LETTERS holds a message for is said for as +// long as it holds any, `bus...max-deliveries`, and the notices waiting behind the controller's consumer +// after the lease write are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`. + +// KindNoticesWaiting is the finding of notices waiting while the controller's consumer stands after the lease write. +const KindNoticesWaiting = "notices-waiting" + +// DeadLettersHeld is DEAD_LETTERS' count per consumer, `.`, read whole: each consumer held writes its +// finding, each one no longer held has it deleted. who words a consumer. +func (w *Watchman) DeadLettersHeld(held map[string]int, who func(stream, consumer string) string) { + w.mu.Lock() + defer w.mu.Unlock() + want := map[string]bool{} + keys := make([]string, 0, len(held)) + for k := range held { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + n := held[key] + stream, consumer, ok := strings.Cut(key, ".") + if !ok || n <= 0 { + continue + } + messages, them := "a message", "it is" + if n > 1 { + messages, them = fmt.Sprintf("%d messages", n), "they are" + } + f := Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Source: "S9", + Machine: consumerMachine(stream, consumer), + Summary: fmt.Sprintf("%s gave up on %s; %s kept in DEAD_LETTERS until delivered again or dropped, with why", + who(stream, consumer), messages, them), + Said: fmt.Sprintf("%d held for %s", n, key), + Needs: "deliver them again or drop them, from the mesh MCP server."} + want[f.Key()] = true + w.find(f) + } + for key := range w.findings { + if strings.HasPrefix(key, ScopeBus+".") && strings.HasSuffix(key, "."+KindDeadLetters) && !want[key] { + w.unfind(key) + } + } +} + +// consumerMachine is the machine a module's consumer on EVENTS is on; empty for the others. +func consumerMachine(stream, consumer string) string { + if stream == "EVENTS" { + if node, _, ok := strings.Cut(consumer, "_"); ok { + return node + } + } + return "" +} + +// NoticesWaiting says whether the lease is handed away while the controller's consumer on DEAD_LETTER_NOTICES still +// stands: every notice then waits in the queue, kept a week, until the lease write's step removes the consumer. +func (w *Watchman) NoticesWaiting(waiting bool) { + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: "DEAD_LETTER_NOTICES.controller", Kind: KindNoticesWaiting, Severity: Warning, Source: "dead-letters", + Summary: "the lease names no controller but the controller's consumer on DEAD_LETTER_NOTICES still stands, so every " + + "notice of a message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS", + Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the lease write", + Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, the lease write's step."} + if waiting { + w.find(f) + return + } + w.unfind(f.Key()) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/finding.go b/modules/mesh-delivery-watchman/internal/watch/finding.go new file mode 100644 index 0000000..fce809b --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/finding.go @@ -0,0 +1,159 @@ +package watch + +import ( + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The watchman is an observer of the seat mesh-watchdogs (novox/hq ADR 0564 §2, §4): what it observes wrong it writes as a +// finding, one key `..` of its own bucket `findings` in mesh-sdk's layout (health.Finding), only when what +// it says changes, and it deletes the key when it no longer observes it. The condition keeper (mesh-conditions) raises and +// clears the conditions from them; the watchman writes no condition and reads none (Limits 11). + +// FindingsBucket is the watchman's findings. +var FindingsBucket = health.FindingsBucket(health.WatchmanModule) + +// Observer is the name a finding of the watchman carries. +const Observer = health.WatchmanModule + +// The scopes and severities a finding of the watchman names, as the controller's conditions name them, so the keeper's +// `compare` meets the same keys. +const ( + ScopeMachine = health.ScopeMachine + ScopeDelivery = health.ScopeDelivery + ScopeModule = health.ScopeModule + ScopeProvider = health.ScopeProvider + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + + Urgent = string(health.Urgent) + Warning = string(health.Warning) +) + +// Finding is what the watchman builds of one thing it observes wrong: its subject (scope, id, the machine and the others +// it concerns), kind and severity, its evidence (Summary, and Said with Since, its one observation), the words the +// operator reads (ADR 0253; empty leaves the keeper's wording of the kind), and the row or fact it comes from (Source). +// It is written as health.Finding. +type Finding struct { + Scope, ID, Token, Kind, Machine string + Also []string + Severity string + Summary, Said string + Headline, Explanation string + Needs, Resolved string + Source string + Observer string + // Since is when what it says began, as the watchman saw it: the finding's time and its observation's, so the same + // thing observed again writes the same bytes. + Since time.Time +} + +// FindingKey composes a finding's key as the controller composes a condition's (health.Key). +func FindingKey(scope, id, token string) string { return health.FindingKey(scope, id, token) } + +// Key is the finding's key: its token, or its kind when it has none. +func (f Finding) Key() string { + token := f.Token + if token == "" { + token = f.Kind + } + return FindingKey(f.Scope, f.ID, token) +} + +// EncodeFinding is f in mesh-sdk's layout, refusing one the keeper could not make a condition of. +func EncodeFinding(f Finding) ([]byte, error) { + said := f.Said + if said == "" { + said = f.Summary + } + return health.EncodeFinding(health.Finding{Key: f.Key(), Kind: f.Kind, Token: f.Token, + Subject: health.Subject{Scope: f.Scope, ID: f.ID, Machine: f.Machine, Also: f.Also}, + Severity: health.Severity(f.Severity), Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, + Resolved: f.Resolved, Needs: f.Needs, Evidence: []health.Evidence{{At: f.Since.UTC(), Said: said}}, + Source: f.Source, Observer: Observer, At: f.Since.UTC()}) +} + +// DecodeFinding reads a finding the watchman wrote. +func DecodeFinding(data []byte) (Finding, error) { + h, err := health.DecodeFinding(data) + if err != nil { + return Finding{}, err + } + f := Finding{Scope: h.Subject.Scope, ID: h.Subject.ID, Token: h.Token, Kind: h.Kind, Machine: h.Subject.Machine, + Also: h.Subject.Also, Severity: string(h.Severity), Summary: h.Summary, Headline: h.Headline, Explanation: h.Explanation, + Needs: h.Needs, Resolved: h.Resolved, Source: h.Source, Observer: h.Observer, Since: h.At} + if len(h.Evidence) > 0 { + f.Said = h.Evidence[0].Said + } + return f, nil +} + +// find writes a finding when what it says differs from what the bucket holds (ADR 0289 §1). +func (w *Watchman) find(f Finding) { + value, err := EncodeFinding(f) + if err != nil { + w.say("no finding: %v", err) + return + } + key := f.Key() + delete(w.seededFindings, key) + if string(w.findings[key]) == string(value) { + return + } + if err := w.out.PutFinding(key, value); err != nil { + w.say("cannot write the finding %s: %v", key, err) + return + } + w.findings[key] = value +} + +// unfind deletes a finding the watchman no longer observes. +func (w *Watchman) unfind(key string) { + if _, ok := w.findings[key]; !ok { + return + } + if err := w.out.DeleteFinding(key); err != nil { + w.say("cannot delete the finding %s: %v", key, err) + return + } + delete(w.findings, key) +} + +// SeedFinding tells the watchman a finding that stands in its bucket at its start; ReconcileFindings deletes it unless what +// the watchman reads at its start observes it again. +func (w *Watchman) SeedFinding(key string, value []byte) { + w.mu.Lock() + defer w.mu.Unlock() + w.findings[key] = value + w.seededFindings[key] = true +} + +// Findings is every finding the watchman holds now, by key. +func (w *Watchman) Findings() map[string]Finding { + w.mu.Lock() + defer w.mu.Unlock() + out := map[string]Finding{} + for k, v := range w.findings { + if f, err := DecodeFinding(v); err == nil { + out[k] = f + } + } + return out +} + +// Rewrite writes every finding the watchman holds again (resync, ADR 0564 §12): a bucket cleared or a key lost by hand is +// written back as the watchman observes it. It answers how many it wrote. +func (w *Watchman) Rewrite() int { + w.mu.Lock() + defer w.mu.Unlock() + n := 0 + for key, value := range w.findings { + if err := w.out.PutFinding(key, value); err != nil { + w.say("cannot write the finding %s again: %v", key, err) + continue + } + n++ + } + return n +} diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go new file mode 100644 index 0000000..030d477 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -0,0 +1,200 @@ +package watch + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func haveOf(t *testing.T, phase delivery.Phase, health delivery.Health, detail string) []byte { + t.Helper() + b, err := delivery.EncodeHave(delivery.Have{FP: fpNew, Phase: phase, Health: health, Detail: detail}) + if err != nil { + t.Fatal(err) + } + return b +} + +// expire hears the expiry of the pulse at pk. +func expire(w *Watchman, c *clock, pk string, after time.Duration) { + c.t = c.t.Add(after) + w.Pulse(pk, Expired, c.t) +} + +// The watchman's findings (novox/hq ADR 0564 §2, §4, §5, issue 1952 WM2-WM5): per row of the signals table a key not renewed +// writes its finding and a renewed one none; the haves' rows; stuck keys and presence as its own findings; the dead letters. +func TestTheWatchmansFindings(t *testing.T) { + cases := []struct { + name string + run func(t *testing.T, w *Watchman, o *fakeOut, c *clock) + // want are the finding keys standing at the end, with their severity. + want map[string]string + }{ + {"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Step("go", "n1.ask1", runningStep("ask1", t0), Put) + expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second) + k := FindingKey(ScopeDelivery, stuckKeyOf(t, checksKey, "ask1"), KindStuck) + if f := o.findings[k]; f.Kind != KindStuck || !strings.Contains(f.Summary, "the ask ask1 to end on n1") { + t.Fatalf("findings = %v", o.findings) + } + w.Step("go", "n1.ask1", endedStep("ask1", delivery.StepPassed, t0), Put) + }, map[string]string{}}, + {"WM2 S1: a node offline past its bound is silent, and back online it is not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") + w.Disconnect("node.ace") + expire(w, c, pulseKeyOf(t, presenceNS, "ace"), 120*time.Second) + if _, ok := o.findings["machine.ace.silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + w.Connect("node.ace") + }, map[string]string{}}, + {"WM2 S1: a node that said it sleeps is not silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.lap") + w.Power("lap", "sleeping") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + }, map[string]string{}}, + {"WM2 S1: a node back from sleep and still offline is silent after its bound", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.lap") + w.Power("lap", "sleeping") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + w.Power("lap", "woke") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + }, map[string]string{"machine.lap.silent": Warning}}, + {"WM2 S11: a tool runner not heard for three intervals is silent; a heartbeat ends it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.ToolsAlive("ace", 0) + pk := signalKey("S11", "ace") + if o.ttls[pk] != 3*HeartbeatEvery { + t.Fatalf("ttl = %v", o.ttls[pk]) + } + expire(w, c, pk, 3*HeartbeatEvery) + if _, ok := o.findings["machine.ace.tools-silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + w.ToolsAlive("ace", 20*time.Second) + if o.ttls[pk] != time.Minute { + t.Fatalf("ttl = %v, want three of the 20 s it says", o.ttls[pk]) + } + w.ToolsAlive("g14", 0) // renewed, never expired: no finding + }, map[string]string{}}, + {"WM2 S10: the self-check not heard for three runs is urgent; not before its first run", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + if len(o.pulses) != 0 { + t.Fatal("a pulse before the first run") + } + w.Checked("novox") + expire(w, c, signalKey("S10", "doctor"), 3*SelfCheckEvery) + }, map[string]string{"core.doctor.self-check-silent": Urgent}}, + {"WM3 S8: a failing word is its finding; not said again for 30 min it is silent; recovered ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + st, err := ReadStanding("mesh.mod.keycloak.event.provisioner.failing", + []byte(`{"provider":"oidc","provider-node":"ace","consumer":"photos","node":"g14","class":"refused","attempts":4,"since":"2026-10-18T11:00:00Z"}`)) + if err != nil { + t.Fatal(err) + } + w.Stood(st) + if f := o.findings["provider.keycloak.ace.photos.failing"]; f.Kind != KindProviderFailing || len(f.Also) != 1 { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S8", "keycloak.ace.photos"), ProviderSaysAgainWithin) + if _, ok := o.findings["provider.keycloak.ace.photos.silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + st.Failing = false + w.Stood(st) + }, map[string]string{}}, + {"WM3 S9: an advisory is a finding counted, ended by an hour's quiet", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + a, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.EVENTS.ace_web", []byte(`{"stream":"EVENTS","consumer":"ace_web"}`), + func(s, c string) string { return c + " on " + s }) + if !ok { + t.Fatal("a mesh consumer's deletion is not read") + } + if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.KV_x.abc", []byte(`{"stream":"KV_x","consumer":"abc"}`), nil); ok { + t.Fatal("a reader's own consumer is said") + } + w.Advised(a) + w.Advised(a) + k := "bus.EVENTS.ace_web.consumer-lost" + if f := o.findings[k]; !strings.Contains(f.Summary, "2 times") || f.Machine != "ace" { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S9", k), AdvisoryQuiet) + }, map[string]string{}}, + {"WM3 S21: a have failed for one apply interval is apply-failing; applied again it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.web", haveOf(t, delivery.PhaseFailed, delivery.HealthUnknown, "refused: the package manager (pacman) does not answer"), Put) + expire(w, c, signalKey("S21", "ace.web"), ApplyEvery) + if f := o.findings["machine.ace.web.apply-failing"]; !strings.Contains(f.Said, "pacman") { + t.Fatalf("findings = %v", o.findings) + } + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseFailed, delivery.HealthUnknown, "refused"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + expire(w, c, signalKey("S21", "ace.mail"), ApplyEvery) // applied before its bound: nothing + }, map[string]string{}}, + {"WM3: unhealthy only at the second write in a row, urgent after four hours", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed"), Put) + if len(o.findings) != 0 { + t.Fatalf("one write raised %v", o.findings) + } + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed"), Put) + if o.findings["module.web.ace.unhealthy"].Severity != Warning { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) + }, map[string]string{"module.web.ace.unhealthy": Urgent, "module.mail.ace.needs-operator": Warning}}, + {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "waits"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put) + }, map[string]string{}}, + {"WM4: resync after a cleared bucket writes the same findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "waits"), Put) + o.findings = map[string]Finding{} + if n := w.Rewrite(); n != 1 { + t.Fatalf("rewrote %d", n) + } + }, map[string]string{"module.mail.ace.needs-operator": Warning}}, + {"WM4: a finding read at the start and not observed again is deleted; a signal's stays", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for _, f := range []Finding{ + {Scope: ScopeDelivery, ID: "x.y", Kind: KindStuck, Severity: Warning, Summary: "old", Source: "stuck"}, + {Scope: ScopeMachine, ID: "ace", Kind: KindToolsSilent, Severity: Warning, Summary: "silent", Source: "S11"}, + } { + v, _ := EncodeFinding(f) + o.findings[f.Key()], _ = DecodeFinding(v) + w.SeedFinding(f.Key(), v) + } + w.Reconcile() + }, map[string]string{"machine.ace.tools-silent": Warning}}, + {"WM5: a consumer DEAD_LETTERS holds for is its finding until it holds none", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + who := func(s, c string) string { return c } + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 2, "SEAT_X.y_worker": 1}, who) + if f := o.findings["bus.EVENTS.ace_web.max-deliveries"]; f.Machine != "ace" || !strings.Contains(f.Summary, "2 messages") { + t.Fatalf("findings = %v", o.findings) + } + w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who) + w.NoticesWaiting(true) + }, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + tc.run(t, w, o, c) + got := map[string]string{} + for k, f := range o.findings { + got[k] = f.Severity + if f.Key() != k || f.Observer != Observer { + t.Errorf("finding %s = %+v", k, f) + } + } + if len(got) != len(tc.want) { + t.Fatalf("findings = %v, want %v", got, tc.want) + } + for k, sev := range tc.want { + if got[k] != sev { + t.Fatalf("findings = %v, want %v", got, tc.want) + } + } + }) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index 78e54c5..4eee236 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -116,6 +116,7 @@ func (w *Watchman) Have(key string, value []byte, op Op) { return case Delete: delete(w.haves, key) + w.haveFindings(key, node, module, nil) return } have, err := delivery.DecodeHave(value) @@ -124,6 +125,7 @@ func (w *Watchman) Have(key string, value []byte, op Op) { return } w.haves[key] = have.FP + w.haveFindings(key, node, module, &have) wk, err := delivery.WantKey(node, module) if err != nil { return diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go new file mode 100644 index 0000000..949d6d5 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -0,0 +1,524 @@ +package watch + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The rows of the signals table the watchman holds (novox/hq ADR 0564 §4, Limits 3 and 11). Each is a key of the pulse +// bucket whose time to live is the row's bound, renewed by the signal's own event: its expiry is the finding, so no loop +// looks every half minute. A signal's pulse is kept under the pulse bucket's pseudo-bucket `signal`, `signal..`, +// and its value says what its expiry does (signalDue), so a pulse that stood across a restart still acts when it expires. +// +// What each row hears is on the bus, never in the controller's store: +// - S1, a machine silent: the watchman's own presence of `node.` (its stuck key), not while the machine said it +// sleeps or shuts down (the power seat's events, ADR 0211); +// - S8, a provider's failing word repeated: the providers' `provisioner.failing` and `provisioner.recovered` (ADR 0224); +// the failing word is itself the finding `provider-failing`, and its pulse of 30 minutes, renewed by each repeat, is S8; +// - S9, the bus's advisories: each occurrence a finding, cleared by the expiry of an hour's pulse renewed by the next; +// - S10, the self-check's `checked`: a pulse of three runs' length, from the first run heard; +// - S11, a machine's tool runner heartbeat on `mesh.control..tools-alive`: a pulse of three intervals; +// - S21, a module's have failed: a pulse of one apply interval from the first failure, whose expiry while it still fails +// is the finding `apply-failing`; +// - a module's have unhealthy in two writes in a row, `unhealthy`, urgent after four hours; and a have that waits for +// the operator, `needs-operator` (ADR 0240 §12, §17). + +// The bounds of the rows. +const ( + // ProviderSaysAgainWithin is how long a provider's failing word stays current without being said again (S8): twice + // the quarter of an hour it repeats it at. + ProviderSaysAgainWithin = 30 * time.Minute + // AdvisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9). + AdvisoryQuiet = time.Hour + // SelfCheckEvery is the self-check's run interval; S10's bound is three of them. + SelfCheckEvery = 5 * time.Minute + // HeartbeatEvery is a tool runner's interval when it says none; S11's bound is three of them. + HeartbeatEvery = 60 * time.Second + // ApplyEvery is one apply interval (S21): a failure still there after it is not one the next apply clears. + ApplyEvery = 5 * time.Minute + // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. + UnhealthyUrgentAfter = 4 * time.Hour +) + +// signalBucket is the pseudo-bucket of a signal's pulse key. +const signalBucket = "signal" + +// signalDue is what a signal's pulse does when it expires: raise the finding (find), or end it (end, an advisory's quiet +// hour). Kept as the pulse's Owes, so it is read back at a restart. +type signalDue struct { + Do string `json:"do"` + Finding Finding `json:"finding"` + // While names what must still stand at the expiry for a find to be written: a have's key still failing or unhealthy; + // empty for a row whose expiry alone is the finding. + While string `json:"while,omitempty"` +} + +const ( + dueFind = "find" + dueEnd = "end" +) + +// signalKey is the pulse key of a row's signal about id. +func signalKey(row, id string) string { + parts := []string{signalBucket, row} + for _, p := range strings.Split(id, ".") { + if p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_"); p == "" { + p = "_" + } + parts = append(parts, p) + } + return strings.Join(parts, ".") +} + +// unsafeKey is what a key's token may not hold: the bus takes letters, digits and `-_/=`. +var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) + +// signal writes (or renews) the pulse of a row's signal, whose expiry does due. +func (w *Watchman) signal(row, id string, ttl time.Duration, due signalDue) { + pk := signalKey(row, id) + owes, err := json.Marshal(due) + if err != nil { + w.say("no signal %s: %v", pk, err) + return + } + value, err := delivery.EncodePulse(delivery.Pulse{Bucket: signalBucket, Key: strings.TrimPrefix(pk, signalBucket+"."), Owes: string(owes)}) + if err != nil { + w.say("no signal %s: %v", pk, err) + return + } + if err := w.out.PutPulse(pk, value, ttl); err != nil { + w.say("cannot write the signal %s: %v", pk, err) + return + } + w.signals[pk] = due +} + +// unsignal ends a row's signal about id: its pulse deleted, so it never expires. +func (w *Watchman) unsignal(row, id string) { + pk := signalKey(row, id) + if _, ok := w.signals[pk]; !ok { + return + } + delete(w.signals, pk) + if err := w.out.DeletePulse(pk); err != nil { + w.say("cannot delete the signal %s: %v", pk, err) + } +} + +// SeedSignal tells the watchman a signal's pulse that stands at its start, with its value: it is kept, since its event +// renews it and its expiry still acts. +func (w *Watchman) SeedSignal(key string, value []byte) { + w.mu.Lock() + defer w.mu.Unlock() + if due, ok := dueOf(value); ok { + w.signals[key] = due + } +} + +func dueOf(value []byte) (signalDue, bool) { + p, err := delivery.DecodePulse(value) + if err != nil || p.Bucket != signalBucket { + return signalDue{}, false + } + var due signalDue + if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd) { + return signalDue{}, false + } + return due, true +} + +// signalExpired is a signal's pulse expired: its due is done. +func (w *Watchman) signalExpired(pk string) { + due, ok := w.signals[pk] + if !ok { + return + } + delete(w.signals, pk) + switch due.Do { + case dueEnd: + w.unfind(due.Finding.Key()) + case dueFind: + if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { + return + } + w.find(due.Finding) + } +} + +// stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for +// unhealthy. +func (w *Watchman) stillWrong(haveKey, kind string) bool { + h, ok := w.haveState[haveKey] + if !ok { + return false + } + switch kind { + case KindApplyFailing: + return h.Phase == delivery.PhaseFailed + case KindUnhealthy: + return h.Health == delivery.HealthFailed + } + return false +} + +// The kinds the watchman's rows write, as the controller's conditions name them. +const ( + KindSilent = "silent" + KindStuck = "stuck" + KindProviderFailing = "provider-failing" + KindProviderSilent = "provider-silent" + KindSelfCheckSilent = "self-check-silent" + KindToolsSilent = "tools-silent" + KindApplyFailing = "apply-failing" + KindUnhealthy = "module-unhealthy" + KindNeedsOperator = "needs-operator" + KindDeadLetters = "max-deliveries" +) + +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } + +// ---- S1 and the power seat ---- + +// PowerEvents are the power seat's events that say whether a machine is about to be away or is back (ADR 0211). +var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true} + +// Power is a machine's power event: sleeping and shutting-down say it is away, booted and woke that it is back. A machine +// that said it is away is not silent; one back that is still offline is watched again from now. +func (w *Watchman) Power(node, state string) { + if !PowerEvents[state] || node == "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + away := state == "sleeping" || state == "shutting-down" + w.away[node] = away + key, err := delivery.PresenceNodeKey(node) + if err != nil { + return + } + if away { + w.unfind(FindingKey(ScopeMachine, node, KindSilent)) + return + } + if p, ok := w.presence[key]; ok && !p.online { + p.since = w.now() + w.presence[key] = p + w.ensureOffline(key) + } +} + +// stuckFinding is the finding of a stuck key as it stands (ADR 0433 §2, §3, now the watchman's own): a node's presence not +// moved is S1, the machine silent, unless it said it is away; any other key not moved is `delivery..stuck`; a key +// moved ends its finding. +func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { + if s.Bucket == delivery.PresenceBucket { + if node, part, _, err := delivery.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { + key := FindingKey(ScopeMachine, node, KindSilent) + if s.Moved || w.away[node] { + w.unfind(key) + return + } + w.find(Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", + Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), delivery.PulseMoveBound), + Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}) + return + } + } + key := FindingKey(ScopeDelivery, sk, KindStuck) + if s.Moved { + w.unfind(key) + return + } + waited := s.Waited + if waited == "" { + waited = "a move" + } + w.find(Finding{Scope: ScopeDelivery, ID: sk, Kind: KindStuck, Severity: Warning, Source: "stuck", + Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", sk, stamp(s.Since), waited), + Said: fmt.Sprintf("%s: waiting for %s since %s", sk, waited, s.Since.UTC().Format(time.RFC3339)), Since: s.Since}) +} + +// ---- S8 ---- + +// Standing is a provider's word about one consumer (ADR 0224): Module is the emitter, read from the subject. +type Standing struct { + Module string `json:"-"` + Failing bool `json:"-"` + Provider string `json:"provider"` + ProviderNode string `json:"provider-node"` + Consumer string `json:"consumer"` + Node string `json:"node"` + Class string `json:"class,omitempty"` + Error string `json:"error,omitempty"` + Since time.Time `json:"since"` + Attempts int `json:"attempts"` + Why string `json:"why,omitempty"` +} + +// ReadStanding reads a provider's word from its subject, `mesh.mod..event.provisioner.failing` or `.recovered`. +func ReadStanding(subject string, body []byte) (Standing, error) { + rest, ok := strings.CutPrefix(subject, "mesh.mod.") + if !ok { + return Standing{}, fmt.Errorf("%s is not a provider's word", subject) + } + module, event, ok := strings.Cut(rest, ".event.") + if !ok || module == "" || (event != "provisioner.failing" && event != "provisioner.recovered") { + return Standing{}, fmt.Errorf("%s is not a provider's word", subject) + } + var st Standing + if err := json.Unmarshal(body, &st); err != nil { + return Standing{}, fmt.Errorf("%s's word could not be read: %w", module, err) + } + if st.Consumer == "" { + return Standing{}, fmt.Errorf("%s's word names no consumer", module) + } + st.Module, st.Failing = module, event == "provisioner.failing" + return st, nil +} + +// Stood is a provider's word: failing writes `provider....failing` and renews S8's pulse, whose +// expiry writes `provider-silent`; recovered deletes both (Limits 11). +func (w *Watchman) Stood(st Standing) { + w.mu.Lock() + defer w.mu.Unlock() + id := st.Module + "." + st.ProviderNode + "." + st.Consumer + failing := FindingKey(ScopeProvider, id, "failing") + silentKey := FindingKey(ScopeProvider, id, "silent") + if !st.Failing { + w.unfind(failing) + w.unfind(silentKey) + w.unsignal("S8", id) + return + } + class := st.Class + if class == "" { + class = "failing" + } + whom := st.Consumer + if st.Node != "" { + whom += " on " + st.Node + } + said := class + if st.Error != "" { + said += ": " + firstLine(st.Error) + } + var also []string + if st.Node != "" && st.Node != st.ProviderNode { + also = []string{st.Node} + } + w.find(Finding{Scope: ScopeProvider, ID: id, Token: "failing", Kind: KindProviderFailing, Machine: st.ProviderNode, Also: also, + Severity: Warning, Source: "provisioner.failing", Since: st.Since, Said: said, + Summary: fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode, whom, class, + st.Attempts, stamp(st.Since))}) + w.unfind(silentKey) + now := w.now() + w.signal("S8", id, ProviderSaysAgainWithin, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeProvider, ID: id, Token: "silent", + Kind: KindProviderSilent, Machine: st.ProviderNode, Severity: Warning, Source: "S8", Since: now, + Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped saying anything, so "+ + "its last word is all the mesh has", st.Module, st.ProviderNode, st.Consumer, stamp(now)), + Said: fmt.Sprintf("not said again within %s", ProviderSaysAgainWithin)}}) +} + +func firstLine(s string) string { + s, _, _ = strings.Cut(strings.TrimSpace(s), "\n") + return s +} + +// ---- S9 ---- + +// Advisory is one thing the bus said (S9): its kind (slow-consumer, connection-closed, max-deliveries, refused, +// consumer-lost), what it is about, its words, and the machine when it names one. +type Advisory struct { + Kind, ID, Said, Machine string +} + +// Advised is an advisory heard: its finding `bus..` is written at once, counting the times since its first, and +// an hour's pulse renewed by each, whose expiry ends it. +func (w *Watchman) Advised(a Advisory) { + if a.Kind == "" || a.ID == "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: a.ID, Kind: a.Kind, Machine: a.Machine, Severity: Warning, Source: "S9", Said: a.Said} + key := f.Key() + n := w.advised[key] + 1 + if _, open := w.findings[key]; !open { + n = 1 + w.advisedSince[key] = w.now() + } + w.advised[key] = n + since := w.advisedSince[key] + f.Since, f.Summary = since, a.Said + if n > 1 { + f.Summary = fmt.Sprintf("%s (%d times since %s)", a.Said, n, since.UTC().Format("15:04 MST")) + } + w.find(f) + w.signal("S9", key, AdvisoryQuiet, signalDue{Do: dueEnd, Finding: f}) +} + +// ---- S10 ---- + +// Checked is the self-check's `checked` heard: its pulse of three runs' length is renewed, and its silence ended. Before +// the first run heard there is no pulse, so a self-check not yet placed is not said silent. +func (w *Watchman) Checked(machine string) { + w.mu.Lock() + defer w.mu.Unlock() + w.unfind(FindingKey(ScopeCore, "doctor", KindSelfCheckSilent)) + now := w.now() + w.signal("S10", "doctor", 3*SelfCheckEvery, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeCore, ID: "doctor", + Kind: KindSelfCheckSilent, Machine: machine, Severity: Urgent, Source: "S10", Since: now, + Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's invariants are not "+ + "being checked", stamp(now), SelfCheckEvery), + Said: fmt.Sprintf("no run within %s", 3*SelfCheckEvery)}}) +} + +// ---- S11 ---- + +// ToolsAlive is a machine's tool runner heartbeat, saying its interval (zero: HeartbeatEvery): its pulse of three +// intervals is renewed and its silence ended. +func (w *Watchman) ToolsAlive(node string, every time.Duration) { + if node == "" { + return + } + if every <= 0 { + every = HeartbeatEvery + } + w.mu.Lock() + defer w.mu.Unlock() + w.unfind(FindingKey(ScopeMachine, node, KindToolsSilent)) + if w.away[node] { + w.unsignal("S11", node) + return + } + now := w.now() + w.signal("S11", node, 3*every, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindToolsSilent, + Machine: node, Severity: Warning, Source: "S11", Since: now, + Summary: fmt.Sprintf("the node tools on %s have not said they are there since %s (bound %s): nothing can ask that "+ + "machine anything", node, stamp(now), 3*every), + Said: fmt.Sprintf("silent for more than %s", 3*every)}}) +} + +// ToolsNode is the machine a tool runner heartbeat's subject names, `mesh.control..tools-alive`. +func ToolsNode(subject string) string { + rest, ok := strings.CutPrefix(subject, "mesh.control.") + if !ok { + return "" + } + node, ok := strings.CutSuffix(rest, ".tools-alive") + if !ok || strings.Contains(node, ".") { + return "" + } + return node +} + +// ---- S21, unhealthy and needs-operator, from the haves ---- + +// haveFindings reads a have's change for the rows the haves feed: S21 (failed for one apply interval), unhealthy (two +// writes in a row, urgent after four hours) and needs-operator (it waits for the operator). A have removed or deleted ends +// them all. +func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) { + applyKey := FindingKey(ScopeMachine, node+"."+module, KindApplyFailing) + unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") + waitsKey := FindingKey(ScopeModule, module+"."+node, KindNeedsOperator) + if h == nil || h.Phase == delivery.PhaseRemoved { + delete(w.haveState, key) + delete(w.unhealthy, key) + w.unsignal("S21", node+"."+module) + w.unsignal("unhealthy", module+"."+node) + w.unfind(applyKey) + w.unfind(unhealthyKey) + w.unfind(waitsKey) + return + } + was, had := w.haveState[key] + w.haveState[key] = *h + now := w.now() + + // S21: a have refused or applied in part says phase failed, with the node's words in its detail. + if h.Phase == delivery.PhaseFailed { + if !had || was.Phase != delivery.PhaseFailed { + w.signal("S21", node+"."+module, ApplyEvery, signalDue{Do: dueFind, While: key, Finding: Finding{Scope: ScopeMachine, + ID: node + "." + module, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: now, + Summary: fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", node, module, stamp(now), ApplyEvery), + Said: firstLine(orWords(h.Detail, "no reason given")), + Headline: node + " is not applying its update", Resolved: node + " applies what it is sent again"}}) + } + } else { + w.unsignal("S21", node+"."+module) + w.unfind(applyKey) + } + + // unhealthy: two writes in a row that say it failed. + if h.Health == delivery.HealthFailed { + w.unhealthy[key]++ + if w.unhealthy[key] == 2 { + f := Finding{Scope: ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: KindUnhealthy, Machine: node, + Severity: Warning, Source: "health", Since: now, + Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, orWords(h.Detail, "its check failed")), + Said: firstLine(orWords(h.Detail, "failed")), + Headline: fmt.Sprintf("%s not working on %s", module, node)} + w.find(f) + f.Severity = Urgent + w.signal("unhealthy", module+"."+node, UnhealthyUrgentAfter, signalDue{Do: dueFind, While: key, Finding: f}) + } + } else { + delete(w.unhealthy, key) + w.unsignal("unhealthy", module+"."+node) + w.unfind(unhealthyKey) + } + + // needs-operator: a part waits for the operator's secret or setting. + if h.Health == delivery.HealthWaits { + w.find(Finding{Scope: ScopeModule, ID: module + "." + node, Kind: KindNeedsOperator, Machine: node, Severity: Warning, + Source: "health", Since: now, + Summary: fmt.Sprintf("%s on %s waits for the operator: %s", module, node, orWords(h.Detail, "a part waits for a secret or a setting")), + Said: firstLine(orWords(h.Detail, "waits"))}) + } else { + w.unfind(waitsKey) + } +} + +func orWords(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +// SignalRows are the rows the watchman holds, in the table's order, for `signals`. +var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ + {"S1", "a machine's presence (not while it said it sleeps or shuts down)", delivery.PulseMoveBound.String(), KindSilent}, + {"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent}, + {"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"}, + {"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent}, + {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, + {"S21", "a have refused or applied in part", ApplyEvery.String(), KindApplyFailing}, + {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, + {"health", "a have that waits for the operator", "at once", KindNeedsOperator}, +} + +// Signal is one signal's pulse as `signals` shows it. +type Signal struct { + Key string `json:"key"` + Do string `json:"do"` + Finding Finding `json:"finding"` +} + +// Signals is every signal's pulse the watchman holds, sorted by key. +func (w *Watchman) Signals() []Signal { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]Signal, 0, len(w.signals)) + for k, d := range w.signals { + out = append(out, Signal{Key: k, Do: d.Do, Finding: d.Finding}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index e81613b..50a42e6 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -31,6 +31,9 @@ type Out interface { // DeletePulse deletes the pulse key (a delete, not an expiry). DeletePulse(key string) error PutStuck(key string, value []byte) error + // PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes. + PutFinding(key string, value []byte) error + DeleteFinding(key string) error } type presence struct { @@ -60,6 +63,20 @@ type Watchman struct { wants map[string]string // the fingerprint of each want that stands, by want key haves map[string]string // the fingerprint each node runs of a module, by have key bound time.Duration + // findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed + // again. + findings map[string][]byte + seededFindings map[string]bool + // signals is every signal's pulse standing, by pulse key, with what its expiry does. + signals map[string]signalDue + // away is each machine that said it sleeps or shuts down, until it says it is back. + away map[string]bool + // haveState is each module's have as last read, unhealthy its writes in a row that said it failed, advised each + // advisory's count while its finding stands and advisedSince since when. + haveState map[string]delivery.Have + unhealthy map[string]int + advised map[string]int + advisedSince map[string]time.Time // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -67,7 +84,9 @@ type Watchman struct { // New is a watchman writing to out, reading the time from now. func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, - pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound} + pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, + haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}} } // Logf sets where a write that failed is said; it is silent until set. @@ -178,6 +197,7 @@ func (w *Watchman) putStuck(sk string, s delivery.Stuck) { } s.V = delivery.Version w.stuck[sk] = s + w.stuckFinding(sk, s) } // Pulse is a change heard on the pulse bucket. Only the server's expiry marker of a pulse the watchman made writes a @@ -188,6 +208,10 @@ func (w *Watchman) Pulse(key string, op Op, at time.Time) { if op != Expired { return } + if _, ok := w.signals[key]; ok { + w.signalExpired(key) + return + } p, ok := w.pulses[key] if !ok || p.seeded { return @@ -202,6 +226,7 @@ func (w *Watchman) SeedStuck(key string, value []byte) { defer w.mu.Unlock() if s, err := delivery.DecodeStuck(value); err == nil { w.stuck[key] = s + w.stuckFinding(key, s) } } @@ -211,7 +236,7 @@ func (w *Watchman) SeedPulse(key string) { w.mu.Lock() defer w.mu.Unlock() bucket, k, err := delivery.ParsePulseKey(key) - if err != nil { + if err != nil || bucket == signalBucket { return } w.pulses[key] = &pulse{bucket: bucket, key: k, seeded: true} @@ -227,6 +252,38 @@ func (w *Watchman) Reconcile() { w.deletePulse(pk) } } + w.reconcileFindings() +} + +// reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not +// observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a +// machine's silent tools, a silent self-check): those end by their own events or expiry. +func (w *Watchman) reconcileFindings() { + kept := map[string]bool{} + for _, due := range w.signals { + kept[due.Finding.Key()] = true + } + for key := range w.seededFindings { + delete(w.seededFindings, key) + if kept[key] || signalKind(w.findings[key]) { + continue + } + w.unfind(key) + } +} + +// signalKind says a finding is one a signal row writes and ends by its own event (S8, S9, S10, S11): it outlives a +// restart, since what ends it is heard, not read. +func signalKind(value []byte) bool { + f, err := DecodeFinding(value) + if err != nil { + return false + } + switch f.Source { + case "provisioner.failing", "S8", "S9", "S10", "S11": + return true + } + return false } func describe(format string, args ...any) string { return fmt.Sprintf(format, args...) } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 28ea6f0..edcdec9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -19,10 +19,31 @@ type fakeOut struct { pulseDeletes []string stuck map[string]delivery.Stuck stuckWrites int + findings map[string]Finding + findingWrites int } func newOut() *fakeOut { - return &fakeOut{presence: map[string]bool{}, pulses: map[string]delivery.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]delivery.Stuck{}} + return &fakeOut{presence: map[string]bool{}, pulses: map[string]delivery.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]delivery.Stuck{}, + findings: map[string]Finding{}} +} + +func (o *fakeOut) PutFinding(key string, value []byte) error { + f, err := DecodeFinding(value) + if err != nil { + return err + } + if f.Key() != key { + return errors.New("a finding written under another key than its own: " + key) + } + o.findings[key] = f + o.findingWrites++ + return nil +} + +func (o *fakeOut) DeleteFinding(key string) error { + delete(o.findings, key) + return nil } func (o *fakeOut) PutPresence(key string, online bool) error { @@ -116,7 +137,12 @@ func stuckKeyOf(t *testing.T, bucket, key string) string { return k } -func TestPresenceFromEvents(t *testing.T) { // 1135/AC1 +func TestPresenceFromEventsWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 + t.Run("from events", presenceFromEvents) + t.Run("written only when it differs", presenceWrittenOnlyWhenItDiffers) +} + +func presenceFromEvents(t *testing.T) { // 1135/AC1 w, o, _ := setup() w.Connect("node.ace") w.Connect("ace.mesh-delivery-view") @@ -135,7 +161,7 @@ func TestPresenceFromEvents(t *testing.T) { // 1135/AC1 } } -func TestPresenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 +func presenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 w, o, _ := setup() w.Connect("node.ace") w.Connect("node.ace") @@ -178,7 +204,12 @@ func connzAnswer(users ...string) []byte { return b } -func TestRebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 +func TestRebuildAsksConnectionsAndTakesWhoIsGone(t *testing.T) { // 1135/AC2 + t.Run("asks the connections with auth", rebuildAsksConnectionsWithAuth) + t.Run("takes the presence of who is gone", rebuildTakesThePresenceOfWhoIsGone) +} + +func rebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 w, o, _ := setup() var body []byte err := w.Rebuild(func(b []byte) ([]byte, error) { @@ -201,7 +232,7 @@ func TestRebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 } } -func TestRebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2 +func rebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2 w, o, c := setup() w.SeedPresence("ace", true, t0.Add(-time.Hour)) w.SeedPresence("g14", true, t0.Add(-time.Hour)) @@ -416,7 +447,12 @@ func TestPresenceStuckThenBackOnline(t *testing.T) { // 1135/AC4 } } -func TestExpiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 +func TestOnlyTheExpiryOfItsOwnPulseWritesStuck(t *testing.T) { // 1135/AC5 + t.Run("the expiry marker writes stuck", expiryMarkerWritesStuck) + t.Run("its own delete of a pulse writes nothing", ownDeleteOfAPulseWritesNothing) +} + +func expiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 w, o, c := setup() w.Step("go", "n1.askid1", runningStep("askid1", t0), Put) c.t = t0.Add(120 * time.Second) @@ -430,7 +466,7 @@ func TestExpiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 } } -func TestOwnDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5 +func ownDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5 w, o, c := setup() w.Step("go", "n1.askid1", runningStep("askid1", t0), Put) pk := pulseKeyOf(t, checksKey, "askid1") diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 58e2171..3b8a9a0 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -25,19 +25,28 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` Context *struct { Repository string `json:"repository"` } `json:"context"` } +type claim struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` +} + type manifest struct { Module string `json:"module"` + Claims []claim `json:"claims"` + Tools []string `json:"tools"` Moved bool `json:"moved"` Version string `json:"version"` Slug string `json:"slug"` @@ -75,16 +84,24 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if m.Module != "mesh-delivery-watchman" || m.Moved || m.Slug != "watchman" { t.Errorf("module %q moved %v (it has no template, so it is not moved) slug %q", m.Module, m.Moved, m.Slug) } - want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}} + want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}, {Name: "findings", History: 5}} if !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } reads := slices.Clone(m.Reads) slices.Sort(reads) - if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) { + if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease", + "mesh-delivery-assigner.wants", "node-engine.have"}) { t.Errorf("reads = %v", reads) } - for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} { + // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). + if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { + t.Errorf("claims = %+v", m.Claims) + } + if !reflect.DeepEqual(m.Tools, []string{"watchman_findings"}) { + t.Errorf("tools = %v", m.Tools) + } + for _, k := range []string{"emits", "context", "items", "consumes", "provides"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the watchman has none", k) } @@ -97,25 +114,28 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } } -func TestManifestIsWholeInItsRepository(t *testing.T) { // 1135/AC6 - m, _ := load(t) - if len(m.Build.Artifacts) != 1 { - t.Fatalf("artifacts = %+v, want the one bundle program", m.Build.Artifacts) +// The manifest is whole in its repository: the program, and the tools bundle the runtime loads (1952/WM1). +func manifestIsWholeInItsRepository(t *testing.T, m manifest) { // 1135/AC6 + if len(m.Build.Artifacts) != 2 { + t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts) } - a := m.Build.Artifacts[0] - if a.Name != "program" || a.Kind != "bundle" || a.Language != "go" || a.System != "arch" || a.Context != nil { - t.Errorf("artifact = %+v", a) - } - if st, err := os.Stat(a.From); err != nil || !st.IsDir() { - t.Errorf("the artifact builds from %q, which is not a directory of this module", a.From) - } - if a.Binary != "mesh-delivery-watchman" || !strings.HasSuffix(a.From, a.Binary) { - t.Errorf("binary %q from %q", a.Binary, a.From) + for i, name := range []string{"program", "tools"} { + a := m.Build.Artifacts[i] + if a.Name != name || a.Kind != "bundle" || a.Language != "go" || a.System != "arch" || a.Context != nil { + t.Errorf("artifact = %+v", a) + } + if st, err := os.Stat(a.From); err != nil || !st.IsDir() { + t.Errorf("the artifact builds from %q, which is not a directory of this module", a.From) + } + if !strings.HasSuffix(a.From, a.Binary) || (name == "tools") != reflect.DeepEqual(a.Loads, []string{a.Binary}) { + t.Errorf("binary %q from %q loads %v", a.Binary, a.From, a.Loads) + } } } func TestManifestResources(t *testing.T) { // 1135/AC6 m, _ := load(t) + manifestIsWholeInItsRepository(t, m) byType := map[string][]map[string]any{} for _, r := range m.Resources { ty, _ := r["type"].(string) diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index c1c128f..45e46eb 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -2,17 +2,35 @@ "module": "mesh-delivery-watchman", "version": "1", "slug": "watchman", + "claims": [ + { + "name": "mesh-watchdogs", + "scope": "mesh", + "serves": [ + "signals", + "stuck", + "presence", + "dead-letters", + "resync" + ] + } + ], "reads": [ "mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", + "mesh-controller.lease", "mesh-delivery-assigner.wants", "node-engine.have" ], + "tools": [ + "watchman_findings" + ], "state": [ { "name": "presence", "history": 20 }, { "name": "pulse", "key-ttl": true }, - { "name": "stuck", "history": 20 } + { "name": "stuck", "history": 20 }, + { "name": "findings", "history": 5 } ], "own-secrets": { "broker": "${dir:state}/broker" @@ -34,6 +52,14 @@ "place": ".", "unknown": "program" }, + { + "id": "socket-dir", + "type": "directory", + "path": "/var/lib/mesh-delivery-watchman-tools", + "mode": "0755", + "owner": "mesh-delivery-watchman", + "unknown": "program" + }, { "id": "watchman", "type": "process", @@ -60,6 +86,17 @@ "system": "arch", "from": "cmd/mesh-delivery-watchman", "binary": "mesh-delivery-watchman" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-watchman-tools", + "binary": "mesh-delivery-watchman-tools", + "loads": [ + "mesh-delivery-watchman-tools" + ] } ] } From b8e8981730b3f008ee819ca236d3c29275f1f9b0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:46:29 +0100 Subject: [PATCH 06/35] Grants part: hear root-not-free from the seat the take-over key names, never both, so the switch to the condition keeper moves it with the router (issue 1952 G1) Watchman: subscribe exactly what its grant lets it hear (buscompose.WatchmanHears). Both pin mesh-sdk 69fcb6c. --- .../cmd/mesh-delivery-watchman/main_test.go | 14 +++++ .../cmd/mesh-delivery-watchman/signals.go | 55 +++++++++++++------ modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 +- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 +- modules/mesh-grants/grants_test.go | 33 +++++++++++ modules/mesh-grants/internal/grants/follow.go | 45 ++++++++++++++- modules/mesh-grants/internal/grants/inputs.go | 2 +- .../mesh-grants/internal/grants/rootfree.go | 25 ++++++++- modules/mesh-grants/module.json | 5 +- 11 files changed, 162 insertions(+), 29 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 66ce007..892c16a 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -24,6 +24,20 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Errorf("an essential bucket refused: %v, want its grant named", err) } }) + t.Run("1952/WM2: every subject its grant lets it hear goes to a row", func(t *testing.T) { + if len(HeardSubjects) < 7 { + t.Fatalf("hears %v", HeardSubjects) + } + for _, s := range HeardSubjects { + concrete := strings.ReplaceAll(strings.ReplaceAll(s, "*", "x"), ">", "x.y") + if routeOf(concrete) == "" { + t.Errorf("%s (%s) goes to no row", s, concrete) + } + } + if routeOf("mesh.mod.web.event.started") != "" { + t.Error("another module's event goes to a row") + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go index 6f7f742..9913ce6 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-watchman/internal/deadletters" "mesh-delivery-watchman/internal/socket" @@ -21,40 +22,62 @@ import ( // The subjects the watchman hears for the signals table's rows (novox/hq ADR 0564 §4, Limits 3 and 11), each on the bus, // none in the controller's store. Its grant names them (mesh-sdk buscompose, the watchman's principal). +const CheckedSubject = "mesh.seat." + health.SelfCheckModule + ".event." + health.EventChecked + +// HeardSubjects is every subject the watchman subscribes for its rows, beside the bus's connect and disconnect: the ones +// its grant lets it hear (buscompose.WatchmanHears), so the two cannot drift. +var HeardSubjects = buscompose.WatchmanHears() + +// The rows a heard subject goes to. const ( - PowerSubject = "mesh.mod.power.event.*" - FailingSubject = "mesh.mod.*.event.provisioner.failing" - RecoveredSubject = "mesh.mod.*.event.provisioner.recovered" - ToolsAliveSubject = "mesh.control.*.tools-alive" - CheckedSubject = "mesh.seat.mesh-self-check.event.checked" - DeadKeptSubject = "mesh.events.dead.>" + routePower = "power" + routeStanding = "standing" + routeTools = "tools-alive" + routeChecked = "checked" + routeAdvisory = "advisory" + routeDead = "dead-letter" ) -// HeardSubjects is every subject the watchman subscribes for its rows, beside the bus's connect and disconnect. -var HeardSubjects = []string{PowerSubject, FailingSubject, RecoveredSubject, ToolsAliveSubject, CheckedSubject, - watch.ConsumerDeletedAdvisories, DeadKeptSubject} +// routeOf is the row a heard subject goes to; empty for one the watchman does not read. +func routeOf(subject string) string { + switch { + case strings.HasPrefix(subject, "mesh.mod.power.event."): + return routePower + case strings.HasSuffix(subject, ".event.provisioner.failing") || strings.HasSuffix(subject, ".event.provisioner.recovered"): + return routeStanding + case strings.HasSuffix(subject, ".tools-alive"): + return routeTools + case subject == CheckedSubject: + return routeChecked + case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."): + return routeAdvisory + case strings.HasPrefix(subject, "mesh.events.dead."): + return routeDead + } + return "" +} // heard hands one message of a row's subject to the watchman. func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, ...any)) { - switch { - case strings.HasPrefix(m.Subject, "mesh.mod.power.event."): + switch routeOf(m.Subject) { + case routePower: w.Power(m.Header.Get("x-node"), strings.TrimPrefix(m.Subject, "mesh.mod.power.event.")) - case strings.HasSuffix(m.Subject, ".event.provisioner.failing") || strings.HasSuffix(m.Subject, ".event.provisioner.recovered"): + case routeStanding: st, err := watch.ReadStanding(m.Subject, m.Data) if err != nil { logf("%v; left", err) return } w.Stood(st) - case strings.HasSuffix(m.Subject, ".tools-alive"): + case routeTools: var body struct { Every float64 `json:"every"` } _ = json.Unmarshal(m.Data, &body) w.ToolsAlive(watch.ToolsNode(m.Subject), time.Duration(body.Every*float64(time.Second))) - case m.Subject == CheckedSubject: + case routeChecked: w.Checked(m.Header.Get("x-node")) - case strings.HasPrefix(m.Subject, "$JS.EVENT.ADVISORY."): + case routeAdvisory: a, ok := watch.ReadAdvisory(m.Subject, m.Data, deadletters.ConsumerInWords) if !ok { return @@ -64,7 +87,7 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, . return } w.Advised(a) - case strings.HasPrefix(m.Subject, "mesh.events.dead."): + case routeDead: dead.count() } } diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 1390d78..98eb8a7 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index e796a10..00b5c5c 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf h1:Yy51rUWqyk+cZPAcSX02dnK4JVSTUJIfLa9j+741+1Y= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204315-2a04bf0ac5bf/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index d73ed08..8eac37d 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 52700cd..ce421f4 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index f6391bc..44cf7c9 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/outward" "git.novox.be/novox/mesh-sdk/go/seal" @@ -656,8 +657,40 @@ func TestGrants(t *testing.T) { if passes() != before { t.Errorf("a condition of another kind woke a pass") } + // 1952/G1 (ADR 0564 §9): the keeper's events are not heard while the take-over key follows the controller + handle("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + time.Sleep(100 * time.Millisecond) + if passes() != before || wf.Follows() != "mesh-controller" { + t.Errorf("the keeper's event woke a pass while the part follows %s", wf.Follows()) + } handle(grants.EventConditionCleared, []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) waitFor(t, func() bool { return passes() > before }) + // and from the take-over only the keeper's are heard, never both + watcher := wf.Watch.(*fakeWatcher) + waitFor(t, func() bool { + watcher.mu.Lock() + defer watcher.mu.Unlock() + return watcher.changes[grants.ReadTakeover] != nil + }) + watcher.mu.Lock() + toKeeper := watcher.changes[grants.ReadTakeover] + watcher.mu.Unlock() + toKeeper(health.TakeoverKey, []byte(`{"v":1,"follow":"keeper","by":"operator","at":"2026-10-29T20:00:00Z"}`)) + if wf.Follows() != health.KeeperModule { + t.Fatalf("the part follows %s after the take-over", wf.Follows()) + } + before = passes() + handle("mesh.seat.mesh-controller.event.condition-raised", []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + time.Sleep(100 * time.Millisecond) + if passes() != before { + t.Errorf("the controller's event woke a pass after the take-over") + } + handle(grants.EventKeeperRaised, []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + waitFor(t, func() bool { return passes() > before }) + toKeeper(health.TakeoverKey, nil) // handed back: the key deleted follows the controller + if wf.Follows() != "mesh-controller" { + t.Errorf("the part follows %s after the hand-back", wf.Follows()) + } wcancel() <-wdone if !grants.IsRootNotFree([]byte(`{"kind":"root-not-free"}`)) || !grants.IsRootNotFree([]byte(`{"key":"machine.novox.root-not-free"}`)) || diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 98ba29b..86073f5 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // Watcher watches another part's state: changed hears each change after what stands, ended hears the watch end. A watch @@ -19,7 +20,7 @@ type Watcher interface { // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. -var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease} +var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect). @@ -49,6 +50,19 @@ type Follower struct { refusedMans string // leaseOn is whether the lease was handed away at its last change heard, once leaseSeen. leaseOn, leaseSeen bool + // follows is the seat whose condition events the part hears, as the take-over key names it: the controller's until + // the operator's take-over (an absent key, or one not readable, follows the controller). + follows string +} + +// Follows is the seat whose condition events the part hears now. +func (f *Follower) Follows() string { + f.mu.Lock() + defer f.mu.Unlock() + if f.follows == "" { + return "mesh-controller" + } + return f.follows } func (f *Follower) init() { @@ -88,6 +102,25 @@ func (f *Follower) logf(format string, args ...any) { // heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the // bus module's loaded list moving. func (f *Follower) heard(state, key string, value []byte) bool { + if state == ReadTakeover { + // Which seat's condition events are heard: no pass, since nothing composed reads it. + if key != health.TakeoverKey { + return false + } + t, err := health.DecodeTakeover(value, len(value) > 0) + if err != nil { + f.logf("the take-over key cannot be read (%v): the controller's condition events are heard", err) + t = health.Takeover{Follow: health.FollowController} + } + f.mu.Lock() + was := f.follows + f.follows = t.Seat() + f.mu.Unlock() + if was != t.Seat() { + f.logf("root-free words are heard from %s's condition events (the take-over key follows %s)", t.Seat(), t.Follow) + } + return false + } if state == ReadLease { // Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6). on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil) @@ -136,7 +169,15 @@ func (f *Follower) open() { } if f.Hear != nil && !f.heard_ { err := f.Hear(func(event string, body []byte) { - if IsAccountIssued(event) || IsRootNotFree(body) { + if IsAccountIssued(event) { + f.Wake() + return + } + // A condition event of the seat the take-over key does not name is not heard (ADR 0564 §9). + if from := HeardFrom(event); from != "" && from != f.Follows() { + return + } + if IsRootNotFree(body) { f.Wake() } }) diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 0637541..e0f702d 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -42,7 +42,7 @@ const ( // controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write // they place nothing, and the first change after it takes them out (../assignments-readers names the part until then). var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement, ReadLease} + ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( diff --git a/modules/mesh-grants/internal/grants/rootfree.go b/modules/mesh-grants/internal/grants/rootfree.go index 614bc4b..568b516 100644 --- a/modules/mesh-grants/internal/grants/rootfree.go +++ b/modules/mesh-grants/internal/grants/rootfree.go @@ -8,27 +8,46 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" ) // CallRootFree is the controller seat's verb that judges machines root-free (novox/hq ADR 0259 §8): the one judgement // the composer needs and the part cannot make from the bus (issue 1945). Only the controller answers it, on purpose. const CallRootFree = "seat:mesh-controller.root-free" -// The controller seat's condition events the part hears (never a timer): a machine's root-free word changes with the -// raising and the clearing of its `root-not-free` condition. +// The condition events the part hears (never a timer): a machine's root-free word changes with the raising and the +// clearing of its `root-not-free` condition. They are the controller seat's until the operator's take-over, and the +// condition keeper's (mesh-conditions) from it: the part hears only the seat the take-over key names, never both (novox/hq +// ADR 0564 §9, issue 1952 G1). const ( EventConditionRaised = "mesh-controller.condition-raised" EventConditionCleared = "mesh-controller.condition-cleared" + EventKeeperRaised = health.KeeperModule + ".condition-raised" + EventKeeperCleared = health.KeeperModule + ".condition-cleared" // KindRootNotFree is the condition kind whose events wake a pass. KindRootNotFree = "root-not-free" ) +// ReadTakeover is the condition keeper's take-over key's state (`switch`): which seat's condition events the part hears. +const ReadTakeover = health.KeeperModule + "." + health.TakeoverState + +// HeardFrom is the seat a condition event heard comes from, by its name (`.`) or its bus subject +// (`mesh.seat..event.`); empty for any other. +func HeardFrom(event string) string { + for _, seat := range []string{health.KeeperModule, "mesh-controller"} { + if strings.HasPrefix(event, seat+".") || strings.HasPrefix(event, "mesh.seat."+seat+".event.") { + return seat + } + } + return "" +} + // EventAccountIssued is the vault's word that it minted an account (novox/hq ADR 0561 §1): its hash is in the vault's // bus-accounts, so a pass composes the list with it. const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent // Consumes is every event the part hears, as its manifest's consumes name them. -var Consumes = []string{EventConditionRaised, EventConditionCleared, EventAccountIssued} +var Consumes = []string{EventConditionRaised, EventConditionCleared, EventKeeperRaised, EventKeeperCleared, EventAccountIssued} // IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject. func IsAccountIssued(event string) bool { diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 4417453..2294408 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -13,11 +13,14 @@ "mesh-delivery-assigner.placement", "node-engine.have", "mesh-vault.bus-accounts", - "mesh-controller.lease" + "mesh-controller.lease", + "mesh-conditions.takeover" ], "consumes": [ "mesh-controller.condition-raised", "mesh-controller.condition-cleared", + "mesh-conditions.condition-raised", + "mesh-conditions.condition-cleared", "mesh-vault.account-issued" ], "uses": [ From 73280f85c3a8ee20b5a85079dc59900e908b533a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:49:25 +0100 Subject: [PATCH 07/35] Assigner: write each condition it says as a finding too, and say it straight only until the take-over, so its holds reach the operator through the condition keeper and never twice (issue 1952 WM6) --- .../cmd/mesh-delivery-assigner/main.go | 73 +++++++++++- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../internal/assign/asks_test.go | 54 ++++++++- .../internal/assign/findings.go | 104 ++++++++++++++++++ .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 4 +- 7 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-assigner/internal/assign/findings.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 34f00a4..996d423 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -15,6 +15,7 @@ import ( "os" "os/signal" "slices" + "sync/atomic" "syscall" "time" @@ -26,8 +27,11 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provider" + + "mesh-delivery-assigner/internal/assign" ) func main() { @@ -66,8 +70,13 @@ var readBuckets = []string{ delivery.PresenceBucket, delivery.HaveBucket, delivery.StepsBucket(goBuildModule), + takeoverBucket, } +// takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said +// conditions go straight to the router or only as findings (issue 1952 WM6). +var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState) + // goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes // their steps in its steps state (novox/hq ADR 0482 clause 4). const goBuildModule = "mesh-build-go" @@ -76,7 +85,7 @@ const goBuildModule = "mesh-build-go" // (ADR 0567 §1), named after the assigner as every module state is. var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket, - delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket} + delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket} // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). @@ -98,6 +107,7 @@ var keptStates = []announce.State{ {Name: "differences", Readable: true}, {Name: "puts", Readable: true}, {Name: "wants", Readable: true}, + {Name: health.FindingsState, Readable: true}, } // buckets is every bucket the assigner opened, by name. @@ -146,6 +156,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil { + // New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the + // router and kept as no finding, as before. + logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err) + continue + } if sourceOfGrants(name) && err != nil { // The grants part's grants and the controller's lease choose where grants are read from (issue 774): without // them the controller's grants are read, as before the lease is handed away. @@ -194,7 +210,60 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } thisNode = account.Node logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) - return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger) + return follow(ctx, js, conn, kvs, findingsOf(ctx, kvs, logger.Printf).Say(sayVia(ctx, js)), emitting{said}, logger) +} + +// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the +// condition keeper's take-over key, watched, so a condition is said straight to the router only until the take-over. A +// bucket it cannot open is said: without its findings the conditions are said straight, as before; without the take-over +// key they are said straight too, the controller being followed. +func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings { + var f assign.Findings + if kv := kvs[assign.FindingsBucket]; kv != nil { + f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err } + f.Delete = func(key string) error { return kv.Delete(ctx, key) } + } + var keeper atomic.Bool + f.ToKeeper = keeper.Load + takeover := takeoverBucket + kv := kvs[takeover] + if kv == nil { + return f + } + w, err := kv.Watch(ctx, health.TakeoverKey) + if err != nil { + process.NotWatched(logf, takeover, err, "the controller is followed: the conditions are said straight to the router") + return f + } + go func() { + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + logf("the watch of %s ended: the last source it named is followed until the assigner starts again", takeover) + return + } + if e == nil { + continue + } + var raw []byte + if e.Operation() == jetstream.KeyValuePut { + raw = e.Value() + } + t, err := health.DecodeTakeover(raw, raw != nil) + if err != nil { + logf("the take-over key cannot be read (%v): the controller is followed", err) + } + if was := keeper.Swap(t.Follow == health.FollowKeeper); was != keeper.Load() { + logf("the assigner's conditions are said by %s from now (the take-over key follows %s)", t.Seat(), t.Follow) + } + } + } + }() + return f } // thisNode is the node this process runs on, as its credential names it: said with each of the seat's events. diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index e050a8b..97fcb8a 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 043e05f..ae67790 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index 0f0fbdb..231ea07 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -1,12 +1,14 @@ package assign import ( + "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at @@ -73,7 +75,13 @@ func TestTheRestoreAskIsValid(t *testing.T) { // 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own // name (ADR 0437 clause 9), and is cleared under the same key. -func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) { +func TestTheAssignersConditions(t *testing.T) { + t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords) + t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy) + t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings) +} + +func heldConditionNamesModuleNodeAndWords(t *testing.T) { c := HeldCondition("mail", fNew, "ace", "connection refused", true, true) if _, err := asks.EncodeCondition(c); err != nil { t.Fatalf("the condition is refused: %v", err) @@ -87,7 +95,7 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) { } // 1906/H2: a hold lifted by itself clears the held condition under its key, saying why. -func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) { +func liftedHoldClearsItsConditionSayingWhy(t *testing.T) { c := LiftedCondition("mail", fNew) if _, err := asks.EncodeCondition(c); err != nil { t.Fatalf("the condition is refused: %v", err) @@ -97,6 +105,48 @@ func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) { } } +// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under +// `..`, written at the raise and deleted at the clear; it is said straight to the router while the take-over +// key follows the controller, and only as a finding from the take-over. +func saidConditionsAreFindings(t *testing.T) { + at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + for _, c := range []struct { + raised, cleared asks.Condition + key string + }{ + {HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"}, + {NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"}, + {NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"}, + {PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"}, + {SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"}, + {BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"}, + } { + puts, deletes, said := map[string]health.Finding{}, []string{}, 0 + keeper := false + f := Findings{Put: func(k string, v []byte) error { + got, err := health.DecodeFinding(v) + puts[k] = got + return err + }, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }} + say := f.Say(func(asks.Condition) error { said++; return nil }) + if err := say(c.raised); err != nil { + t.Fatalf("%s: %v", c.key, err) + } + got, ok := puts[c.key] + if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule || + got.Subject.Machine != c.raised.Subject.Machine || said != 1 { + t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said) + } + keeper = true + if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 { + t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err) + } + } + if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil { + t.Error("a condition not the assigner's is a finding") + } +} + // 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in // the view, and name the move and the node. func TestTheRecordAndBackupAsksAreValid(t *testing.T) { diff --git a/modules/mesh-delivery-assigner/internal/assign/findings.go b/modules/mesh-delivery-assigner/internal/assign/findings.go new file mode 100644 index 0000000..cd0baaf --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/findings.go @@ -0,0 +1,104 @@ +package assign + +import ( + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says +// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair +// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its +// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router +// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the +// assigner no longer says it straight (never both). + +// FindingsBucket is the assigner's findings. +var FindingsBucket = health.FindingsBucket(delivery.AssignerModule) + +// FindingOf is a said condition as a finding: its key `..`, from the condition's own key +// `mesh-delivery-assigner..` (the subject's id when the key names none), its words unchanged. raised is false for +// a clear, whose key is to be deleted; value is nil then. +func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) { + rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".") + if !ok || rest == "" { + return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key) + } + kind, id, _ := strings.Cut(rest, ".") + if id == "" { + id = c.Subject.ID + } + scope := c.Subject.Scope + if scope == "" { + scope = health.ScopeModule + } + key = health.FindingKey(scope, id, kind) + if c.State == asks.StateCleared { + return key, nil, false, nil + } + if c.State != asks.StateRaised { + return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key) + } + summary := c.Summary + if summary == "" { + summary = c.Headline + } + var actions []health.Action + for _, a := range c.Actions { + actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)}) + } + value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine}, + Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, + Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind, + Observer: delivery.AssignerModule, At: at.UTC()}) + return key, value, true, err +} + +// Findings is where the assigner's findings go: the bucket's put and delete, and whether the keeper's conditions are the +// ones said (the take-over key names the keeper). +type Findings struct { + Put func(key string, value []byte) error + Delete func(key string) error + ToKeeper func() bool + Now func() time.Time +} + +// Say is say with each condition also written as a finding (or its finding deleted), and said straight to the router only +// while the take-over key follows the controller. A finding that cannot be written is the error, beside the say's. +func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error { + return func(c asks.Condition) error { + var errs []error + if f.Put != nil { + now := time.Now + if f.Now != nil { + now = f.Now + } + key, value, raised, err := FindingOf(c, now()) + switch { + case err != nil: + errs = append(errs, err) + case raised: + if err := f.Put(key, value); err != nil { + errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err)) + } + default: + if err := f.Delete(key); err != nil { + errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err)) + } + } + } + if f.ToKeeper == nil || !f.ToKeeper() { + if say != nil { + if err := say(c); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index c4d473c..3365117 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -33,7 +33,7 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", + want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"} if !reflect.DeepEqual(reads, want) { @@ -44,7 +44,7 @@ func TestManifest(t *testing.T) { states[s.Name] = s.History } if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20, - "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0}; !reflect.DeepEqual(states, want) { + "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 191d590..28aaf55 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -19,6 +19,7 @@ ], "reads": [ "mesh-build-go.steps", + "mesh-conditions.takeover", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", @@ -66,7 +67,8 @@ { "name": "compositions", "history": 5 }, { "name": "differences", "history": 64 }, { "name": "puts", "history": 20 }, - { "name": "wants", "history": 20 } + { "name": "wants", "history": 20 }, + { "name": "findings", "history": 5 } ], "streams": [ { From e1fd3eb0f660aab97a698ada81ff1602bd771890 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:52:42 +0100 Subject: [PATCH 08/35] View: answer status from the buckets it reads and derive motion with mesh-sdk's MotionOf, so status leaves the controller and the keeper judges a delivery as the view does (issue 1952 V1) --- .../cmd/mesh-delivery-view-tools/main.go | 17 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/read.go | 10 + .../internal/view/status.go | 218 ++++++++++++++++++ .../mesh-delivery-view/internal/view/view.go | 35 ++- .../internal/view/view_test.go | 79 ++++++- modules/mesh-delivery-view/manifest_test.go | 4 +- modules/mesh-delivery-view/module.json | 3 + 9 files changed, 341 insertions(+), 31 deletions(-) create mode 100644 modules/mesh-delivery-view/internal/view/status.go diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 55721c7..3abc440 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strings" + "time" "mesh-delivery-view/internal/view" @@ -19,6 +20,14 @@ import ( // Tools are the read tools over a modules state, and build_usage over the build modules' states. func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { return []stdio.Tool{ + {Name: view.StatusTool, + Description: "The mesh's status, derived by the view from the buckets it reads (novox/hq ADR 0564 §10), nothing written as a " + + "summary: every open condition of the condition keeper (mesh-conditions), urgent first then oldest; what is quiet (each " + + "node the watchman says is offline, each key it says stopped moving); each module whose want and have differ on a node " + + "(moving, or stuck with the have's or the watchman's words), as mesh-sdk's delivery motion derives it; and what " + + "mesh-healers did in the last seven days, by outcome. A part that cannot be read says so, never as none; well is true " + + "only when nothing is open, quiet or stuck and every part was read. Read-only.", + Run: func(map[string]any) (any, error) { return r.Status(time.Now()), nil }}, {Name: view.ReachReadTool, Description: "Whether a merge is on the machines: per repository, the reached head (the newest merge every node runs a build " + "containing, as the view derives it from the indexer's modules, the wants and the haves; a merge at or before it is on the " + @@ -232,9 +241,11 @@ func read(state stdio.KeptState) view.StateRead { func main() { r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")), - ReachState: read(stdio.State("reach")), - HeardState: read(stdio.State("mesh-delivery-indexer.heard")), - SignersState: read(stdio.State("mesh-vault.signers"))} + ReachState: read(stdio.State("reach")), + HeardState: read(stdio.State("mesh-delivery-indexer.heard")), + SignersState: read(stdio.State("mesh-vault.signers")), + ConditionsState: read(stdio.State(view.StatusConditionsState)), AttemptsState: read(stdio.State(view.StatusAttemptsState)), + PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState))} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 537359b..82a7256 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 043e05f..ae67790 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 9fdbe2f..58e45a3 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -46,6 +47,15 @@ type Reader struct { HeardState StateRead // SignersState is the vault's public signing halves (ADR 0550 §2); unset reads none. SignersState StateRead + // ConditionsState, AttemptsState, PresenceState and StuckState are what status reads beside the wants and the haves + // (ADR 0564 §10): the condition keeper's open conditions, mesh-healers' attempts, the watchman's presence and stuck keys. + ConditionsState, AttemptsState, PresenceState, StuckState StateRead +} + +// Status is the view's derivation of status at now (status.go). +func (r Reader) Status(now time.Time) Status { + return DeriveStatus(StatusRead{Conditions: r.ConditionsState, Attempts: r.AttemptsState, Presence: r.PresenceState, + Stuck: r.StuckState, Wants: r.WantsState, Haves: r.HavesState}, now) } // ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556). diff --git a/modules/mesh-delivery-view/internal/view/status.go b/modules/mesh-delivery-view/internal/view/status.go new file mode 100644 index 0000000..d8a0837 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/status.go @@ -0,0 +1,218 @@ +package view + +import ( + "encoding/json" + "fmt" + "sort" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// `status` is the view's derivation (novox/hq ADR 0564 §10, Limits 7, issue 1952 V1): what is wrong, quiet and on its way, +// read from the buckets it watches, never from a summary someone writes (research 072 §2), so the controller's status, its +// summary and the probe D9 go with its lease. Each part comes from one bucket: +// - Conditions: the condition keeper's open conditions (mesh-conditions.conditions), urgent first, then oldest; +// - Quiet: the watchman's presence of each node that is offline, and its stuck keys not moved; +// - Moving: each module whose want and have differ, as mesh-sdk's delivery.MotionOf derives it (moving or stuck); +// - Heals: what mesh-healers did in the last seven days (mesh-healers.attempts), by outcome. +// A bucket that cannot be read says so in its part's Unread, never as none. + +// StatusTool is the view's tool that answers status. +const StatusTool = "view_status" + +// The states status reads, as the view's manifest names them. +const ( + StatusConditionsState = health.KeeperModule + "." + health.ConditionsState + StatusAttemptsState = health.HealersModule + "." + health.AttemptsState + StatusPresenceState = delivery.WatchmanModule + ".presence" + StatusStuckState = delivery.WatchmanModule + ".stuck" +) + +// HealsWithin is how far back status counts the healers' acts. +const HealsWithin = 7 * 24 * time.Hour + +// StatusRead is the five buckets status reads, as a tool reads them. +type StatusRead struct { + Conditions, Attempts, Presence, Stuck, Wants, Haves StateRead +} + +// Status is the view's answer to status. +type Status struct { + Conditions []StatusCondition `json:"conditions"` + ConditionsUnread string `json:"conditions-unread,omitempty"` + Quiet []StatusQuiet `json:"quiet"` + QuietUnread string `json:"quiet-unread,omitempty"` + Moving []StatusMoving `json:"moving"` + MovingUnread string `json:"moving-unread,omitempty"` + Heals map[string]int `json:"heals-this-week"` + HealsUnread string `json:"heals-unread,omitempty"` + // Well says nothing is open, quiet or stuck; a part unread is not well. + Well bool `json:"well"` +} + +// StatusCondition is one open condition in brief. +type StatusCondition struct { + Key string `json:"key"` + Severity string `json:"severity"` + Machine string `json:"machine,omitempty"` + Summary string `json:"summary"` + Raised time.Time `json:"raised"` + Silenced bool `json:"silenced,omitempty"` +} + +// StatusQuiet is a node offline, or a key the watchman says stopped moving. +type StatusQuiet struct { + Key string `json:"key"` + What string `json:"what"` + Since time.Time `json:"since,omitzero"` +} + +// StatusMoving is a module whose want and have differ on a node. +type StatusMoving struct { + Node string `json:"node"` + Module string `json:"module"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Words string `json:"words,omitempty"` + FromFP string `json:"from_fp,omitempty"` + ToFP string `json:"to_fp"` +} + +func values(s StateRead) (map[string]json.RawMessage, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("not read here") + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + out := map[string]json.RawMessage{} + for _, k := range keys { + v, err := s.Get(k) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + if v != nil { + out[k] = v + } + } + return out, nil +} + +// DeriveStatus is status from the buckets r reads, at now. +func DeriveStatus(r StatusRead, now time.Time) Status { + st := Status{Conditions: []StatusCondition{}, Quiet: []StatusQuiet{}, Moving: []StatusMoving{}, Heals: map[string]int{}} + + if raw, err := values(r.Conditions); err != nil { + st.ConditionsUnread = "the condition keeper's conditions cannot be read: " + err.Error() + } else { + var open []health.Condition + for _, v := range raw { + var c health.Condition + if json.Unmarshal(v, &c) == nil && c.Key != "" { + open = append(open, c) + } + } + health.Order(open) + for _, c := range open { + st.Conditions = append(st.Conditions, StatusCondition{Key: c.Key, Severity: string(c.Severity), Machine: c.Subject.Machine, + Summary: c.Summary, Raised: c.Raised, Silenced: c.SilencedAt(now)}) + } + } + + presence, perr := values(r.Presence) + stuck, serr := values(r.Stuck) + switch { + case perr != nil: + st.QuietUnread = "the watchman's presence cannot be read: " + perr.Error() + case serr != nil: + st.QuietUnread = "the watchman's stuck keys cannot be read: " + serr.Error() + default: + for k, v := range presence { + if p, err := delivery.DecodePresence(v); err == nil && !p.Online { + st.Quiet = append(st.Quiet, StatusQuiet{Key: k, What: "offline"}) + } + } + for k, v := range stuck { + if s, err := delivery.DecodeStuck(v); err == nil && !s.Moved { + st.Quiet = append(st.Quiet, StatusQuiet{Key: k, What: "stopped moving: it waited for " + s.Waited, Since: s.Since}) + } + } + sort.Slice(st.Quiet, func(i, j int) bool { return st.Quiet[i].Key < st.Quiet[j].Key }) + } + + wants, werr := values(r.Wants) + haves, herr := values(r.Haves) + switch { + case werr != nil: + st.MovingUnread = "the assigner's wants cannot be read: " + werr.Error() + case herr != nil: + st.MovingUnread = "the nodes' haves cannot be read: " + herr.Error() + default: + for wk, v := range wants { + node, module, err := delivery.ParseWantKey(wk) + if err != nil { + continue + } + w, err := delivery.DecodeWant(v) + if err != nil { + continue + } + hk, _ := delivery.HaveKey(node, module) + var h delivery.Have + hv, has := haves[hk] + if has { + if h, err = delivery.DecodeHave(hv); err != nil { + has = false + } + } + words, stopped := "", false + if sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk); err == nil && serr == nil { + if s, err := delivery.DecodeStuck(stuck[sk]); err == nil && !s.Moved { + words, stopped = s.Waited, true + } + } + m := delivery.MotionOf(w, true, h, has, words, stopped) + if m.State == "" || m.State == delivery.MotionSettled { + continue + } + row := StatusMoving{Node: node, Module: module, State: m.State, Outcome: m.Outcome, Words: m.Words, ToFP: w.FP} + if has { + row.FromFP = h.FP + } + st.Moving = append(st.Moving, row) + } + sort.Slice(st.Moving, func(i, j int) bool { + if st.Moving[i].Node != st.Moving[j].Node { + return st.Moving[i].Node < st.Moving[j].Node + } + return st.Moving[i].Module < st.Moving[j].Module + }) + } + + if raw, err := values(r.Attempts); err != nil { + st.HealsUnread = "mesh-healers' attempts cannot be read: " + err.Error() + } else { + for _, v := range raw { + var a health.Attempt + if json.Unmarshal(v, &a) != nil || now.Sub(a.Begun) > HealsWithin { + continue + } + outcome := a.Outcome + if outcome == "" { + outcome = health.OutcomeActing + } + st.Heals[outcome]++ + } + } + + stuckMoving := false + for _, m := range st.Moving { + stuckMoving = stuckMoving || m.State == delivery.MotionStuck + } + st.Well = len(st.Conditions) == 0 && len(st.Quiet) == 0 && !stuckMoving && + st.ConditionsUnread == "" && st.QuietUnread == "" && st.MovingUnread == "" + return st +} diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index 7f8acc1..05391bd 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -1185,9 +1185,9 @@ func splitKey(key string) (node, module string) { // module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it // is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition. const ( - MotionMoving = "moving" - MotionSettled = "settled" - MotionStuck = "stuck" + MotionMoving = delivery.MotionMoving + MotionSettled = delivery.MotionSettled + MotionStuck = delivery.MotionStuck // MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound). MotionBoundSeconds = 1800 ) @@ -1316,7 +1316,6 @@ func (m *Model) motionOf(hk string) []MotionEvent { id := func(kind string) string { return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:") } - applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed) var out []MotionEvent say := func(kind string, b MotionBody) { if m.told[id(kind)] { @@ -1325,31 +1324,25 @@ func (m *Model) motionOf(hk string) []MotionEvent { m.told[id(kind)] = true out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b}) } - switch { - case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK: + // The derivation is mesh-sdk's (delivery.MotionOf, issue 1952 S7), shared with the condition keeper's hold. + words, stuck := m.stuckWant(w.value, node, module) + mo := delivery.MotionOf(w.value, true, h.have, has, words, stuck) + if mo.State == delivery.MotionSettled { if m.told[id(MotionMoving)] { b := body - b.Outcome = "running" + b.Outcome = mo.Outcome say(MotionSettled, b) } return out - case !m.told[id(MotionMoving)]: + } + if !m.told[id(MotionMoving)] { say(MotionMoving, body) } - stop := body - switch { - case has && h.have.Phase == delivery.PhaseHeld: - stop.Outcome, stop.Words = "stopped", h.have.Detail - case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP: - stop.Outcome, stop.Words = "failed", h.have.Detail - default: - if s, ok := m.stuckWant(w.value, node, module); ok { - stop.Outcome, stop.Words = "stopped", s - } else { - return out - } + if mo.State == delivery.MotionStuck { + stop := body + stop.Outcome, stop.Words = mo.Outcome, mo.Words + say(MotionStuck, stop) } - say(MotionStuck, stop) return out } diff --git a/modules/mesh-delivery-view/internal/view/view_test.go b/modules/mesh-delivery-view/internal/view/view_test.go index f4f8840..9980d92 100644 --- a/modules/mesh-delivery-view/internal/view/view_test.go +++ b/modules/mesh-delivery-view/internal/view/view_test.go @@ -1,6 +1,7 @@ package view import ( + "encoding/json" "errors" "reflect" "strings" @@ -8,6 +9,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) @@ -212,7 +214,13 @@ func stuckE(t *testing.T, bucket, key, waited string, since time.Time, moved boo return Entry{Bucket: delivery.StuckBucket, Key: bucket + "." + key, Value: b, At: since} } -func TestOfflineAndStuckAreListed(t *testing.T) { // 1168/AC2 +func TestTheWatchmansKeysAndStatus(t *testing.T) { + t.Run("offline and stuck are listed", offlineAndStuckAreListed) + t.Run("an unknown watchman version is unreadable, not dropped", unknownWatchmanVersionIsUnreadableNotDropped) + t.Run("1952/V1: status derives each part from its bucket, and says a part it cannot read", statusDerivesEachPartFromItsBucket) +} + +func offlineAndStuckAreListed(t *testing.T) { // 1168/AC2 m := New() fill(m, base(t)...) fill(m, @@ -240,7 +248,7 @@ func TestOfflineAndStuckAreListed(t *testing.T) { // 1168/AC2 } } -func TestUnknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC3 +func unknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC3 m := New() fill(m, Entry{Bucket: delivery.PresenceBucket, Key: "c", Value: []byte(`{"v":9,"online":false}`), At: t0}, @@ -267,3 +275,70 @@ func TestUnknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC t.Fatal("a readable value must clear the unreadable mark") } } + +// 1952/V1 (ADR 0564 §10): status reads the keeper's open conditions, the watchman's presence and stuck keys, the wants against +// the haves and the healers' attempts of the week; nothing writes a summary, and a part it cannot read is said, never well. +func statusDerivesEachPartFromItsBucket(t *testing.T) { + now := t0.Add(time.Hour) + composedC := "sha256:" + strings.Repeat("c", 64) + fp1, fp2, fp3 := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64), delivery.SourcePrefix2+strings.Repeat("3", 64) + enc := func(v any) string { b, _ := json.Marshal(v); return string(b) } + want := func(fp string) string { + b, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: composedC}) + return string(b) + } + have := func(fp string, phase delivery.Phase, health delivery.Health, detail string) string { + b, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: composedC, Phase: phase, Health: health, Detail: detail}) + if err != nil { + t.Fatal(err) + } + return string(b) + } + presence := func(on bool) string { b, _ := delivery.EncodePresence(delivery.Presence{Online: on}); return string(b) } + stuckOf := func(bucket, key, waited string, moved bool) string { + b, _ := delivery.EncodeStuck(delivery.Stuck{Bucket: bucket, Key: key, Waited: waited, Since: t0, Moved: moved}) + return string(b) + } + stuckWant, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.b.web") + fin := t0.Add(time.Minute) + r := StatusRead{ + Conditions: states(map[string]string{ + "machine.b.silent": enc(health.Condition{Key: "machine.b.silent", Severity: health.Warning, Summary: "b silent", Raised: t0}), + "core.doctor.self-check": enc(health.Condition{Key: "core.doctor.self-check", Severity: health.Urgent, Summary: "no run", Raised: t0.Add(time.Minute)}), + }), + Presence: states(map[string]string{"a": presence(true), "b": presence(false)}), + Stuck: states(map[string]string{stuckWant: stuckOf(delivery.AssignerWantsBucket, "want.b.web", "b to run web", false), + "x.moved": stuckOf("x", "moved", "y", true)}), + Wants: states(map[string]string{"want.a.web": want(fp2), "want.a.mail": want(fp1), "want.b.web": want(fp2), "want.a.db": want(fp3)}), + Haves: states(map[string]string{"a.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.mail": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), + "b.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.db": have(fp3, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed")}), + Attempts: states(map[string]string{"1": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: t0, Finished: &fin}), + "2": enc(health.Attempt{Outcome: health.OutcomeEscalated, Begun: t0}), "old": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: now.Add(-8 * 24 * time.Hour)})}), + } + st := DeriveStatus(r, now) + if len(st.Conditions) != 2 || st.Conditions[0].Key != "core.doctor.self-check" || st.Conditions[1].Key != "machine.b.silent" { + t.Errorf("conditions = %+v, want the urgent one first", st.Conditions) + } + if len(st.Quiet) != 2 || st.Quiet[0].Key != "b" || st.Quiet[1].Key != stuckWant { + t.Errorf("quiet = %+v", st.Quiet) + } + wantMoving := []StatusMoving{ + {Node: "a", Module: "db", State: delivery.MotionStuck, Outcome: delivery.OutcomeFailed, Words: "the unit failed", FromFP: fp3, ToFP: fp3}, + {Node: "a", Module: "web", State: delivery.MotionMoving, FromFP: fp1, ToFP: fp2}, + {Node: "b", Module: "web", State: delivery.MotionStuck, Outcome: delivery.OutcomeStopped, Words: "b to run web", FromFP: fp1, ToFP: fp2}, + } + if !reflect.DeepEqual(st.Moving, wantMoving) { + t.Errorf("moving = %+v\nwant %+v", st.Moving, wantMoving) + } + if !reflect.DeepEqual(st.Heals, map[string]int{health.OutcomeActed: 1, health.OutcomeEscalated: 1}) || st.Well { + t.Errorf("heals = %v, well %v", st.Heals, st.Well) + } + // A part it cannot read is said, and the mesh is not well; with nothing wrong and every part read it is. + if st := DeriveStatus(StatusRead{}, now); st.ConditionsUnread == "" || st.QuietUnread == "" || st.MovingUnread == "" || st.HealsUnread == "" || st.Well { + t.Errorf("nothing read = %+v", st) + } + empty := states(map[string]string{}) + if st := DeriveStatus(StatusRead{Conditions: empty, Presence: empty, Stuck: empty, Wants: empty, Haves: empty, Attempts: empty}, now); !st.Well { + t.Errorf("an empty mesh = %+v, want well", st) + } +} diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 47727fe..70a65c1 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-vault.signers", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 895fa21..5640a25 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -4,6 +4,7 @@ "moved": true, "slug": "view", "tools": [ + "view_status", "indexer_module", "indexer_modules", "indexer_candidate", @@ -53,6 +54,8 @@ "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", + "mesh-conditions.conditions", + "mesh-healers.attempts", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-vault.signers", From 7dbf7a19a1c187b2571b65f30f6ac25dfa2fa70f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:56:13 +0100 Subject: [PATCH 09/35] Grants part: compose a principal for every own bus account, a tools bundle's with what it may remove, so the bus tools remove the controller's dead-letter consumer and the engine's acts on their own account (issue 1952 S5) Pins mesh-sdk 7000bd5. --- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- modules/mesh-grants/grants_test.go | 13 ++++++++++++- modules/mesh-grants/internal/grants/records.go | 18 ++++++++---------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 8eac37d..17483ea 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index ce421f4..6dce500 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec h1:OLXHSQkAHnf03hzJTxRrUsl7vMfMhOx43M5yJpi5L5Y= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 44cf7c9..c6d9ec7 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -1077,7 +1077,9 @@ func TestGrants(t *testing.T) { file("node-tools-novox-licence-subjects.json", &live) licence, _ := os.ReadFile(filepath.Join("testdata", "claude-licence-manager.json")) n := newFakeBus() - n.manifestOf("nats", natsManifest) + // 1952/S5 (ADR 0564 Limits 4): the bus module's tools bundle declares its own account and what it may remove. + n.manifestOf("nats", strings.TrimSuffix(natsManifest, "}")+`,"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go",`+ + `"system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts"]}}]}}`) n.manifestOf("claude-licence-manager", string(licence)) n.assign("novox", "nats") n.assign("novox", "claude-licence-manager") @@ -1087,6 +1089,15 @@ func TestGrants(t *testing.T) { pl, _ = delivery.EncodePlacement(seed) n.states[grants.ReadPlacement] = map[string][]byte{"placement.novox": pl} recs, _ := grants.Records(grants.ReadInputs(n, &grants.Manifests{})) + var tools []buscompose.ProcessPrincipal + for _, d := range recs.Assigned["novox"] { + if d.Module == "nats" { + tools = d.Processes + } + } + if len(tools) != 1 || tools[0].Process != "nats-tools" || !slices.Equal(tools[0].Removes, []string{"DEAD_LETTER_NOTICES.controller", "mesh-host_acts"}) { + t.Errorf("the bus module's principals = %+v, want nats-tools carrying both removals", tools) + } if m := grants.Memberships(recs)["novox.claude-licence-manager"]; len(m.Membership.Seats) == 0 { t.Errorf("the licence manager's membership under the seeded placement carries no seats: %+v", m.Membership) } diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index a0e575f..6194d3f 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -294,21 +294,19 @@ func queueOwnersOf(m catalogue.Manifest) []buscompose.QueueOwner { } func processesOf(m catalogue.Manifest) []buscompose.ProcessPrincipal { - buses := m.ProcessBuses() - if len(buses) == 0 { + accounts := m.BusAccounts() + if len(accounts) == 0 { return nil } queues := queueOwnersOf(m) var out []buscompose.ProcessPrincipal - for _, r := range m.Resources { - id := fmt.Sprint(r["id"]) - b, declared := buses[id] - if !declared { - continue - } - pr := buscompose.ProcessPrincipal{Process: id, Emits: b.Emits, State: b.State, Asks: b.Asks, Invokes: b.Invokes} + // Every own bus account, processes first and then tools bundles (catalogue.BusAccounts): a tools bundle's account carries + // what it may remove from the bus (ADR 0564 Limits 4, issue 1952 S5). + for _, a := range accounts { + pr := buscompose.ProcessPrincipal{Process: a.ID, Emits: a.Bus.Emits, State: a.Bus.State, Asks: a.Bus.Asks, + Invokes: a.Bus.Invokes, Removes: a.Bus.Removes} for _, q := range queues { - if q.Process == id { + if q.Process == a.ID { pr.Queues = append(pr.Queues, q) } } From 15a15cbda51b0c5aba4bf6d8e86104e498d96e13 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:59:46 +0100 Subject: [PATCH 10/35] Watchman: observe S1, S9 and S21 as the controller does and stop the needs-operator row, so the keeper's compare can empty with no condition lost (issue 1952 WM2, WM3, WM5) S1 is urgent after half an hour for a control node (the node running mesh-controller). S9 also hears the bus module's slow consumers and closed connections, each closure counted once. S21 is one finding per heard machine from its failed haves and its provisioner clients' reports (PROV_APPLIED). DEAD_LETTERS is counted again at a letter's age and every five minutes while letters are held, on the watchman's own pulse. needs-operator waits for L6: its inputs are not on the bus. Pins mesh-sdk b6ef01d. --- .../cmd/mesh-delivery-watchman/main.go | 3 + .../cmd/mesh-delivery-watchman/signals.go | 74 ++++ modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 2 + .../internal/watch/dead.go | 15 +- .../internal/watch/findings_test.go | 76 +++- .../internal/watch/presence.go | 3 + .../internal/watch/signals.go | 350 ++++++++++++++++-- .../internal/watch/watch.go | 10 +- 9 files changed, 488 insertions(+), 47 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index af4d80b..4d50197 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -298,8 +298,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} + w.SetRecount(dead.count) dead.count() + dead.ages() dead.check() + provisioned(ctx, js, w, logger.Printf) w.Reconcile() if err := hear(conn, w, dead, logger.Printf); err != nil { return err diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go index 9913ce6..501800a 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go @@ -36,11 +36,23 @@ const ( routeChecked = "checked" routeAdvisory = "advisory" routeDead = "dead-letter" + routeBusFault = "bus-fault" + routeProv = "provisioned" +) + +// The bus module's events of the connections the bus closed (S9). +const ( + SlowConsumerSubject = "mesh.mod.nats.event.bus.slow-consumer" + ConnectionClosedSubject = "mesh.mod.nats.event.bus.connection-closed" ) // routeOf is the row a heard subject goes to; empty for one the watchman does not read. func routeOf(subject string) string { switch { + case subject == SlowConsumerSubject || subject == ConnectionClosedSubject: + return routeBusFault + case strings.HasPrefix(subject, "prov.applied."): + return routeProv case strings.HasPrefix(subject, "mesh.mod.power.event."): return routePower case strings.HasSuffix(subject, ".event.provisioner.failing") || strings.HasSuffix(subject, ".event.provisioner.recovered"): @@ -87,7 +99,30 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, . return } w.Advised(a) + case routeBusFault: + var c watch.BusClosure + if err := json.Unmarshal(m.Data, &c); err != nil { + logf("the bus module's word on %s cannot be read, left: %v", m.Subject, err) + return + } + kind := watch.KindSlowConsumer + if m.Subject == ConnectionClosedSubject { + kind = watch.KindConnectionClosed + } + w.Closed(kind, c) + case routeProv: + var r watch.ProvReport + if err := json.Unmarshal(m.Data, &r); err != nil { + logf("the provisioner client's report on %s cannot be read, left: %v", m.Subject, err) + return + } + w.Provisioned(r) case routeDead: + if stream, consumer, ok := buscompose.DeadLetterOf(m.Subject); ok { + if at, err := time.Parse(time.RFC3339Nano, m.Header.Get(deadletters.HeaderAgesOut)); err == nil { + w.DeadAging(stream+"."+consumer, at) + } + } dead.count() } } @@ -115,6 +150,44 @@ type deadKeeper struct { taking jetstream.ConsumeContext } +// ages pulses each consumer's restricted letters by when they age out, read from DEAD_LETTERS' headers (at the start and +// at resync; each letter kept later is pulsed as it is heard). +func (d *deadKeeper) ages() { + letters, _, err := deadletters.List(d.ctx, d.js, "", 0) + if err != nil { + d.logf("the dead letters' ages cannot be read, so one aging out ends its finding only at the next count: %v", err) + return + } + for _, l := range letters { + if l.Restricted && !l.AgesOut.IsZero() { + d.w.DeadAging(l.Stream+"."+l.Consumer, l.AgesOut) + } + } +} + +// provisioned reads each provisioner client's newest report kept by PROV_APPLIED (as the controller's +// ProvAppliedReports): S21 judges what stood at the start as what is said later. +func provisioned(ctx context.Context, js jetstream.JetStream, w *watch.Watchman, logf func(string, ...any)) { + s, err := js.Stream(ctx, buscompose.ProvAppliedStream) + if err == nil { + var info *jetstream.StreamInfo + if info, err = s.Info(ctx, jetstream.WithSubjectFilter(buscompose.ProvAppliedSubjects)); err == nil { + for subject := range info.State.Subjects { + msg, gerr := s.GetLastMsgForSubject(ctx, subject) + if gerr != nil { + continue // aged out between the two reads + } + var r watch.ProvReport + if json.Unmarshal(msg.Data, &r) == nil { + w.Provisioned(r) + } + } + return + } + } + logf("the provisioner clients' reports cannot be read (%v): S21 hears only those said from now", err) +} + // count reads DEAD_LETTERS' count per consumer into the findings. func (d *deadKeeper) count() { held, err := deadletters.Held(d.ctx, d.js) @@ -180,6 +253,7 @@ func answer(a socket.Ask, w *watch.Watchman, dead *deadKeeper, ask func([]byte) return nil, err } dead.count() + dead.ages() dead.check() return map[string]any{"findings-written": w.Rewrite(), "presence": "rebuilt from the bus's connections"}, nil case socket.AskDeadLetters: diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 98eb8a7..d5fd93f 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index 00b5c5c..d215b52 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7 h1:/XEAhD7/8QoMjqI7XiX9HM4HkpKubWheQosqgfe2WrU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/dead.go b/modules/mesh-delivery-watchman/internal/watch/dead.go index 8919dd6..51b0747 100644 --- a/modules/mesh-delivery-watchman/internal/watch/dead.go +++ b/modules/mesh-delivery-watchman/internal/watch/dead.go @@ -4,12 +4,18 @@ import ( "fmt" "sort" "strings" + "time" ) // The dead letters' findings (novox/hq ADR 0564 §5, issue 330): a consumer DEAD_LETTERS holds a message for is said for as // long as it holds any, `bus...max-deliveries`, and the notices waiting behind the controller's consumer // after the lease write are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`. +// DeadRecountEvery is how often DEAD_LETTERS is counted again while a consumer's finding stands: a letter delivered again +// or dropped by the controller's verb, which the watchman does not hear, ends its finding within it. The expiry of the +// watchman's own pulse, never a loop. +const DeadRecountEvery = 5 * time.Minute + // KindNoticesWaiting is the finding of notices waiting while the controller's consumer stands after the lease write. const KindNoticesWaiting = "notices-waiting" @@ -38,7 +44,8 @@ func (w *Watchman) DeadLettersHeld(held map[string]int, who func(stream, consume Machine: consumerMachine(stream, consumer), Summary: fmt.Sprintf("%s gave up on %s; %s kept in DEAD_LETTERS until delivered again or dropped, with why", who(stream, consumer), messages, them), - Said: fmt.Sprintf("%d held for %s", n, key), + Said: fmt.Sprintf("%d held for %s (counted again every %s while any is held, so one delivered again or dropped "+ + "ends this within it)", n, key, DeadRecountEvery), Needs: "deliver them again or drop them, from the mesh MCP server."} want[f.Key()] = true w.find(f) @@ -48,6 +55,12 @@ func (w *Watchman) DeadLettersHeld(held map[string]int, who func(stream, consume w.unfind(key) } } + if len(want) == 0 { + w.unsignal("dead", "recount") + return + } + w.signal("dead", "recount", DeadRecountEvery, signalDue{Do: dueRecount, + Finding: Finding{Scope: ScopeBus, ID: "DEAD_LETTERS", Kind: KindDeadLetters, Severity: Warning, Summary: "counted again", Source: "S9"}}) } // consumerMachine is the machine a module's consumer on EVENTS is on; empty for the others. diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 030d477..f31b704 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -121,16 +121,26 @@ func TestTheWatchmansFindings(t *testing.T) { } expire(w, c, signalKey("S9", k), AdvisoryQuiet) }, map[string]string{}}, - {"WM3 S21: a have failed for one apply interval is apply-failing; applied again it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"WM3 S21: a heard machine's have failed for one apply interval is apply-failing, naming the package manager; applied again it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") w.Have("ace.web", haveOf(t, delivery.PhaseFailed, delivery.HealthUnknown, "refused: the package manager (pacman) does not answer"), Put) - expire(w, c, signalKey("S21", "ace.web"), ApplyEvery) - if f := o.findings["machine.ace.web.apply-failing"]; !strings.Contains(f.Said, "pacman") { + if len(o.findings) != 0 { + t.Fatalf("raised before its bound: %v", o.findings) + } + expire(w, c, signalKey("S21", "ace"), ApplyEvery+time.Second) + if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "pacman does not answer") || f.Headline != "ace cannot install packages" { t.Fatalf("findings = %v", o.findings) } w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) - w.Have("ace.mail", haveOf(t, delivery.PhaseFailed, delivery.HealthUnknown, "refused"), Put) - w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) - expire(w, c, signalKey("S21", "ace.mail"), ApplyEvery) // applied before its bound: nothing + }, map[string]string{}}, + {"WM3 S21: a provisioner client's failed files are apply-failing past the bound; a silent machine's are not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") + w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: ProvFailed, Reason: "disk full", At: t0.Add(-10 * time.Minute)}) + if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "provisioner has failed to place the files of web") { + t.Fatalf("findings = %v", o.findings) + } + w.Provisioned(ProvReport{Node: "g14", Module: "web", Status: ProvFailed, Reason: "x", At: t0.Add(-10 * time.Minute)}) + w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: "applied", At: t0}) }, map[string]string{}}, {"WM3: unhealthy only at the second write in a row, urgent after four hours", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed"), Put) @@ -142,19 +152,63 @@ func TestTheWatchmansFindings(t *testing.T) { t.Fatalf("findings = %v", o.findings) } expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter) - w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) - }, map[string]string{"module.web.ace.unhealthy": Urgent, "module.mail.ace.needs-operator": Warning}}, + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) // needs-operator is not observed here + }, map[string]string{"module.web.ace.unhealthy": Urgent}}, {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "waits"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put) }, map[string]string{}}, + {"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + w.Connect("node.novox") + w.Disconnect("node.novox") + expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second) + if o.findings["machine.novox.silent"].Severity != Warning { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter) + }, map[string]string{"machine.novox.silent": Urgent}}, + {"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0} + w.Closed(KindSlowConsumer, cl) + w.Closed(KindSlowConsumer, cl) // said again when the bus module starts + cl.CID = 8 + w.Closed(KindSlowConsumer, cl) + f := o.findings["bus.ace.web.slow-consumer"] + if f.Machine != "ace" || !strings.Contains(f.Summary, "web on ace") || !strings.Contains(f.Summary, "2 times") { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S9", "bus.ace.web.slow-consumer"), AdvisoryQuiet) + }, map[string]string{}}, + {"WM5: while letters are held DEAD_LETTERS is counted again at its own pulse's expiry, and at a letter's age", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + n := 0 + w.SetRecount(func() { n++ }) + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c }) + if o.ttls[signalKey("dead", "recount")] != DeadRecountEvery || !strings.Contains(o.findings["bus.EVENTS.ace_web.max-deliveries"].Said, "every 5m0s") { + t.Fatalf("pulses %v findings %v", o.ttls, o.findings) + } + w.DeadAging("EVENTS.ace_web", c.t.Add(time.Minute)) + expire(w, c, signalKey("dead", "EVENTS.ace_web"), time.Minute) + expire(w, c, signalKey("dead", "recount"), DeadRecountEvery) + time.Sleep(10 * time.Millisecond) + if n != 2 { + t.Fatalf("counted again %d times", n) + } + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c }) // the count renews its pulse + w.DeadLettersHeld(map[string]int{}, nil) + if _, ok := o.pulses[signalKey("dead", "recount")]; ok { + t.Fatal("still counted again with nothing held") + } + }, map[string]string{}}, {"WM4: resync after a cleared bucket writes the same findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "waits"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) o.findings = map[string]Finding{} if n := w.Rewrite(); n != 1 { t.Fatalf("rewrote %d", n) } - }, map[string]string{"module.mail.ace.needs-operator": Warning}}, + }, map[string]string{"module.mail.ace.unhealthy": Warning}}, {"WM4: a finding read at the start and not observed again is deleted; a signal's stays", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { for _, f := range []Finding{ {Scope: ScopeDelivery, ID: "x.y", Kind: KindStuck, Severity: Warning, Summary: "old", Source: "stuck"}, diff --git a/modules/mesh-delivery-watchman/internal/watch/presence.go b/modules/mesh-delivery-watchman/internal/watch/presence.go index b6bcab2..369e73c 100644 --- a/modules/mesh-delivery-watchman/internal/watch/presence.go +++ b/modules/mesh-delivery-watchman/internal/watch/presence.go @@ -87,6 +87,9 @@ func (w *Watchman) settle(key string) { return } w.presence[key] = presence{online: online, since: w.now()} + if node, part, _, err := delivery.ParsePresenceKey(key); err == nil && part == "" && node != "" { + defer w.applyCheck(node) // S21 is judged only while the machine is heard + } if online { w.moved(delivery.PresenceBucket, key) return diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 949d6d5..4c3a067 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -44,6 +44,10 @@ const ( ApplyEvery = 5 * time.Minute // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. UnhealthyUrgentAfter = 4 * time.Hour + // SilentBound is the bound S1 says, three of the intervals a machine has always used, and ControlNodeUrgentAfter + // how long a control node may be silent before it is urgent (the controller's silentBound, controlNodeUrgentAfter). + SilentBound = 3 * HeartbeatEvery + ControlNodeUrgentAfter = 30 * time.Minute ) // signalBucket is the pseudo-bucket of a signal's pulse key. @@ -62,6 +66,9 @@ type signalDue struct { const ( dueFind = "find" dueEnd = "end" + // dueApply judges a machine's S21 again (applyCheck); dueRecount counts DEAD_LETTERS again (a restricted letter aged out). + dueApply = "apply" + dueRecount = "recount" ) // signalKey is the pulse key of a row's signal about id. @@ -127,7 +134,7 @@ func dueOf(value []byte) (signalDue, bool) { return signalDue{}, false } var due signalDue - if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd) { + if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd && due.Do != dueApply && due.Do != dueRecount) { return signalDue{}, false } return due, true @@ -141,6 +148,12 @@ func (w *Watchman) signalExpired(pk string) { } delete(w.signals, pk) switch due.Do { + case dueApply: + w.applyCheck(due.While) + case dueRecount: + if w.recount != nil { + go w.recount() + } case dueEnd: w.unfind(due.Finding.Key()) case dueFind: @@ -154,6 +167,10 @@ func (w *Watchman) signalExpired(pk string) { // stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for // unhealthy. func (w *Watchman) stillWrong(haveKey, kind string) bool { + if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { + _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] + return open + } h, ok := w.haveState[haveKey] if !ok { return false @@ -202,8 +219,10 @@ func (w *Watchman) Power(node, state string) { if err != nil { return } + defer w.applyCheck(node) if away { w.unfind(FindingKey(ScopeMachine, node, KindSilent)) + w.unsignal("S1", node) return } if p, ok := w.presence[key]; ok && !p.online { @@ -222,11 +241,26 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { key := FindingKey(ScopeMachine, node, KindSilent) if s.Moved || w.away[node] { w.unfind(key) + w.unsignal("S1", node) + w.applyCheck(node) return } - w.find(Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", - Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), delivery.PulseMoveBound), - Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}) + f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", + Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound), + Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since} + // A control node, the machine running the controller, is urgent after half an hour silent, as the controller + // says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop. + if w.control[node] { + if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter { + f.Severity = Urgent + } else { + u := f + u.Severity = Urgent + w.signal("S1", node, ControlNodeUrgentAfter-age+time.Second, signalDue{Do: dueFind, While: "silent:" + node, Finding: u}) + } + } + w.find(f) + w.applyCheck(node) return } } @@ -418,42 +452,41 @@ func ToolsNode(subject string) string { return node } -// ---- S21, unhealthy and needs-operator, from the haves ---- +// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ---- -// haveFindings reads a have's change for the rows the haves feed: S21 (failed for one apply interval), unhealthy (two -// writes in a row, urgent after four hours) and needs-operator (it waits for the operator). A have removed or deleted ends -// them all. +// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck), +// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have +// removed or deleted ends its part of them. +// +// needs-operator is not observed here: the controller excuses a wait only after checking it against the module's manifest +// and against whether a person gave the secret on that machine (its operator_wait.go), and neither each resource's wait nor +// what a person gave is on the bus (the have carries the node's words only). The row waits for those facts' home. func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) { - applyKey := FindingKey(ScopeMachine, node+"."+module, KindApplyFailing) unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") - waitsKey := FindingKey(ScopeModule, module+"."+node, KindNeedsOperator) + if module == ControllerModule { + w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved + } if h == nil || h.Phase == delivery.PhaseRemoved { delete(w.haveState, key) delete(w.unhealthy, key) - w.unsignal("S21", node+"."+module) + delete(w.failedSince, key) w.unsignal("unhealthy", module+"."+node) - w.unfind(applyKey) w.unfind(unhealthyKey) - w.unfind(waitsKey) + w.applyCheck(node) return } - was, had := w.haveState[key] w.haveState[key] = *h now := w.now() // S21: a have refused or applied in part says phase failed, with the node's words in its detail. if h.Phase == delivery.PhaseFailed { - if !had || was.Phase != delivery.PhaseFailed { - w.signal("S21", node+"."+module, ApplyEvery, signalDue{Do: dueFind, While: key, Finding: Finding{Scope: ScopeMachine, - ID: node + "." + module, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: now, - Summary: fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", node, module, stamp(now), ApplyEvery), - Said: firstLine(orWords(h.Detail, "no reason given")), - Headline: node + " is not applying its update", Resolved: node + " applies what it is sent again"}}) + if _, ok := w.failedSince[key]; !ok { + w.failedSince[key] = now } } else { - w.unsignal("S21", node+"."+module) - w.unfind(applyKey) + delete(w.failedSince, key) } + w.applyCheck(node) // unhealthy: two writes in a row that say it failed. if h.Health == delivery.HealthFailed { @@ -473,16 +506,268 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav w.unsignal("unhealthy", module+"."+node) w.unfind(unhealthyKey) } +} - // needs-operator: a part waits for the operator's secret or setting. - if h.Health == delivery.HealthWaits { - w.find(Finding{Scope: ScopeModule, ID: module + "." + node, Kind: KindNeedsOperator, Machine: node, Severity: Warning, - Source: "health", Since: now, - Summary: fmt.Sprintf("%s on %s waits for the operator: %s", module, node, orWords(h.Detail, "a part waits for a secret or a setting")), - Said: firstLine(orWords(h.Detail, "waits"))}) - } else { - w.unfind(waitsKey) +// ControllerModule is the module whose have names a control node. +const ControllerModule = "mesh-controller" + +// ProvReport is a machine's provisioner client's word on one module's files (`prov.applied..`, ADR 0362 §3). +type ProvReport struct { + Node string `json:"node"` + Module string `json:"module"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + At time.Time `json:"at"` +} + +// ProvFailed is the status of a report whose files were not placed. +const ProvFailed = "failed" + +// Provisioned is a provisioner client's newest report: a failure is kept for S21, any other word ends it. +func (w *Watchman) Provisioned(r ProvReport) { + if r.Node == "" || r.Module == "" { + return } + w.mu.Lock() + defer w.mu.Unlock() + if w.provFailed[r.Node] == nil { + w.provFailed[r.Node] = map[string]ProvReport{} + } + if r.Status == ProvFailed { + w.provFailed[r.Node][r.Module] = r + } else { + delete(w.provFailed[r.Node], r.Module) + } + w.applyCheck(r.Node) +} + +// heard says the watchman hears the machine now: online, and not said away. S21 is judged only while it is (the +// controller's watchApplyFailing): S1 says silent, and asleep is not lost. +func (w *Watchman) heard(node string) bool { + key, err := delivery.PresenceNodeKey(node) + if err != nil || w.away[node] { + return false + } + p, ok := w.presence[key] + return ok && p.online +} + +// applyCheck is S21 for one machine, `machine..apply-failing` (the controller's watchApplyFailing): raised once the +// first failure it still has is older than one apply interval while the machine is heard, naming the modules its node +// failed to apply (their haves failed), or else those whose files its provisioner client failed to place; ended by the +// clean apply. Before the bound a pulse judges it again at the bound. +func (w *Watchman) applyCheck(node string) { + key := FindingKey(ScopeMachine, node, KindApplyFailing) + type failure struct{ module, words string } + var applied, placed []failure + var since time.Time + for hk, at := range w.failedSince { + n, module, _, err := delivery.ParseHaveKey(hk) + if err != nil || n != node { + continue + } + applied = append(applied, failure{module, firstLine(orWords(w.haveState[hk].Detail, "no reason given"))}) + if since.IsZero() || at.Before(since) { + since = at + } + } + if len(applied) == 0 { + for module, r := range w.provFailed[node] { + placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))}) + if since.IsZero() || r.At.Before(since) { + since = r.At + } + } + } + if since.IsZero() || !w.heard(node) { + w.unsignal("S21", node) + w.unfind(key) + return + } + if age := w.now().Sub(since); age <= ApplyEvery { + if _, pending := w.signals[signalKey("S21", node)]; !pending { + w.signal("S21", node, ApplyEvery-age+time.Second, signalDue{Do: dueApply, While: node, + Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Severity: Warning, Summary: "judged again at its bound", Source: "S21"}}) + } + return + } + failed := append(applied, placed...) + sort.Slice(failed, func(i, j int) bool { return failed[i].module < failed[j].module }) + var names, words []string + tool := "" + for _, f := range failed { + names = append(names, f.module) + words = append(words, f.module+": "+f.words) + if rest, ok := strings.CutPrefix(f.words, "refused: "); ok { + f.words = rest + } + if i := strings.Index(f.words, PackageManagerKey); i >= 0 && tool == "" { + tool = strings.TrimSuffix(strings.SplitN(f.words[i+len(PackageManagerKey):], ")", 2)[0], ")") + } + } + f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since, + Said: strings.Join(words, "; "), Resolved: node + " applies what it is sent again", Headline: node + " is not applying its update"} + switch { + case len(applied) == 0: + f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node, + strings.Join(names, ", "), stamp(since), ApplyEvery) + f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.", + node, strings.Join(names, ", ")) + case tool != "": + f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, "+ + "since %s (bound %s)", node, tool, stamp(since), ApplyEvery) + f.Headline = node + " cannot install packages" + default: + f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", node, strings.Join(names, ", "), stamp(since), ApplyEvery) + f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", node) + } + w.find(f) +} + +// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed. +const PackageManagerKey = "the package manager (" + +// ---- S9: the bus module's closures ---- + +// BusClosure is one connection the bus closed, as the bus module says it (`bus.slow-consumer`, `bus.connection-closed`). +type BusClosure struct { + User string `json:"user"` + Name string `json:"name,omitempty"` + CID uint64 `json:"cid"` + Reason string `json:"reason"` + Opened time.Time `json:"opened"` + Closed time.Time `json:"closed"` + Pending int64 `json:"pending_bytes,omitempty"` +} + +// The kinds of the bus module's closures, as conditions name them. +const ( + KindSlowConsumer = "slow-consumer" + KindConnectionClosed = "connection-closed" +) + +// Closed is one closure the bus module said (S9, the controller's BusClosures and busUserClosed): one finding per user and +// kind, `bus..`, each closure counted once however often it is said, ended an hour after the newest. +func (w *Watchman) Closed(kind string, c BusClosure) { + if c.User == "" || c.Closed.IsZero() || (kind != KindSlowConsumer && kind != KindConnectionClosed) { + return + } + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: c.User, Kind: kind, Severity: Warning, Source: "S9"} + key := f.Key() + if _, open := w.findings[key]; !open { + w.closures[key] = map[string]time.Time{} + } + identity := fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano)) + if _, seen := w.closures[key][identity]; seen { + return + } + w.closures[key][identity] = c.Closed + var first, last time.Time + for _, at := range w.closures[key] { + if first.IsZero() || at.Before(first) { + first = at + } + if at.After(last) { + last = at + } + } + n := len(w.closures[key]) + who, machine := BusUserWords(c.User) + f.Machine, f.Since = machine, first + times, count := "once", "once" + if n > 1 { + times = fmt.Sprintf("%d times since %s", n, first.UTC().Format("15:04 MST")) + count = fmt.Sprintf("%d times", n) + } + if kind == KindSlowConsumer { + f.Said = fmt.Sprintf("the bus closed %s's connection %d as a slow consumer: %s", c.User, c.CID, c.Reason) + f.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, newest at %s: it "+ + "could not take what it was sent fast enough, and what was on its way to it was lost", who, times, last.UTC().Format("15:04 MST")) + f.Headline = clip(capital(who)+" was too slow for the bus", 60) + f.Explanation = capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus closed its connection (%s "+ + "in the last hour) and what was on its way was lost. It connects again by itself; this clears an hour after the last time.", who, count)) + f.Resolved = "Resolved: " + who + " kept up for an hour" + } else { + reason := strings.ToLower(c.Reason) + f.Said = fmt.Sprintf("the bus closed %s's connection %d: %s", c.User, c.CID, c.Reason) + f.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, newest at %s: %s", + who, times, last.UTC().Format("15:04 MST"), reason) + f.Headline = clip(capital(who)+" was cut off by the bus", 60) + f.Explanation = capital(fmt.Sprintf("The bus closed the connection of %s because it broke the bus's rules (%s, %s in the "+ + "last hour), so what it was sending was lost. This clears an hour after the last time.", who, reason, count)) + f.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour" + } + w.find(f) + if quiet := AdvisoryQuiet - w.now().Sub(last); quiet > 0 { + w.signal("S9", key, quiet, signalDue{Do: dueEnd, Finding: f}) + } else { + w.unfind(key) + } +} + +// BusUserWords is a bus user as the operator says it, and its machine (the controller's busUserWords). +func BusUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + cut := s[:n] + if i := strings.LastIndex(cut, " "); i > 0 { + cut = cut[:i] + } + return cut +} + +// ---- dead letters aging out ---- + +// SetRecount sets how DEAD_LETTERS is counted again when a restricted letter ages out. +func (w *Watchman) SetRecount(f func()) { + w.mu.Lock() + defer w.mu.Unlock() + w.recount = f +} + +// DeadAging is a restricted letter of a consumer (`.`) that ages out at at: a pulse expiring a second +// after it counts DEAD_LETTERS again, so the consumer's finding ends when its letters are gone (ADR 0376 §6), with no +// timer. The earliest of a consumer's letters is the one pulsed. +func (w *Watchman) DeadAging(key string, at time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + left := at.Sub(w.now()) + if left <= 0 { + left = 0 + } + pk := signalKey("dead", key) + if cur, ok := w.signals[pk]; ok && !cur.Finding.Since.IsZero() && cur.Finding.Since.Before(at) && cur.Finding.Since.After(w.now()) { + return + } + w.signal("dead", key, left+time.Second, signalDue{Do: dueRecount, + Finding: Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Summary: "counted again when it ages out", Source: "S9", Since: at}}) } func orWords(s, otherwise string) string { @@ -494,14 +779,13 @@ func orWords(s, otherwise string) string { // SignalRows are the rows the watchman holds, in the table's order, for `signals`. var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ - {"S1", "a machine's presence (not while it said it sleeps or shuts down)", delivery.PulseMoveBound.String(), KindSilent}, + {"S1", "a machine's presence (not while it said it sleeps or shuts down); a control node urgent after " + ControlNodeUrgentAfter.String(), SilentBound.String(), KindSilent}, {"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent}, {"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"}, {"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent}, {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, - {"S21", "a have refused or applied in part", ApplyEvery.String(), KindApplyFailing}, + {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, - {"health", "a have that waits for the operator", "at once", KindNeedsOperator}, } // Signal is one signal's pulse as `signals` shows it. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 50a42e6..0861f54 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -77,6 +77,13 @@ type Watchman struct { unhealthy map[string]int advised map[string]int advisedSince map[string]time.Time + // control is each node running the controller; failedSince each have's first failure still standing; provFailed each + // machine's provisioner client's failed reports; closures each bus closure counted, by finding key and identity. + control map[string]bool + failedSince map[string]time.Time + provFailed map[string]map[string]ProvReport + closures map[string]map[string]time.Time + recount func() // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -86,7 +93,8 @@ func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, - haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}} + haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, + control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}} } // Logf sets where a write that failed is said; it is silent until set. From 3b6f3af38e491b214ae79dda990a4974e037b8a8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:19:35 +0100 Subject: [PATCH 11/35] Grants part: create a module's log with the max age it declares, so a log's entries age out as its manifest says (issue 1952) Its nats fixture gives the tools bundle's account a secret the tool runtime can reach, as mesh-sdk 2e5cc4f asks. Pins mesh-sdk 2e5cc4f. --- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- modules/mesh-grants/grants_test.go | 13 +++++++++++-- modules/mesh-grants/internal/grants/records.go | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 17483ea..5676e43 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 6dce500..fe45f51 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec h1:OLXHSQkAHnf03hzJTxRrUsl7vMfMhOx43M5yJpi5L5Y= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index c6d9ec7..891abee 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -54,6 +54,7 @@ type fakeBus struct { hold chan struct{} // creates are the streams the part asked to create, refused when refuseCreate. creates []string + createdAge map[string]time.Duration refuseCreate bool // refuseUsers refuses the bus module's nats_users: the loaded list cannot be read. refuseUsers bool @@ -133,6 +134,10 @@ func (b *fakeBus) CreateDeclared(cfg jetstream.StreamConfig) error { b.mu.Lock() defer b.mu.Unlock() b.creates = append(b.creates, cfg.Name) + if b.createdAge == nil { + b.createdAge = map[string]time.Duration{} + } + b.createdAge[cfg.Name] = cfg.MaxAge if b.refuseCreate { return errors.New("Permissions Violation for Publish to $JS.API.STREAM.CREATE." + cfg.Name) } @@ -315,7 +320,7 @@ func (b *fakeBus) assign(node, module string) { const ( natsManifest = `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}` xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` + - `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history"}]}` + `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history","max-age-days":30}]}` yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}` ) @@ -1078,7 +1083,8 @@ func TestGrants(t *testing.T) { licence, _ := os.ReadFile(filepath.Join("testdata", "claude-licence-manager.json")) n := newFakeBus() // 1952/S5 (ADR 0564 Limits 4): the bus module's tools bundle declares its own account and what it may remove. - n.manifestOf("nats", strings.TrimSuffix(natsManifest, "}")+`,"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go",`+ + n.manifestOf("nats", strings.Replace(strings.TrimSuffix(natsManifest, "}"), `"broker":"/var/lib/nats/broker"`, `"broker":"/var/lib/nats/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"`, 1)+ + `,"resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711"}],"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go",`+ `"system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts"]}}]}}`) n.manifestOf("claude-licence-manager", string(licence)) n.assign("novox", "nats") @@ -1796,6 +1802,9 @@ func TestGrants(t *testing.T) { !strings.HasPrefix(first.Asserted[0], "mod-x at ") || !slices.Contains(b.events, grants.EventAsserted) { t.Fatalf("the first pass created %v, said %v, events %v", b.creates, first.Asserted, b.events) } + if b.createdAge["LOG_mod-x_history"] != 30*24*time.Hour { // a log's max-age-days is its stream's MaxAge (mesh-sdk 2e5cc4f) + t.Errorf("the log was created with max age %v, want 30 days", b.createdAge["LOG_mod-x_history"]) + } a := assertion() if !a.Asserted || a.Says != grants.CreatesMissing { t.Errorf("the assertion kept = %+v", a) diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index 6194d3f..cf2556e 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -233,7 +233,7 @@ func bucketsOf(m catalogue.Manifest) []buscompose.Bucket { func logsOf(m catalogue.Manifest) []buscompose.Log { var out []buscompose.Log for _, l := range m.Logs { - out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()}) + out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap(), MaxAgeDays: l.MaxAgeDays}) } return out } From 5235bb2e79c3e18cace2fbd992b16085bf0cd723 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:24:00 +0100 Subject: [PATCH 12/35] Assigner and grants part: record hand acts and warrants through the seat mesh-hand-acts, the controller's log only while that seat has no holder and the controller holds its lease (issue 1953 D8) Pins mesh-sdk 2d2aa7b in the L5 modules. --- .../cmd/mesh-delivery-assigner/main.go | 6 +-- .../cmd/mesh-delivery-assigner/pass_test.go | 4 +- .../cmd/mesh-delivery-assigner/puts.go | 16 +++++--- .../cmd/mesh-delivery-assigner/seat.go | 25 +++++++++---- .../cmd/mesh-delivery-assigner/seat_tools.go | 3 -- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../mesh-delivery-assigner/manifest_test.go | 3 +- modules/mesh-delivery-assigner/module.json | 2 + modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 +- modules/mesh-grants/grants_test.go | 9 ++++- modules/mesh-grants/internal/grants/inputs.go | 2 +- .../mesh-grants/internal/grants/takeover.go | 37 +++++++++++++++++++ modules/mesh-grants/module.json | 4 +- 15 files changed, 90 insertions(+), 33 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 996d423..23a76f4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -387,7 +387,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, return } if bus != nil { - p.acts = newPutActs(bus, wake, logger.Printf) + p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway) } logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket) } @@ -450,10 +450,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}} p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf) if bus != nil && kvs[delivery.AssignerPutsBucket] != nil { - p.acts = newPutActs(bus, wake, logger.Printf) + p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway) } if bus != nil { - p.seat = newSeatActs(bus, wake, logger.Printf) + p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway) } if js != nil { p.publish = streamPublisher(ctx, js, thisNode) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 35dc5af..6b7f420 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -187,7 +187,7 @@ func putOfTheBus(t *testing.T) { p := newPasser(kvs, now) bus := &fakeBus{calls: make(chan string, 8)} woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done - p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}) + p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil) p.acts.snapshot = func(context.Context, string, string) (string, error) { return "novox's restore point of nats (streams) taken 22:01", nil } @@ -288,7 +288,7 @@ func putOfTheBus(t *testing.T) { if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold { t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err) } - if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) { + if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat t.Errorf("the warrant was recorded as %q", call) } world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"]) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 10e48f9..26be2fa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "mesh-delivery-assigner/internal/assign" ) @@ -46,14 +47,16 @@ type putActs struct { snapped map[string]string snapNo map[string]string warranted map[string]bool + // rec records the put's warrant in the hand-act log (ADR 0565 §11). + rec handacts.Recorder warrantNo map[string]string said map[string]string // snapshot takes the bus's snapshot on node; a variable so a test takes none. snapshot func(ctx context.Context, node, module string) (string, error) } -func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs { - a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{}, +func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs { + a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{}, said: map[string]string{}} a.snapshot = a.snapshotTheBus return a @@ -381,8 +384,9 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri } } -// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the -// router's record of the teller's ask says who chose and how (ADR 0274). +// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and +// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's +// record of the teller's ask says who chose and how (ADR 0274). func (a *putActs) warrant(id, ask string) { a.mu.Lock() if a.warranted[id] { @@ -392,10 +396,10 @@ func (a *putActs) warrant(id, ask string) { a.warranted[id] = true a.mu.Unlock() go func() { - if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil { + if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil { a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err) a.mu.Lock() - a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err) + a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err) a.mu.Unlock() a.wake() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 42aa0bc..7ba0b4d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/compose/judge" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" @@ -67,6 +68,8 @@ type seatActs struct { export map[string]exportCheck // calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to. calls map[string]string + // rec records the seat's acts in the hand-act log (ADR 0565 §11). + rec handacts.Recorder } // exportCheck is the export read again for one proposal: running, its answer, or why it was not read. @@ -76,8 +79,9 @@ type exportCheck struct { err string } -func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs { - return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}} +func newSeatActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *seatActs { + return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}, + rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}} } // exportOf is the export read again for a proposal, asked once in the background. @@ -114,26 +118,31 @@ func (a *seatActs) forget(id string) { a.mu.Unlock() } -// record records once, in the controller's hand-act log, that placement was written on the operator's warrant (§5): the -// router's record of the teller's ask says who chose and how (ADR 0274). +// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's +// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and +// the controller's while that seat has no holder and the controller holds its lease (ADR 0565 §11, handacts.Recorder). func (a *seatActs) record(warrant string) { if a == nil || warrant == "" { return } ask, _, _ := strings.Cut(warrant, "@") go func() { - if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil { + if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil { a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err) + } else { + a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log) } }() } -// handAct records a hand-back in the controller's hand-act log (§7). +// handAct records a hand-back in the hand-act log (§7), as record does. func (a *seatActs) handAct(prop delivery.PlacementProposal) { go func() { - if _, err := a.bus.Call(handActCall, map[string]any{"what": "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "), - "why": prop.Why, "cause": "asked by " + prop.By + ", approved by the operator"}); err != nil { + if taken, err := a.rec.Record(handacts.Request{What: "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "), + Why: prop.Why, Cause: "asked by " + prop.By + ", approved by the operator"}); err != nil { a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err) + } else { + a.logf("the hand-back %s is recorded in the log of %s", prop.ID, taken.Log) } }() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index ebbf4fb..3c683f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -318,9 +318,6 @@ func (s *seatTools) leased() bool { return away } -// handActCall records a hand act in the controller's hand-act log. -const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act" - // doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back // proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys, // records the hand act and says placement-changed; every node returns to the controller's compositions. diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 97fcb8a..e1d9aad 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index ae67790..685613a 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 3365117..3964db8 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -79,7 +79,8 @@ func TestManifest(t *testing.T) { } if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin", "seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin", - "seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { + "seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.warranted", // ADR 0565 §11: the seat, the controller's only while it has no holder + "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if !slices.Equal(m.Emits, []string{"put"}) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 28aaf55..267eb62 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -47,6 +47,8 @@ "seat:mesh-controller.unassign", "seat:mesh-controller.unpin", "seat:mesh-controller.warranted", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now" ], diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 5676e43..361c8a5 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index fe45f51..e8df9ed 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 891abee..53b8e9b 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -1388,9 +1388,14 @@ func TestGrants(t *testing.T) { row.between() } said = nil + calls := len(b.calls) if err := p.Warranted(row.option, warrant(id)); err != nil { t.Fatal(err) } + // 1953/D8 (ADR 0565 §11): a switch made is recorded in the hand-act log's seat; one refused or declined is not. + if recorded := slices.Contains(b.calls[calls:], grants.CallRecordWarrant); recorded != (row.outcome == "recorded") { + t.Errorf("recorded in the hand-act log %v, the act %s", recorded, row.outcome) + } l := last() if l.Said != row.outcome || l.Ask != id || p.SwitchState() != row.state { t.Errorf("the act = %+v, the switch %s; want %s and %s", l, p.SwitchState(), row.outcome, row.state) @@ -1642,8 +1647,8 @@ func TestGrants(t *testing.T) { } } for _, c := range b.calls { - if c != grants.CallUsers && c != grants.CallStreams { - t.Errorf("called %s: the part calls only the bus module's read tools", c) + if c != grants.CallUsers && c != grants.CallStreams && c != grants.CallRecordWarrant { + t.Errorf("called %s: the part calls only the bus module's read tools, and the hand-act log for a switch", c) } } for _, e := range b.events { diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index e0f702d..556ff70 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -52,7 +52,7 @@ const ( ) // Invokes is every tool the part calls. -var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree} +var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant, CallRecordWarrantController} // Reader reads another part's state, named `.`. A refusal (a read not granted yet, a bucket not there) is // an error the part names as a gap and composes past: it never stops for one. diff --git a/modules/mesh-grants/internal/grants/takeover.go b/modules/mesh-grants/internal/grants/takeover.go index bd5e678..d5c94f7 100644 --- a/modules/mesh-grants/internal/grants/takeover.go +++ b/modules/mesh-grants/internal/grants/takeover.go @@ -1,6 +1,8 @@ package grants import ( + "git.novox.be/novox/mesh-sdk/go/handacts" + "crypto/rand" "encoding/hex" "encoding/json" @@ -378,6 +380,7 @@ func (p *Part) Warranted(o asks.Option, w asks.Warrant) error { if err := p.finish(s, a); err != nil { return err } + p.recordWarrant(w.Ask) // The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8). if err := p.syncFollow(s.State); err != nil { return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err) @@ -512,3 +515,37 @@ func (p *Part) TakeOverState() (any, error) { } return out, nil } + +// Module is the grants part's module, the asker of its take-over and hand-back. +const Module = "mesh-grants" + +// The hand-act log's verbs the part records its switches by (novox/hq ADR 0565 §11, ADR 0547 §6): the seat's, and the +// controller's while the seat has no holder and the controller holds its lease (handacts.Recorder). +const ( + CallRecordWarrant = "seat:" + handacts.Seat + ".warranted" + CallRecordWarrantController = "seat:mesh-controller.warranted" +) + +// recordWarrant records, in the hand-act log, that a take-over or a hand-back was made on the operator's warrant for ask. +// Said, never fatal: the switch stands either way. +func (p *Part) recordWarrant(ask string) { + if p.Call == nil { + return + } + rec := handacts.Recorder{Ask: p.Call.Call, LeaseAway: func() bool { + if p.Read == nil { + return false + } + away, _ := leaseHandedAway(p.Read) + return away + }} + taken, err := rec.RecordWarrant(Module, ask) + if p.Logf == nil { + return + } + if err != nil { + p.Logf("the switch on the warrant %s is not recorded in the hand-act log: %v", ask, err) + return + } + p.Logf("the switch on the warrant %s is recorded in the log of %s", ask, taken.Log) +} diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 2294408..2d25d3a 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -30,7 +30,9 @@ "nats.nats_streams", "nats.nats_user_can", "nats.nats_users", - "seat:mesh-controller.root-free" + "seat:mesh-controller.root-free", + "seat:mesh-hand-acts.warranted", + "seat:mesh-controller.warranted" ], "emits": [ "asserted", From 2531cd42b6ffe97a2dee1321e95374817eea10d1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:24:50 +0100 Subject: [PATCH 13/35] Checker: hold that the change's own failing test ends its level there, as mesh-sdk's build asks now do, so a failing check costs only its own tests (issue 1289) Pins mesh-sdk 2d2aa7b. --- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- .../internal/check/examiner_test.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index b431a85..39195d1 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 043e05f..685613a 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 5d708df..d80dd7e 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -425,9 +425,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1 } } -// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run -// after it (so one check names every failure), and no later level runs. -func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) { +// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends +// there: no other package of the level and no later level runs, so the rest of the suite is not paid for. +func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) { b := newBench(t) b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"} b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} @@ -443,8 +443,8 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing. t.Fatalf("verdict = %+v", out.Verdict) } tests := b.steps.tool("test") - if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" { - t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests) + if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" { + t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests) } if n := len(b.steps.tool("digest")); n != 1 { t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n) From 2488d839ab90abb503d9505f746d1e294593de13 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:29:04 +0100 Subject: [PATCH 14/35] Bridge, indexer and view: carry the catalogue's predefined list on its merged message, and keep it and each open pull request's config declarations in the indexer's states, so mesh-config reads them in place of the controller's mirror and pull-declares (issue 1953 D1-D3) The view's tools read both states. Pins mesh-sdk 2d2aa7b. --- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 +- .../internal/bridge/bridge.go | 19 ++ .../internal/bridge/files_test.go | 21 ++ .../cmd/mesh-delivery-indexer/main.go | 6 +- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 +- .../internal/index/candidates.go | 16 + .../internal/index/control_records.go | 292 ++++++++++++++++++ .../internal/index/control_records_test.go | 88 ++++++ .../internal/index/manifests_test.go | 9 +- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 2 + .../cmd/mesh-delivery-view-tools/main.go | 21 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/read.go | 35 +++ modules/mesh-delivery-view/manifest_test.go | 4 +- modules/mesh-delivery-view/module.json | 4 + 19 files changed, 521 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/control_records.go create mode 100644 modules/mesh-delivery-indexer/internal/index/control_records_test.go diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index f7661dc..418479d 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 6245202..685613a 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index 087279a..9538477 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -46,6 +46,13 @@ var ErrNotFound = errors.New("the forge has no such file") // says is not there is ErrNotFound, any other failure any other error (issue 1448). type Files func(owner, repo, path, ref string) ([]byte, error) +// CatalogRepository is the catalogue, whose merged message carries its predefined list, and PredefinedFile that list's +// file at its root (novox/hq ADR 0565 §2, ADR 0348 §7). +const ( + CatalogRepository = "novox/mesh-catalog" + PredefinedFile = "predefined.json" +) + // LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465 // clause 1: go.sum for a Go module, package-lock.json for a Node one). var LockFiles = []string{"go.sum", "package-lock.json"} @@ -315,6 +322,18 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag } f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks} + if repository == CatalogRepository { + // The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. + raw, err := files(p.Owner, p.Repo, PredefinedFile, p.MergeSHA) + switch { + case errors.Is(err, ErrNotFound): + f.PredefinedRead = true + case err != nil: + return Message{}, fmt.Errorf("%w: %s: %s of %s at the merge commit %s: %v", ErrRetry, subject, PredefinedFile, repository, p.MergeSHA, err) + default: + f.Predefined = json.RawMessage(raw) + } + } body, err := delivery.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index 3e5e681..6c558c8 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -97,6 +97,27 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 t.Errorf("the forge was asked for %s", c.path) } } + // 1953/D1 (ADR 0565 §2): the catalogue's merged message carries its predefined.json at the merge, or says it holds + // none; another repository's asks the forge for none. + if !f.PredefinedRead || len(f.Predefined) != 0 { + t.Errorf("a merge of the catalogue with no predefined.json does not say so: %s", msg.Data) + } + forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`) + msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead { + t.Errorf("predefined = %s (%v)", f.Predefined, err) + } + forge.calls = nil + msg, _ = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + for _, c := range forge.calls { + if c.path == PredefinedFile { + t.Errorf("another repository's merge asked for %s", c.path) + } + } + forge.errs = map[string]error{PredefinedFile: errors.New("forge down")} + if _, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); !errors.Is(err, ErrRetry) { + t.Errorf("a forge that cannot say is %v, want a retry", err) + } // Issue 1126: a merge reads the forge only for the module directories whose tree changed. t.Run("a merge reads only the directories whose tree changed", mergeReadsOnlyChangedDirs) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index e55b265..05e6a27 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -274,6 +274,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then // a merge's taken key is said and not written, and the rest of the indexer runs. taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf) + // The control records (ADR 0565 §2), new with this delivery and bound when they appear: until then nothing is written to them. + predefined := process.AwaitBucket(ctx, js, conn, index.PredefinedBucket, "no predefined list is written", logger.Printf) + configCandidates := process.AwaitBucket(ctx, js, conn, index.ConfigCandidatesBucket, "no pull request's config declarations are written", logger.Printf) // The controller's build records only name the directory of a module whose key names none (fillPaths): without them // such a module waits for its path, every other is indexed (issue 1945). builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) @@ -303,7 +306,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer calls.Close() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf, + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 96d5ca6..52694c1 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 043e05f..685613a 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index ebb36fe..5f85619 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -48,6 +48,8 @@ type Indexer struct { Modules Store // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store + // Predefined and ConfigCandidates are the control records' states (control_records.go, ADR 0565 §2); nil writes none. + Predefined, ConfigCandidates Store // Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none. Taken Store // Now is the clock of a taken value; nil is time.Now. @@ -121,6 +123,14 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil { return applied, err } + // The control records (ADR 0565 §2): the catalogue's predefined list, and the open pull requests' config + // declarations judged again against the trunk this merge moved. + if err := ix.writePredefined(ctx, f); err != nil { + return applied, err + } + if err := ix.configCandidatesAfterMerge(ctx, f, writtenModules(w.written)); err != nil { + return applied, err + } } ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) } @@ -134,6 +144,12 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, if held, err = ix.heldModules(ctx); err != nil { return applied, err } + if err := ix.configCandidateOfPull(ctx, f, held); err != nil { + if !errors.Is(err, ErrUnreadable) { + return applied, err + } + ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err) + } } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go new file mode 100644 index 0000000..0feb4ec --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -0,0 +1,292 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The indexer's two control records (novox/hq ADR 0565 §2, issue 1953 D2, D3): the catalogue's predefined list, read from the +// forge bridge's merged message, and each open pull request's config declarations, read from its pull message's manifests +// against the trunk's manifests the indexer holds. mesh-config reads both in place of the controller's mirror and its +// `pull-declares`; a key's write is its event, nothing polls. + +// The two states' buckets. +var ( + PredefinedBucket = delivery.IndexerModule + "_" + delivery.IndexerPredefinedState + ConfigCandidatesBucket = delivery.IndexerModule + "_" + delivery.IndexerConfigCandidatesState +) + +// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older +// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1). +func (ix *Indexer) writePredefined(ctx context.Context, f delivery.Forge) error { + if ix.Predefined == nil || len(f.Predefined) == 0 { + return nil + } + return ix.notYet(PredefinedBucket, write(ctx, ix.Predefined, delivery.PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) { + if found { + var cur delivery.PredefinedKept + if err := json.Unmarshal(value, &cur); err == nil { + if cur.Commit == f.Merge && string(cur.List) == string(f.Predefined) { + return nil, false, nil + } + older, err := ix.older(ctx, f.Repository, f.Merge, cur.Commit) + if err != nil { + return nil, false, err + } + if older { + ix.logf("%s: the predefined list of %.12s is older than the one kept (%.12s): left", f.Repository, f.Merge, cur.Commit) + return nil, false, nil + } + } + } + next, err := json.Marshal(delivery.PredefinedKept{V: delivery.Version, Commit: f.Merge, At: ix.now(), List: f.Predefined}) + return next, false, err + })) +} + +// notYet says a state not on the bus yet (it comes with this delivery, bound when it appears) and lets the message go: +// what it would have written is written at the next message that carries it. +func (ix *Indexer) notYet(bucket string, err error) error { + if errors.Is(err, process.ErrNotYet) { + ix.logf("the state %s is not on the bus yet: nothing is written to it for this message", bucket) + return nil + } + return err +} + +func (ix *Indexer) now() time.Time { + if ix.Now != nil { + return ix.Now() + } + return time.Now() +} + +// PullName is a pull request as mesh-config names it, `owner/repository#number`. +func PullName(repository string, number int) string { return repository + "#" + strconv.Itoa(number) } + +// DeclarationsOf is a manifest's config declarations by key, in the layout of the controller's declared-config mirror: the +// config form's `config`, else the settings form's `settings`, each default and enum encoded the one way. +func DeclarationsOf(raw string) (map[string]delivery.ConfigDeclaration, error) { + var m struct { + Config map[string]delivery.ConfigDeclaration `json:"config"` + Settings map[string]delivery.ConfigDeclaration `json:"settings"` + } + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return nil, err + } + declared := m.Config + if declared == nil { + declared = m.Settings + } + out := map[string]delivery.ConfigDeclaration{} + for key, d := range declared { + for _, field := range []*json.RawMessage{&d.Default, &d.Enum} { + if len(*field) == 0 { + continue + } + var v any + if err := json.Unmarshal(*field, &v); err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + canonical, err := json.Marshal(v) + if err != nil { + return nil, err + } + *field = canonical + } + out[key] = d + } + return out, nil +} + +// changedDeclarations are the declarations head adds or changes against trunk. +func changedDeclarations(head, trunk map[string]delivery.ConfigDeclaration) map[string]delivery.ConfigDeclaration { + out := map[string]delivery.ConfigDeclaration{} + for key, d := range head { + if before, there := trunk[key]; !there || !reflect.DeepEqual(before, d) { + out[key] = d + } + } + return out +} + +// trunkManifest is the manifest the indexer holds for a module on the trunk, "" when it holds none. +func (ix *Indexer) trunkManifest(ctx context.Context, held map[string]delivery.Module, module string) (string, error) { + m, ok := held[module] + if !ok || m.Manifest == "" || ix.Manifests == nil { + return "", nil + } + raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest) + if err != nil || !found { + return "", err + } + kept, err := delivery.DecodeManifest(raw) + if err != nil { + return "", err + } + return kept.JSON, nil +} + +// candidateDeclares are, per module of a pull request's head, the declarations its manifest adds or changes against the +// trunk's (an empty set for a module that adds none): heads are the head's manifests by module. +func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]delivery.Module, heads map[string]string) (map[string]map[string]delivery.ConfigDeclaration, error) { + out := map[string]map[string]delivery.ConfigDeclaration{} + for module, raw := range heads { + head, err := DeclarationsOf(raw) + if err != nil { + return nil, fmt.Errorf("%w: the manifest of %s at the head: %v", ErrUnreadable, module, err) + } + trunkRaw, err := ix.trunkManifest(ctx, held, module) + if err != nil { + return nil, err + } + trunk := map[string]delivery.ConfigDeclaration{} + if trunkRaw != "" { + if trunk, err = DeclarationsOf(trunkRaw); err != nil { + return nil, fmt.Errorf("the trunk's manifest of %s: %w", module, err) + } + } + if changed := changedDeclarations(head, trunk); len(changed) > 0 { + out[module] = changed + } + } + return out, nil +} + +// writeConfigCandidate writes a pull request's key of config-candidates from its head's manifests by module: written when +// what it says differs (its time apart), never for the same head and declarations twice. +func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string, number int, head string, held map[string]delivery.Module, heads map[string]string) error { + if ix.ConfigCandidates == nil { + return nil + } + pull := PullName(repository, number) + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + return fmt.Errorf("%w: %v", ErrUnreadable, err) + } + declares, err := ix.candidateDeclares(ctx, held, heads) + if err != nil { + return err + } + return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + var cur delivery.ConfigCandidate + if json.Unmarshal(value, &cur) == nil && cur.Head == head && reflect.DeepEqual(cur.Declares, declares) { + return nil, false, nil + } + } + next, err := json.Marshal(delivery.ConfigCandidate{V: delivery.Version, Pull: pull, Head: head, Declares: declares, At: ix.now()}) + return next, false, err + })) +} + +// dropConfigCandidate deletes a pull request's key: it merged or closed. +func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, number int) error { + if ix.ConfigCandidates == nil { + return nil + } + key, err := delivery.ConfigCandidateKey(PullName(repository, number)) + if err != nil { + return nil + } + return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil })) +} + +// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written, +// a closed one deleted. A head held again without its directories leaves the key as it is. +func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, held map[string]delivery.Module) error { + if f.Action == delivery.PullClosed { + return ix.dropConfigCandidate(ctx, f.Repository, f.Number) + } + if !f.DirsKnown { + return nil + } + heads := map[string]string{} + for dir, raw := range f.Manifests { + heads[moduleOfDir(held, f, dir)] = raw + } + return ix.writeConfigCandidate(ctx, f.Repository, f.Number, f.Head, held, heads) +} + +// configCandidatesAfterMerge writes again every open pull request's key that names a module the merge changed, since what +// it adds is judged against the trunk (ADR 0565 §2), from the head's manifests its candidate keeps; and deletes the merged +// pull request's. +func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Forge, changed []string) error { + if ix.ConfigCandidates == nil { + return nil + } + if err := ix.dropConfigCandidate(ctx, f.Repository, f.Number); err != nil { + return err + } + if len(changed) == 0 || ix.Store == nil || ix.Manifests == nil { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + if _, err := ix.ConfigCandidates.Keys(ctx); errors.Is(err, process.ErrNotYet) { + return nil + } + groups, err := ix.Store.Keys(ctx) + if err != nil { + return err + } + var errs []error + for _, g := range groups { + raw, _, found, err := ix.Store.Get(ctx, g) + if err != nil || !found { + continue + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + continue + } + for _, p := range c.Pulls { + if !touchesAny(p, changed) { + continue + } + heads := map[string]string{} + for module, fp := range p.Fps { + mraw, _, found, err := ix.Manifests.Get(ctx, fp.Manifest) + if err != nil || !found { + continue + } + if kept, err := delivery.DecodeManifest(mraw); err == nil { + heads[module] = kept.JSON + } + } + if err := ix.writeConfigCandidate(ctx, p.Repository, p.Number, p.Head.Head, held, heads); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func touchesAny(p delivery.CandidatePull, modules []string) bool { + for _, m := range modules { + if _, ok := p.Fps[m]; ok { + return true + } + } + return false +} + +// writtenModules are the modules a merged message wrote, from its written keys (`modules.`). +func writtenModules(written []string) []string { + var out []string + for _, k := range written { + out = append(out, strings.TrimPrefix(k, "modules.")) + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go new file mode 100644 index 0000000..119a7fe --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -0,0 +1,88 @@ +package index + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// 1953/D2, D3 (novox/hq ADR 0565 §2): the indexer keeps the catalogue's predefined list from the merged message, a merge older +// than the one kept never replacing it; and each open pull request's config declarations its head adds or changes against the +// trunk's manifest, an empty set when none, written only when they change and deleted when it merges or closes. +func TestTheControlRecords(t *testing.T) { + ctx := context.Background() + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + t.Run("the predefined list, never replaced by an older merge", func(t *testing.T) { + ix, _, _ := manifestsIndexer(t) + pre := newMem() + ix.Predefined, ix.Now = pre, func() time.Time { return at } + f := mergedMsg(catalogueRepo, map[string]string{"modules/a": "t1"}) + f.Predefined = json.RawMessage(`{"font-size":{"type":"integer","value":13}}`) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(pre.vals[delivery.PredefinedListKey], &kept); err != nil || kept.Commit != merge || string(kept.List) != string(f.Predefined) { + t.Fatalf("kept = %+v (%v)", kept, err) + } + writes := pre.writes + if err := ix.Apply(ctx, f); err != nil || pre.writes != writes { + t.Errorf("the same list at the same merge wrote again (%v)", err) + } + older := f + older.Merge, older.Predefined = strings.Repeat("e", 40), json.RawMessage(`{}`) + ix.Contains = func(context.Context, string, string, string) (bool, error) { return true, nil } + if err := ix.writePredefined(ctx, older); err != nil || pre.writes != writes { + t.Errorf("an older merge replaced the list (%v)", err) + } + ix.Predefined = &memStore{unbound: process.ErrNotYet} + if err := ix.writePredefined(ctx, f); err != nil { + t.Errorf("a state not on the bus yet stopped the message: %v", err) + } + }) + t.Run("a pull request's config declarations against the trunk", func(t *testing.T) { + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + cands := newMem() + ix.ConfigCandidates, ix.Now = cands, func() time.Time { return at } + trunk := `{"module":"avahi","version":"1","config":{"port":{"type":"integer","default":5353,"why":"mDNS"}}}` + if err := ix.Apply(ctx, withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t1"}), map[string]string{"modules/avahi": trunk}, nil)); err != nil { + t.Fatal(err) + } + key := "novox.mesh-catalog.7" + head := strings.Replace(trunk, `"why":"mDNS"}`, `"why":"mDNS"},"name":{"type":"string","default":"x","why":"shown"}`, 1) + p := withManifests(pull(catalogueRepo, 7, "feat/y", h1, "modules/avahi"), map[string]string{"modules/avahi": head}, nil) + if err := ix.Apply(ctx, p); err != nil { + t.Fatal(err) + } + var c delivery.ConfigCandidate + if err := json.Unmarshal(cands.vals[key], &c); err != nil || c.Pull != "novox/mesh-catalog#7" || c.Head != h1 || len(c.Declares["avahi"]) != 1 || + string(c.Declares["avahi"]["name"].Default) != `"x"` { + t.Fatalf("candidate = %+v (%v)", c, err) + } + writes := cands.writes + if err := ix.Apply(ctx, p); err != nil || cands.writes != writes { + t.Errorf("the same head wrote again (%v)", err) + } + same := withManifests(pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40), "modules/avahi"), map[string]string{"modules/avahi": trunk}, nil) + if err := ix.Apply(ctx, same); err != nil { + t.Fatal(err) + } + c = delivery.ConfigCandidate{} + if err := json.Unmarshal(cands.vals[key], &c); err != nil || len(c.Declares) != 0 || c.Declares == nil { + t.Errorf("a head declaring nothing new = %+v, want an empty set", c) + } + closed := pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40)) + closed.Action = delivery.PullClosed + if err := ix.Apply(ctx, closed); err != nil { + t.Fatal(err) + } + if _, ok := cands.vals[key]; ok { + t.Error("a closed pull request's declarations stayed") + } + }) +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 08455bb..064dc48 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -42,7 +42,12 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri return f } -func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 +func TestAMergedManifestIsKeptOnceByItsDigestAndNamedByTheModule(t *testing.T) { + t.Run("kept by its digest and named by the module", mergedManifestIsKeptByItsDigestAndNamedByTheModule) + t.Run("the same manifest applied again is written once", sameManifestAppliedAgainIsWrittenOnce) +} + +func mergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 avahi := catalogueManifest(t, "avahi") ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) @@ -63,7 +68,7 @@ func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // } } -func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 +func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 avahi := catalogueManifest(t, "avahi") ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 900ac2c..603cb46 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -86,7 +86,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want none", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-assigner.placement", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 693205a..faeb9f8 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -16,6 +16,8 @@ { "name": "heard", "history": 64 }, { "name": "manifests", "history": 1 }, { "name": "modules", "history": 20 }, + { "name": "predefined", "history": 5 }, + { "name": "config-candidates", "history": 5 }, { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 } ], diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 3abc440..e0bbef6 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -104,6 +104,23 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { group, _ := args["group"].(string) return r.Candidate(group) }}, + {Name: view.PredefinedReadTool, + Description: "The catalogue's predefined list as the indexer keeps it (novox/hq ADR 0565 §2): its key list, the list whole as " + + "predefined.json holds it with the merge it was read at; a merge older than that never replaces it. mesh-config " + + "publishes the predefined keys from it after its take-over. Read-only.", + Run: func(map[string]any) (any, error) { + return view.ControlRecord(r.PredefinedState, delivery.IndexerPredefinedState, delivery.PredefinedListKey) + }}, + {Name: view.ConfigCandidatesReadTool, + Description: "Each open pull request's config declarations as the indexer keeps them (novox/hq ADR 0565 §2): per pull " + + "request (owner.repository.number), its head and per module the config keys its manifests add or change against the " + + "trunk's, an empty set when none; deleted when it merges or closes. Give key for one. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "one pull request's key, owner.repository.number"}}}, + Run: func(args map[string]any) (any, error) { + key, _ := args["key"].(string) + return view.ControlRecord(r.ConfigCandidatesState, delivery.IndexerConfigCandidatesState, key) + }}, {Name: view.CandidatesReadTool, Description: "Every candidate the indexer holds, by group, each as stored under candidates.; with module, only those with a " + "pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.", @@ -245,7 +262,9 @@ func main() { HeardState: read(stdio.State("mesh-delivery-indexer.heard")), SignersState: read(stdio.State("mesh-vault.signers")), ConditionsState: read(stdio.State(view.StatusConditionsState)), AttemptsState: read(stdio.State(view.StatusAttemptsState)), - PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState))} + PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState)), + PredefinedState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerPredefinedState)), + ConfigCandidatesState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState))} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 82a7256..96438b1 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index ae67790..685613a 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 58e45a3..cc9b8f8 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -50,6 +50,41 @@ type Reader struct { // ConditionsState, AttemptsState, PresenceState and StuckState are what status reads beside the wants and the haves // (ADR 0564 §10): the condition keeper's open conditions, mesh-healers' attempts, the watchman's presence and stuck keys. ConditionsState, AttemptsState, PresenceState, StuckState StateRead + // PredefinedState and ConfigCandidatesState are the indexer's control records (ADR 0565 §2, §15); unset reads none. + PredefinedState, ConfigCandidatesState StateRead +} + +// The read tools of the indexer's control records (ADR 0565 §15): the indexer serves no tool of its own (ADR 0437 §4). +const ( + PredefinedReadTool = "indexer_predefined" + ConfigCandidatesReadTool = "indexer_config_candidates" +) + +// ControlRecord is every key of one of the indexer's control records as stored, or one key, with how many it holds. +func ControlRecord(s StateRead, name, key string) (any, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("the indexer's %s cannot be read here", name) + } + if key = strings.TrimSpace(key); key != "" { + raw, err := s.Get(key) + if err != nil { + return nil, err + } + if raw == nil { + return nil, fmt.Errorf("the indexer's %s holds no key %s", name, key) + } + return raw, nil + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + values, err := s.values(keys) + if err != nil { + return nil, err + } + return map[string]any{"state": name, "holds": len(keys), "keys": values}, nil } // Status is the view's derivation of status at now (status.go). diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 70a65c1..6aea715 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 5640a25..b82fde5 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -9,6 +9,8 @@ "indexer_modules", "indexer_candidate", "indexer_candidates", + "indexer_predefined", + "indexer_config_candidates", "builder_build", "builder_builds", "assigner_wants", @@ -50,8 +52,10 @@ "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", + "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-conditions.conditions", From 3aec7f9b02c942192fbf4dd29ab3ff5e8032cfb7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:34:08 +0100 Subject: [PATCH 15/35] Assigner: hold the seat mesh-wants, keep a person's upgrade policy and judge moves by it, say a policy split with the controller as a finding, and answer the bus step, so the policies and the bus step leave the controller (issue 1953 D4-D6) --- .../cmd/mesh-delivery-assigner/main.go | 30 +- .../cmd/mesh-delivery-assigner/main_test.go | 4 +- .../cmd/mesh-delivery-assigner/pass.go | 64 +++- .../cmd/mesh-delivery-assigner/puts.go | 5 +- .../mesh-delivery-assigner/seat_tools_test.go | 114 +++++++ .../cmd/mesh-delivery-assigner/tools.go | 7 +- .../cmd/mesh-delivery-assigner/wants_tools.go | 300 ++++++++++++++++++ .../internal/assign/findings.go | 1 + .../internal/assign/policy.go | 135 ++++++++ .../internal/assign/wants.go | 13 +- .../internal/assign/wants_test.go | 6 + .../mesh-delivery-assigner/manifest_test.go | 15 +- modules/mesh-delivery-assigner/module.json | 12 +- 13 files changed, 686 insertions(+), 20 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go create mode 100644 modules/mesh-delivery-assigner/internal/assign/policy.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 23a76f4..934df8f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -85,7 +85,7 @@ const goBuildModule = "mesh-build-go" // (ADR 0567 §1), named after the assigner as every module state is. var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket, - delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket} + delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket} // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). @@ -108,6 +108,7 @@ var keptStates = []announce.State{ {Name: "puts", Readable: true}, {Name: "wants", Readable: true}, {Name: health.FindingsState, Readable: true}, + {Name: delivery.AssignerPoliciesState, Readable: true}, } // buckets is every bucket the assigner opened, by name. @@ -156,6 +157,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == assign.PoliciesBucket && err != nil { + // The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no + // person's policy is read, so the controller's beside each composition holds, as before. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil { // New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the // router and kept as no finding, as before. @@ -210,7 +217,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } thisNode = account.Node logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) - return follow(ctx, js, conn, kvs, findingsOf(ctx, kvs, logger.Printf).Say(sayVia(ctx, js)), emitting{said}, logger) + findings := findingsOf(ctx, kvs, logger.Printf) + return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger) } // findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the @@ -222,6 +230,13 @@ func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) ass if kv := kvs[assign.FindingsBucket]; kv != nil { f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err } f.Delete = func(key string) error { return kv.Delete(ctx, key) } + f.Keys = func() ([]string, error) { + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err + } } var keeper atomic.Bool f.ToKeeper = keeper.Load @@ -297,11 +312,12 @@ var watched = []string{ delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket, + assign.PoliciesBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -393,7 +409,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket) { continue // not on the bus at this start (open said so) } started++ @@ -423,10 +439,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket} { + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket} { if kvs[name] == nil && js != nil { name := name - process.AwaitBucket(ctx, js, nc, name, "the controller's grants are read", + process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} }) } } @@ -447,7 +463,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, case <-settled: } } - p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}} + p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}} p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf) if bus != nil && kvs[delivery.AssignerPutsBucket] != nil { p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index f7a91d4..d2a1a83 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -1,6 +1,8 @@ package main import ( + "mesh-delivery-assigner/internal/assign" + "context" "errors" "fmt" @@ -110,7 +112,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}} kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote - go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0)) + go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0)) select { case key := <-wrote: if key != "want.ace.node" { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 24d22c5..facdff2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -27,6 +27,10 @@ type passer struct { // pass says a condition once and a restart says it again (the router keeps one per key). say func(asks.Condition) error said map[string]bool + // findings is the assigner's findings bucket (its said conditions and policy-split, ADR 0564 §2, ADR 0565 §8); splits the + // modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left. + findings assign.Findings + splits map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -129,6 +133,8 @@ func (p *passer) wants(ctx context.Context) { } } wanted := p.w.wantsOf() + chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() puts := p.putsByModule() @@ -154,7 +160,7 @@ func (p *passer) wants(ctx context.Context) { for name := range names { fp := trunk[name].FP2 name := name - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { if placed[node] { @@ -932,3 +938,59 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo } } } + +// chosenOf is a person's choice for module, nil when none. +func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy { + if c, ok := chosen[module]; ok { + return &c + } + return nil +} + +// policySplits writes the finding policy-split for each module whose policy at the assigner differs from one the controller +// writes beside a composition of it, and deletes it once they agree (ADR 0565 §8): written when a split begins, never again +// while it stands. The first pass after a start deletes every one a previous process left that no longer splits. +func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) { + if p.findings.Put == nil { + return + } + now := time.Now + if p.now != nil { + now = p.now + } + split := map[string]bool{} + for _, s := range assign.Splits(manifests, chosen, assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))) { + split[s.Module] = true + if p.splits[s.Module] { + continue + } + value, err := assign.PolicySplitFinding(s, now()) + if err == nil { + err = p.findings.Put(assign.PolicySplitKey(s.Module), value) + } + if err != nil { + p.logf("the policy split of %s is not written: %v", s.Module, err) + delete(split, s.Module) + } + } + gone := p.splits + if gone == nil && p.findings.Keys != nil { + gone = map[string]bool{} + if keys, err := p.findings.Keys(); err == nil { + for _, k := range keys { + if m, ok := strings.CutSuffix(strings.TrimPrefix(k, "module."), "."+assign.KindPolicySplit); ok && strings.HasPrefix(k, "module.") { + gone[m] = true + } + } + } + } + for m := range gone { + if !split[m] { + if err := p.findings.Delete(assign.PolicySplitKey(m)); err != nil { + p.logf("the policy split of %s is not deleted: %v", m, err) + split[m] = true // tried again at the next pass + } + } + } + p.splits = split +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 26be2fa..2def647 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -186,7 +186,10 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo w.Composed, w.Granted = p.composedFor(node, module, put.FP) if d := w.Composed; d != nil { policy := d.Policy - if policy == "" { + chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + if c, ok := chosen[module]; ok { + policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7) + } else if policy == "" { policy, _, _ = catalogue.DerivedUpgrade(manifest) } w.Recorded = policy == catalogue.PolicyRecord diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 89c25dd..cba607c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -12,6 +12,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-assigner/internal/assign" ) func (s memStore) del(state, key string) error { @@ -155,6 +157,118 @@ func TestTheSeatsVerbs(t *testing.T) { if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil { t.Errorf("the seat's verbs as served: %+v", tools) } + t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) +} + +// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is +// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back; +// the bus is refused; compare lists the modules the controller writes another policy for; the process writes policy-split +// for a split and deletes it once they agree; bus names the node behind the indexer's newest. +func wantsVerbs(t *testing.T) { + fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64) + st := memStore{} + for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} { + st[state] = &memKV{values: map[string][]byte{}} + } + manifest := func(name, extra string) { + raw := `{"module":"` + name + `","version":"1"` + extra + `}` + digest := delivery.ManifestDigest([]byte(raw)) + st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw}) + st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"}) + } + manifest("web", "") + manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`) + manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`) + for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} { + st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + } + composed := func(module, policy string) { + key, _ := delivery.ComposedKey("anchor", module, fpNew) + st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64), + Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"}) + } + composed("web", catalogue.PolicyRecord) + composed("photos", catalogue.PolicyRecord) + var asked []string + var said []string + w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key) + return json.RawMessage(`{"ok":true}`), nil + }, func(event, id string, body any) error { said = append(said, event+" "+id); return nil }) + policyOf := func(module string) policyRow { + out, err := w.doUpgrade(map[string]any{"module": module}, "op") + if err != nil { + t.Fatal(err) + } + return out.(map[string]any)["modules"].([]policyRow)[0] + } + if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { + t.Errorf("photos = %+v", p) + } + if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus { + t.Errorf("nats = %+v", p) + } + // compare: web rolls at the assigner and records at the controller. + out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op") + if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil { + t.Errorf("splits = %+v", splits) + } + // The process says the split as its finding, and deletes it once they agree. + findings := map[string][]byte{} + p := &passer{w: newWorld(), logf: func(string, ...any) {}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }, + Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}} + for k, v := range st[composedState].values { + p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now()) + } + man := map[string]catalogue.Manifest{} + for _, name := range []string{"web", "photos"} { + raw, _, _, _ := st.get(modulesState, name) + mod, _ := delivery.DecodeModule(raw) + mraw, _, _, _ := st.get(manifestsState, mod.Manifest) + kept, _ := delivery.DecodeManifest(mraw) + man[name], _ = catalogue.ParseManifest([]byte(kept.JSON)) + } + findings["module.gone.policy-split"] = []byte("{}") + p.policySplits(man, nil) + if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 { + t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings) + } + // a person's record for web: written, recorded, said; the split is gone. + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil { + t.Fatal(err) + } + chosen, _ := assign.DecodePolicies(st[policiesState].values) + if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") || + len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") { + t.Errorf("chosen %+v, asked %v, said %v", c, asked, said) + } + p.policySplits(man, chosen) + if len(findings) != 0 { + t.Errorf("findings after the choice = %v", findings) + } + if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil { + t.Error("the bus was given a policy") + } + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil { + t.Error("a policy without why was set") + } + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 { + t.Errorf("default left %v (%v)", st[policiesState].values, err) + } + // bus: the node behind the indexer's newest. + b, err := w.doBus(nil) + if err != nil { + t.Fatal(err) + } + nodes := b.(map[string]any)["nodes"].([]busNode) + if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") { + t.Errorf("bus = %+v, asked %v", b, asked) + } + if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil { + t.Errorf("the seat's verbs as served: %+v", tools) + } } // readManifest is the module's manifest, read as the controller reads it. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 0e47bb4..74a5180 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -351,7 +351,12 @@ func serveTools() error { time.Sleep(wait) } }() - return stdio.Serve("", append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)) + tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...) + tools = append(tools, newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error { + // The seat's event, said on the seat: a claimed seat's event is never the module's own. + return stdio.SeatPublish(wantsEventSubject(e), b, id) + }).tools()...) + return stdio.Serve("", tools) } // shortFP is a fingerprint's first twelve hex digits. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go new file mode 100644 index 0000000..945a959 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go @@ -0,0 +1,300 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "mesh-delivery-assigner/internal/assign" +) + +// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the +// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies` +// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from +// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads +// `policies` when it judges a move and writes the finding policy-split while a split stands. + +// The states the verbs read, as the runtime names them. +const ( + policiesState = delivery.AssignerPoliciesState + manifestsState = delivery.IndexerModule + ".manifests" + composedState = "mesh-controller.composed" + // PolicyChangedEvent is the seat's event of a person's policy set or taken back. + PolicyChangedEvent = "policy-changed" + // busStepCause is the cause the bus's step is recorded under in the hand-act log. + busStepCause = "bus-upgrade" +) + +// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit. +type wantsTools struct { + st seatStore + ask func(key string, body any) (json.RawMessage, error) + emit func(event, id string, body any) error + now func() time.Time +} + +// wantsEventSubject is the subject of one of the seat's events. +func wantsEventSubject(event string) string { + return "mesh.seat." + catalogue.WantsSeat + ".event." + event +} + +func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools { + return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now} +} + +// leased says the controller's lease is handed away. +func (s *wantsTools) leased() bool { + raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err) + return away +} + +// values reads every key of a state. +func (s *wantsTools) values(state string) (map[string][]byte, error) { + keys, err := s.st.keys(state) + if err != nil { + return nil, fmt.Errorf("%s cannot be listed: %w", state, err) + } + out := map[string][]byte{} + for _, k := range keys { + raw, _, found, err := s.st.get(state, k) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err) + } + if found { + out[k] = raw + } + } + return out, nil +} + +// manifests is every module the indexer holds, with its trunk manifest and fingerprint. +func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) { + mods, err := s.values(modulesState) + if err != nil { + return nil, nil, err + } + out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{} + for name, raw := range mods { + m, err := delivery.DecodeModule(raw) + if err != nil { + continue + } + held[name] = m + if m.Manifest == "" { + continue + } + mraw, _, found, err := s.st.get(manifestsState, m.Manifest) + if err != nil || !found { + continue + } + kept, err := delivery.DecodeManifest(mraw) + if err != nil { + continue + } + if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil { + out[name] = man + } + } + return out, held, nil +} + +// policyRow is one module's policy as upgrade answers it. +type policyRow struct { + Module string `json:"module"` + Policy string `json:"policy"` + From string `json:"from"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` +} + +// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it +// back with default; with compare list the splits. +func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) { + manifests, _, err := s.manifests() + if err != nil { + return nil, err + } + raw, err := s.values(policiesState) + if err != nil { + return nil, err + } + chosen, unreadable := assign.DecodePolicies(raw) + if arg(args, "compare") == "true" { + composed, err := s.values(composedState) + if err != nil { + return nil, err + } + splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed)) + if splits == nil { + splits = []assign.Split{} + } + out := map[string]any{"splits": splits, "empty": len(splits) == 0} + // The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15). + if found, err := s.values(health.FindingsState); err != nil { + out["findings-unread"] = err.Error() + } else { + keys := []string{} + for k := range found { + keys = append(keys, k) + } + sort.Strings(keys) + out["findings"] = keys + } + return out, nil + } + module, policy := arg(args, "module"), arg(args, "policy") + if policy != "" { + return s.setPolicy(module, policy, arg(args, "why"), by, manifests) + } + var rows []policyRow + for name, m := range manifests { + if module != "" && name != module { + continue + } + p, from, why, who := assign.PolicyOf(m, chosen) + rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who}) + } + if module != "" && len(rows) == 0 { + return nil, fmt.Errorf("the indexer holds no manifest of %s", module) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module }) + return map[string]any{"modules": rows, "unreadable": unreadable}, nil +} + +// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log +// and says policy-changed. The bus is never rolled: a choice for it is refused. +func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) { + if module == "" { + return nil, errors.New("a policy is set for one module: name it") + } + m, ok := manifests[module] + if !ok { + return nil, fmt.Errorf("the indexer holds no manifest of %s", module) + } + if catalogue.ProvidesBus(m) { + return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module) + } + if why == "" { + return nil, errors.New("a person's policy says why") + } + if by == "" || by == stdio.Unproven { + return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller") + } + what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy) + if policy == "default" { + if err := s.st.del(policiesState, module); err != nil { + return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err) + } + } else { + p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()} + if err := p.Check(); err != nil { + return nil, err + } + raw, _ := json.Marshal(p) + if err := s.st.put(policiesState, module, raw); err != nil { + return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err) + } + } + rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased} + taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"}) + recorded := taken.Log + if rerr != nil { + recorded = "not recorded: " + rerr.Error() + } + body := map[string]any{"module": module, "policy": policy, "why": why, "by": by} + said := "" + if s.emit != nil { + id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10) + if err := s.emit(PolicyChangedEvent, id, body); err != nil { + said = "not said: " + err.Error() + } + } + return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil +} + +// busNode is one node's bus as bus answers it. +type busNode struct { + Node string `json:"node"` + FP string `json:"fp"` + Behind bool `json:"behind"` +} + +// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind, +// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only. +func (s *wantsTools) doBus(map[string]any) (any, error) { + manifests, held, err := s.manifests() + if err != nil { + return nil, err + } + bus := "" + for name, m := range manifests { + if catalogue.ProvidesBus(m) { + bus = name + } + } + if bus == "" { + return nil, errors.New("the indexer holds no module that provides the bus") + } + newest := held[bus].FP2 + haves, err := s.values(haveState) + if err != nil { + return nil, err + } + var nodes []busNode + for key, raw := range haves { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode || module != bus { + continue + } + h, err := delivery.DecodeHave(raw) + if err != nil || h.Phase == delivery.PhaseRemoved { + continue + } + nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest}) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node }) + out := map[string]any{"module": bus, "newest": newest, "nodes": nodes} + if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil { + out["steps-unread"] = err.Error() + } else { + out["steps"] = steps + } + puts, err := s.values(putsState) + if err != nil { + out["puts-unread"] = err.Error() + } else { + mine := map[string]json.RawMessage{} + for k, v := range puts { + if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus { + mine[k] = v + } + } + out["puts"] = mine + } + return out, nil +} + +// tools are the seat's verbs as the runtime serves them, each named ., with the seat's own schemas. +func (s *wantsTools) tools() []stdio.Tool { + seat, _ := catalogue.SeatNamed(catalogue.WantsSeat) + run := map[string]func(map[string]any) (any, error){ + "upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) }, + "bus": s.doBus, + } + var out []stdio.Tool + for _, v := range seat.Serves { + out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]}) + } + return out +} diff --git a/modules/mesh-delivery-assigner/internal/assign/findings.go b/modules/mesh-delivery-assigner/internal/assign/findings.go index cd0baaf..d6e2ea9 100644 --- a/modules/mesh-delivery-assigner/internal/assign/findings.go +++ b/modules/mesh-delivery-assigner/internal/assign/findings.go @@ -64,6 +64,7 @@ func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised type Findings struct { Put func(key string, value []byte) error Delete func(key string) error + Keys func() ([]string, error) ToKeeper func() bool Now func() time.Time } diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go new file mode 100644 index 0000000..287f8df --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -0,0 +1,135 @@ +package assign + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one +// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's +// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled. +// Until the lease write the controller still writes a policy beside each composition's digest; where it differs from the +// assigner's, the assigner says so as its finding `policy-split`, naming mesh-wants.upgrade as the remedy. + +// PoliciesBucket is the assigner's state `policies`. +var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState + +// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named. +func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) { + out := map[string]catalogue.PersonPolicy{} + var bad []string + for key, raw := range values { + var p catalogue.PersonPolicy + if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key { + bad = append(bad, key) + continue + } + out[key] = p + } + sort.Strings(bad) + return out, bad +} + +// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one. +func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) { + var c *catalogue.PersonPolicy + if p, ok := chosen[m.Module]; ok { + c = &p + } + return catalogue.Upgrade(m, c) +} + +// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it. +func ControllerPolicies(composed map[string][]byte) map[string][]string { + out := map[string][]string{} + for key, raw := range composed { + _, module, _, isNode, err := delivery.ParseComposedKey(key) + if err != nil || isNode { + continue + } + d, err := delivery.DecodeComposedDigest(raw) + if err != nil || d.Policy == "" { + continue + } + if !contains(out[module], d.Policy) { + out[module] = append(out[module], d.Policy) + sort.Strings(out[module]) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// Split is a module whose policy at the assigner differs from one the controller writes beside a composition of it. +type Split struct { + Module string `json:"module"` + Policy string `json:"policy"` + From string `json:"from"` + Controller []string `json:"controller"` +} + +// Splits is every module of manifests the controller writes a policy for that differs from the assigner's, sorted. +func Splits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy, controller map[string][]string) []Split { + var out []Split + for module, theirs := range controller { + m, ok := manifests[module] + if !ok { + continue + } + policy, from, _, _ := PolicyOf(m, chosen) + for _, p := range theirs { + if p != policy { + out = append(out, Split{Module: module, Policy: policy, From: from, Controller: theirs}) + break + } + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// KindPolicySplit is the assigner's finding of a split policy (ADR 0565 §8). +const KindPolicySplit = "policy-split" + +// PolicySplitKey is a module's policy-split finding's key. +func PolicySplitKey(module string) string { + return health.FindingKey(health.ScopeModule, module, KindPolicySplit) +} + +// PolicySplitFinding is the finding of one split, in mesh-sdk's layout. +func PolicySplitFinding(s Split, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s's upgrade policy is %s at the assigner (%s) and %s at the controller", s.Module, s.Policy, s.From, + strings.Join(s.Controller, " and ")) + return health.EncodeFinding(health.Finding{Key: PolicySplitKey(s.Module), Kind: KindPolicySplit, + Subject: health.Subject{Scope: health.ScopeModule, ID: s.Module}, Severity: health.Warning, Summary: summary, + Headline: clipHeadline(s.Module + "'s upgrade policy is split"), + Needs: "set it with mesh-wants.upgrade, so the assigner and the controller agree", + Explanation: asks.NeedsYou + "set it with mesh-wants.upgrade, so the assigner and the controller agree." + " The controller still " + + "writes a policy beside each composition until it hands its lease away; a choice made at its upgrade verb after the copy " + + "is not the assigner's.", + Resolved: s.Module + "'s upgrade policy agrees again", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindPolicySplit, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +func clipHeadline(s string) string { + if len([]rune(s)) <= 60 { + return s + } + return string([]rune(s)[:60]) +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 6d54820..d839b93 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -29,7 +29,9 @@ type Module struct { // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone. GrantedOn func(node string) bool Manifest catalogue.Manifest - Nodes []string + // Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none. + Chosen *catalogue.PersonPolicy + Nodes []string // Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its // digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6). Composed func(node, fp string) (delivery.ComposedDigest, bool) @@ -639,9 +641,14 @@ func (m Module) busBar(before delivery.Want, had bool, running string) (prev str return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP } -// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the -// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. +// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR +// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the +// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. func (m Module) recorded(d delivery.ComposedDigest) bool { + if m.Chosen != nil { + policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen) + return policy == catalogue.PolicyRecord + } if d.Policy != "" { return d.Policy == delivery.PolicyRecord } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 8d75165..3a6bb0c 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -802,6 +802,12 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) } + // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. + with(delivery.PolicyRecord, "person") + m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} + if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew { + t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) + } } // Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 3964db8..6f0c3ca 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -44,7 +44,7 @@ func TestManifest(t *testing.T) { states[s.Name] = s.History } if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20, - "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5}; !reflect.DeepEqual(states, want) { + "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -63,9 +63,14 @@ func TestManifest(t *testing.T) { // ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream // placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its // hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches. - if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) { - t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs", m.Claims) + // ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus. + if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh || + !slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) || + m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) { + t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs and mesh-wants with its two", m.Claims) + } + if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil { + t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat)) } if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil { t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat)) @@ -79,7 +84,7 @@ func TestManifest(t *testing.T) { } if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin", "seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin", - "seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.warranted", // ADR 0565 §11: the seat, the controller's only while it has no holder + "seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat, the controller's only while it has no holder "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 267eb62..74542b6 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -15,6 +15,14 @@ "adopt", "hand-back" ] + }, + { + "name": "mesh-wants", + "scope": "mesh", + "serves": [ + "upgrade", + "bus" + ] } ], "reads": [ @@ -48,6 +56,7 @@ "seat:mesh-controller.unpin", "seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now" @@ -70,7 +79,8 @@ { "name": "differences", "history": 64 }, { "name": "puts", "history": 20 }, { "name": "wants", "history": 20 }, - { "name": "findings", "history": 5 } + { "name": "findings", "history": 5 }, + { "name": "policies", "history": 20 } ], "streams": [ { From 78f03b7422db6209bf2ec70235f06dc65aec3a6e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:37:39 +0100 Subject: [PATCH 16/35] Watchman: say needs-operator from mesh-provisioner's provisions in waits, so the row the haves could not carry has its facts on the bus (issue 1954 D2) Pins mesh-sdk 4a96793. --- .../cmd/mesh-delivery-watchman/main.go | 29 ++++++++++ modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 6 +-- .../internal/watch/findings_test.go | 19 +++++++ .../internal/watch/signals.go | 54 +++++++++++++++++-- .../internal/watch/watch.go | 4 +- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 1 + 8 files changed, 107 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 4d50197..62b84a8 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-watchman/internal/socket" "mesh-delivery-watchman/internal/watch" @@ -297,6 +298,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := w.Rebuild(ask); err != nil { return err } + // The seat mesh-provisioner's provisions (ADR 0570 §7): a provision in waits is needs-operator. Read before the findings are + // reconciled when the bucket is there; bound once it appears when it is not (mesh-provisioner comes in its own delivery). + provisionsWatch := func(kv jetstream.KeyValue) (jetstream.KeyWatcher, error) { + return settle(ctx, kv, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + } + var provWatch jetstream.KeyWatcher + if kv, err := js.KeyValue(ctx, provisions.ProvisionsBucket); err == nil { + if provWatch, err = provisionsWatch(kv); err != nil { + process.NotWatched(logger.Printf, provisions.ProvisionsBucket, err, "no provision waiting for the operator is said until the watchman starts again") + } + } else { + process.AwaitBucket(ctx, js, conn, provisions.ProvisionsBucket, "no provision waiting for the operator is said", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + pw, err := provisionsWatch(kv) + if err != nil { + process.NotWatched(logger.Printf, provisions.ProvisionsBucket, err, "no provision waiting for the operator is said until the watchman starts again") + return + } + go func() { + _ = follow(ctx, pw, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + }() + }) + } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} w.SetRecount(dead.count) dead.count() @@ -309,6 +333,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } errs := make(chan error, 8) + if provWatch != nil { + go func() { + errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + }() + } if kvs.lease != nil { lw, err := kvs.lease.Watch(ctx, buscompose.ControllerLeaseKey) if err != nil { diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d5fd93f..8842356 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index d215b52..b279ec6 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7 h1:/XEAhD7/8QoMjqI7XiX9HM4HkpKubWheQosqgfe2WrU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205750-b6ef01d6c6a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index f31b704..182c5ca 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -6,6 +6,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) func haveOf(t *testing.T, phase delivery.Phase, health delivery.Health, detail string) []byte { @@ -154,6 +155,24 @@ func TestTheWatchmansFindings(t *testing.T) { expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter) w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) // needs-operator is not observed here }, map[string]string{"module.web.ace.unhealthy": Urgent}}, + {"1954/D2: a provision in waits is its consumer's needs-operator; provided, it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + enc := func(state, why string) []byte { + b, err := provisions.Encode(provisions.Provision{Provision: "oidc", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "keycloak", Node: "novox"}, State: state, Why: why, At: t0}) + if err != nil { + t.Fatal(err) + } + return b + } + w.Provision("ace.photos.oidc", enc(provisions.Waits, "the operator gives the client's redirect"), Put) + w.Provision("ace.photos.db", enc(provisions.Provided, ""), Put) + if f := o.findings["module.photos.ace.needs-operator"]; !strings.Contains(f.Summary, "redirect") || f.Machine != "ace" { + t.Fatalf("findings = %v", o.findings) + } + w.Provision("ace.photos.oidc", enc(provisions.Provided, ""), Put) + w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) + w.Provision("ace.web.oidc", nil, Delete) + }, map[string]string{}}, {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 4c3a067..45c2881 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -9,6 +9,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // The rows of the signals table the watchman holds (novox/hq ADR 0564 §4, Limits 3 and 11). Each is a key of the pulse @@ -458,9 +459,8 @@ func ToolsNode(subject string) string { // unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have // removed or deleted ends its part of them. // -// needs-operator is not observed here: the controller excuses a wait only after checking it against the module's manifest -// and against whether a person gave the secret on that machine (its operator_wait.go), and neither each resource's wait nor -// what a person gave is on the bus (the have carries the node's words only). The row waits for those facts' home. +// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state +// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7). func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) { unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") if module == ControllerModule { @@ -511,6 +511,53 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav // ControllerModule is the module whose have names a control node. const ControllerModule = "mesh-controller" +// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its +// consumer waiting for the operator, said as `module...needs-operator` naming each waiting provision and +// why; the finding ends when no provision of that consumer on that node waits. A key deleted, or one that cannot be read, +// waits no more. +func (w *Watchman) Provision(key string, value []byte, op Op) { + node, consumer, _, _, err := provisions.ParseKey(key) + if err != nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + id := consumer + "." + node + if w.waiting[id] == nil { + w.waiting[id] = map[string]string{} + } + delete(w.waiting[id], key) + if op == Put { + if p, err := provisions.Decode(value); err != nil { + w.say("the provision %s cannot be read, and is taken as not waiting: %v", key, err) + } else if p.State == provisions.Waits { + w.waiting[id][key] = fmt.Sprintf("%s from %s on %s: %s", p.Provision, p.Provider.Module, p.Provider.Node, p.Why) + } + } + fkey := FindingKey(ScopeModule, id, KindNeedsOperator) + if len(w.waiting[id]) == 0 { + delete(w.waiting, id) + w.unfind(fkey) + return + } + var said []string + for _, k := range sortedKeys(w.waiting[id]) { + said = append(said, w.waiting[id][k]) + } + w.find(Finding{Scope: ScopeModule, ID: id, Kind: KindNeedsOperator, Machine: node, Severity: Warning, Source: "provisions", + Summary: fmt.Sprintf("%s on %s waits for the operator: %s", consumer, node, strings.Join(said, "; ")), + Said: strings.Join(said, "; ")}) +} + +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + // ProvReport is a machine's provisioner client's word on one module's files (`prov.applied..`, ADR 0362 §3). type ProvReport struct { Node string `json:"node"` @@ -786,6 +833,7 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, + {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, } // Signal is one signal's pulse as `signals` shows it. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 0861f54..84e1ef0 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -84,6 +84,8 @@ type Watchman struct { provFailed map[string]map[string]ProvReport closures map[string]map[string]time.Time recount func() + // waiting is, by `.`, each of its provisions in waits, with its words. + waiting map[string]map[string]string // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -94,7 +96,7 @@ func New(out Out, now func() time.Time) *Watchman { pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, - control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}} + control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 3b8a9a0..21a1ad5 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -91,7 +91,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 reads := slices.Clone(m.Reads) slices.Sort(reads) if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease", - "mesh-delivery-assigner.wants", "node-engine.have"}) { + "mesh-delivery-assigner.wants", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 45e46eb..291908c 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -21,6 +21,7 @@ "mesh-build-node.steps", "mesh-controller.lease", "mesh-delivery-assigner.wants", + "mesh-provisioner.provisions", "node-engine.have" ], "tools": [ From f56eb62d8b273c05622814b2f7ec05d7c4c4f95a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:39:38 +0100 Subject: [PATCH 17/35] =?UTF-8?q?Assigner:=20hold=20a=20want=20until=20eve?= =?UTF-8?q?ry=20provision=20its=20module=20consumes=20on=20that=20node=20i?= =?UTF-8?q?s=20provided,=20and=20say=20a=20failed=20one=20as=20a=20finding?= =?UTF-8?q?,=20so=20a=20module=20never=20moves=20ahead=20of=20what=20it=20?= =?UTF-8?q?needs=20(issue=201954=20D1,=20ADR=200570=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins mesh-sdk 6e39ed1. --- .../cmd/mesh-delivery-assigner/main.go | 13 +++- .../cmd/mesh-delivery-assigner/pass.go | 77 ++++++++++++++++++- .../mesh-delivery-assigner/seat_tools_test.go | 25 ++++++ modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../internal/assign/policy.go | 28 +++++++ .../internal/assign/wants.go | 27 +++++++ .../internal/assign/wants_test.go | 21 +++++ .../mesh-delivery-assigner/manifest_test.go | 2 +- modules/mesh-delivery-assigner/module.json | 1 + 10 files changed, 193 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 934df8f..b7d75eb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -30,6 +30,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-assigner/internal/assign" ) @@ -71,6 +72,7 @@ var readBuckets = []string{ delivery.HaveBucket, delivery.StepsBucket(goBuildModule), takeoverBucket, + provisions.ProvisionsBucket, } // takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said @@ -157,6 +159,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == provisions.ProvisionsBucket && err != nil { + // The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made; + // until then no want waits for a provision, as before. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } if name == assign.PoliciesBucket && err != nil { // The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no // person's policy is read, so the controller's beside each composition holds, as before. @@ -313,6 +321,7 @@ var watched = []string{ delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket, assign.PoliciesBucket, + provisions.ProvisionsBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -409,7 +418,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket) { continue // not on the bus at this start (open said so) } started++ @@ -439,7 +448,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket} { + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket} { if kvs[name] == nil && js != nil { name := name process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index facdff2..9a6c0fd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "git.novox.be/novox/mesh-sdk/go/sdkversion" "mesh-delivery-assigner/internal/assign" @@ -31,6 +32,8 @@ type passer struct { // modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left. findings assign.Findings splits map[string]bool + // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). + failedProvisions map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -134,6 +137,7 @@ func (p *passer) wants(ctx context.Context) { } wanted := p.w.wantsOf() chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + provisionsOf := p.provisionsOf() defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -160,7 +164,7 @@ func (p *passer) wants(ctx context.Context) { for name := range names { fp := trunk[name].FP2 name := name - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { if placed[node] { @@ -994,3 +998,74 @@ func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen ma } p.splits = split } + +// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat +// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed +// provision is also said as the assigner's finding `provider..provision-failed` (a waiting one is the watchman's +// needs-operator), deleted once it is no longer failed. +func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision { + if p.kvs[provisions.ProvisionsBucket] == nil { + return func(string) func(string) map[string]provisions.Provision { return nil } + } + by := map[string]map[string]map[string]provisions.Provision{} // module, node, key + failed := map[string]provisions.Provision{} + for key, raw := range p.w.read(provisions.ProvisionsBucket) { + pr, err := provisions.Decode(raw) + if err != nil { + continue + } + node, module := pr.Consumer.Node, pr.Consumer.Module + if by[module] == nil { + by[module] = map[string]map[string]provisions.Provision{} + } + if by[module][node] == nil { + by[module][node] = map[string]provisions.Provision{} + } + by[module][node][key] = pr + if pr.State == provisions.Failed { + failed[key] = pr + } + } + p.provisionFindings(failed) + return func(module string) func(string) map[string]provisions.Provision { + return func(node string) map[string]provisions.Provision { return by[module][node] } + } +} + +// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed. +func (p *passer) provisionFindings(failed map[string]provisions.Provision) { + if p.findings.Put == nil { + return + } + now := time.Now + if p.now != nil { + now = p.now + } + for key, pr := range failed { + if p.failedProvisions[key] { + continue + } + value, err := assign.ProvisionFailedFinding(key, pr, now()) + if err == nil { + err = p.findings.Put(assign.ProvisionFailedKey(key), value) + } + if err != nil { + p.logf("the failed provision %s is not said: %v", key, err) + continue + } + if p.failedProvisions == nil { + p.failedProvisions = map[string]bool{} + } + p.failedProvisions[key] = true + } + for key := range p.failedProvisions { + if _, still := failed[key]; still { + continue + } + if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil { + p.logf("the failed provision %s is not cleared: %v", key, err) + continue + } + delete(p.failedProvisions, key) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index cba607c..9322fc4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-assigner/internal/assign" ) @@ -158,6 +159,30 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("the seat's verbs as served: %+v", tools) } t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) + t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { + findings := map[string][]byte{} + p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now()) + p.kvs[provisions.ProvisionsBucket] = &memKV{} + of := p.provisionsOf() + if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed { + t.Fatalf("provisions = %+v", got) + } + if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 { + t.Fatalf("findings = %v", findings) + } + provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now()) + p.provisionsOf() + if len(findings) != 0 { + t.Errorf("findings after provided = %v", findings) + } + }) } // wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index e1d9aad..788e658 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 685613a..37f9598 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 h1:tQZYFKHd378XTF6OPGncIJnapxQCtRJAtL2FvlIzNQ4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index 287f8df..6b938a9 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one @@ -133,3 +134,30 @@ func clipHeadline(s string) string { } return string([]rune(s)[:60]) } + +// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4). +const KindProvisionFailed = "provision-failed" + +// ProvisionFailedKey is a failed provision's finding's key, `provider...[.].provision-failed`. +func ProvisionFailedKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed) +} + +// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why. +func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision, + p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module) + return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.", + Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}}, + Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +func also(provider, consumer string) []string { + if provider == "" || provider == consumer { + return nil + } + return []string{provider} +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index d839b93..b2ab36d 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the @@ -29,6 +30,9 @@ type Module struct { // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone. GrantedOn func(node string) bool Manifest catalogue.Manifest + // Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want + // waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it. + Provisions func(node string) map[string]provisions.Provision // Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none. Chosen *catalogue.PersonPolicy Nodes []string @@ -126,9 +130,32 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { if _, ok := m.composition(node, m.FP); !ok { return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) } + if key, p, waiting := m.provisionWaiting(node); waiting { + words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) + if p.Why != "" { + words += ": " + p.Why + } + return false, delivery.RuleProvisions, words + } return true, 0, "" } +// provisionWaiting is the first provision of the module on node, by key, that is not provided (ADR 0570 §4). A provision +// the holder has not composed a key for yet holds nothing: the holder composes each key ahead of the want, from placement +// and the newest build. +func (m Module) provisionWaiting(node string) (string, provisions.Provision, bool) { + if m.Provisions == nil { + return "", provisions.Provision{}, false + } + keyed := m.Provisions(node) + for _, key := range slices.Sorted(maps.Keys(keyed)) { + if p := keyed[key]; p.State != provisions.Provided { + return key, p, true + } + } + return "", provisions.Provision{}, false +} + // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned // elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node // offers it) and holds. diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 3a6bb0c..a6e0191 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } @@ -69,6 +70,12 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { delivery.RuleComposed: func(m *Module, _ *Nodes) { m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false } }, + // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. + delivery.RuleProvisions: func(m *Module, _ *Nodes) { + m.Provisions = func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} + } + }, } { m, n := ready() breakIt(&m, &n) @@ -82,6 +89,20 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { t.Errorf("rule %d broken: g14's want is %+v", rule, w) } + if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.db") || !strings.Contains(out.Rules["g14"].Words, "asked")) { + t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) + } + } + // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. + for _, all := range []func(string) map[string]provisions.Provision{ + func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Provided}} + }, nil} { + m, n := ready() + m.Provisions = all + if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew { + t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) + } } // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 6f0c3ca..0003bae 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -35,7 +35,7 @@ func TestManifest(t *testing.T) { slices.Sort(reads) want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"} + "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 74542b6..ecbdc4b 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -41,6 +41,7 @@ "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", + "mesh-provisioner.provisions", "node-engine.have" ], "uses": [ From 7b7796b6faf670a409fe12407db5d3d1c14ca2d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:44 +0100 Subject: [PATCH 18/35] Assigner: hold a want while a provision its module requires has no key yet, and say it once it stays without one past the delivery bound, so a module never starts without its database unseen (issue 1954 D1) Pins mesh-sdk 66ff33d. --- .../cmd/mesh-delivery-assigner/pass.go | 71 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 5 ++ .../mesh-delivery-assigner/seat_tools_test.go | 22 ++++++ modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../internal/assign/policy.go | 25 +++++++ .../internal/assign/wants.go | 46 ++++++++---- .../internal/assign/wants_test.go | 15 +++- 8 files changed, 172 insertions(+), 18 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 9a6c0fd..9c841f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -34,6 +34,8 @@ type passer struct { splits map[string]bool // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). failedProvisions map[string]bool + // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. + missing, missingSaid map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -138,6 +140,8 @@ func (p *passer) wants(ctx context.Context) { wanted := p.w.wantsOf() chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) provisionsOf := p.provisionsOf() + missing := map[string]bool{} + defer func() { p.provisionsMissing(ctx, missing) }() defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -207,6 +211,11 @@ func (p *passer) wants(ctx context.Context) { last[node] = p.w.lastHealthy(node, name, fp) } out := assign.Wants(m, nodes, wanted[name], last) + for _, node := range m.Nodes { + for _, prov := range m.MissingProvisions(node) { + missing[node+"."+name+"."+prov] = true + } + } p.yeses(ctx, name, m, wanted[name], &out) gatherNeither(neither, name, out) leftNeither(left, m, wanted[name], out) @@ -1069,3 +1078,65 @@ func (p *passer) provisionFindings(failed map[string]provisions.Provision) { delete(p.failedProvisions, key) } } + +// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default +// when its manifest names none (ADR 0564 clause 13 (d)). +var MissingBound = delivery.DefaultDeliveryBound + +// missingPulse is the pulse key of a provision with no key, `provision-missing...`. +func missingPulse(key string) string { return "provision-missing." + key } + +// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is +// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding +// `provider....provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on +// the watch, is the pass that writes it: no timer. +func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) { + pulses := p.kvs[delivery.PlacementPulsesBucket] + if p.missing == nil { + p.missing = map[string]bool{} + } + for key := range missing { + switch { + case !p.missing[key]: + p.missing[key] = true + if pulses != nil { + p.pulse(ctx, missingPulse(key), MissingBound) + } + case pulses == nil || !p.pulsing(missingPulse(key)): + if p.missingSaid[key] || p.findings.Put == nil { + continue + } + value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock()) + if err == nil { + err = p.findings.Put(assign.ProvisionMissingKey(key), value) + } + if err != nil { + p.logf("the missing provision %s is not said: %v", key, err) + continue + } + if p.missingSaid == nil { + p.missingSaid = map[string]bool{} + } + p.missingSaid[key] = true + } + } + for key := range p.missing { + if missing[key] { + continue + } + delete(p.missing, key) + if pulses != nil && p.pulsing(missingPulse(key)) { + if err := pulses.Delete(ctx, missingPulse(key)); err == nil { + p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil) + } + } + if p.missingSaid[key] && p.findings.Delete != nil { + if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil { + p.logf("the missing provision %s is not cleared: %v", key, err) + p.missing[key] = true + continue + } + delete(p.missingSaid, key) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 6b7f420..affbf2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -32,6 +32,11 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) return uint64(len(m.values)), nil } +func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + m.values[key] = value + return uint64(len(m.values)), nil +} + func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { delete(m.values, key) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 9322fc4..af24d5f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "slices" @@ -182,6 +183,27 @@ func TestTheSeatsVerbs(t *testing.T) { if len(findings) != 0 { t.Errorf("findings after provided = %v", findings) } + // A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed. + ctx := context.Background() + p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}} + gone := map[string]bool{"ace.photos.s3": true} + p.provisionsMissing(ctx, gone) + if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 { + t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings) + } + p.provisionsMissing(ctx, gone) + if len(findings) != 0 { + t.Fatal("said before its bound") + } + p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it + p.provisionsMissing(ctx, gone) + if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok { + t.Fatalf("past its bound: findings %v", findings) + } + p.provisionsMissing(ctx, map[string]bool{}) + if len(findings) != 0 { + t.Errorf("keyed again: findings %v", findings) + } }) } diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 788e658..a9b4fbe 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 37f9598..326dffd 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 h1:tQZYFKHd378XTF6OPGncIJnapxQCtRJAtL2FvlIzNQ4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index 6b938a9..c2c9aa8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -161,3 +161,28 @@ func also(provider, consumer string) []string { } return []string{provider} } + +// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound. +const KindProvisionMissing = "provision-missing" + +// ProvisionMissingKey is its key, `provider....provision-missing`. +func ProvisionMissingKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing) +} + +// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `..`). +func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) { + parts := strings.SplitN(key, ".", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("%q is not ..", key) + } + node, module, provision := parts[0], parts[1], parts[2] + summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+ + "no provider of it may be placed", module, node, provision, bound, module) + return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.", + Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()}) +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index b2ab36d..237d38a 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -130,30 +130,52 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { if _, ok := m.composition(node, m.FP); !ok { return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) } - if key, p, waiting := m.provisionWaiting(node); waiting { - words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) - if p.Why != "" { - words += ": " + p.Why - } + if words, waiting := m.provisionWaiting(node); waiting { return false, delivery.RuleProvisions, words } return true, 0, "" } -// provisionWaiting is the first provision of the module on node, by key, that is not provided (ADR 0570 §4). A provision -// the holder has not composed a key for yet holds nothing: the holder composes each key ahead of the want, from placement -// and the newest build. -func (m Module) provisionWaiting(node string) (string, provisions.Provision, bool) { +// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has +// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none. +func (m Module) provisionWaiting(node string) (string, bool) { if m.Provisions == nil { - return "", provisions.Provision{}, false + return "", false + } + if missing := m.MissingProvisions(node); len(missing) > 0 { + return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true } keyed := m.Provisions(node) for _, key := range slices.Sorted(maps.Keys(keyed)) { if p := keyed[key]; p.State != provisions.Provided { - return key, p, true + words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) + if p.Why != "" { + words += ": " + p.Why + } + return words, true } } - return "", provisions.Provision{}, false + return "", false +} + +// MissingProvisions are the provisions the module requires (not optionally) that have no key on node yet, sorted; none +// while the seat's state is not read. +func (m Module) MissingProvisions(node string) []string { + if m.Provisions == nil { + return nil + } + have := map[string]bool{} + for _, p := range m.Provisions(node) { + have[p.Provision] = true + } + var out []string + for _, r := range m.Manifest.Requires { + if !have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(out, r) { + out = append(out, r) + } + } + slices.Sort(out) + return out } // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index a6e0191..06886cb 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -73,7 +73,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. delivery.RuleProvisions: func(m *Module, _ *Nodes) { m.Provisions = func(node string) map[string]provisions.Provision { - return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} } }, } { @@ -89,14 +89,14 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { t.Errorf("rule %d broken: g14's want is %+v", rule, w) } - if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.db") || !strings.Contains(out.Rules["g14"].Words, "asked")) { + if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) { t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) } } // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. for _, all := range []func(string) map[string]provisions.Provision{ func(node string) map[string]provisions.Provision { - return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Provided}} + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}} }, nil} { m, n := ready() m.Provisions = all @@ -104,6 +104,15 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) } } + // 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named. + { + m, n := ready() + m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} } + if out := Wants(m, n, wantedOld(), nil); out.Rules["g14"].Rule != delivery.RuleProvisions || + !strings.Contains(out.Rules["g14"].Words, "provision database has no key yet") || len(m.MissingProvisions("g14")) != 1 { + t.Errorf("a provision with no key: %+v", out.Rules["g14"]) + } + } // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not // healthy held photos, and novox offline left a module with no canary. From 64a5074d58a0324e0b6c34b8c3b22aeef798fb0c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:43:27 +0100 Subject: [PATCH 19/35] =?UTF-8?q?Assigner:=20lay=20each=20held=20value=20o?= =?UTF-8?q?f=20the=20group's=20own=20pull=20requests=20at=20their=20heads?= =?UTF-8?q?=20over=20its=20module's=20values=20in=20the=20composition=20ch?= =?UTF-8?q?eck,=20judged=20against=20the=20head's=20declaration=20and=20na?= =?UTF-8?q?med,=20so=20a=20check=20reads=20the=20held=20values=20itself=20?= =?UTF-8?q?(issue=201953=20D7,=20ADR=200565=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-assigner/main.go | 11 +++- .../cmd/mesh-delivery-assigner/world.go | 8 +++ .../internal/assign/compose.go | 63 +++++++++++++++++++ .../internal/assign/compose_test.go | 28 +++++++++ .../mesh-delivery-assigner/manifest_test.go | 2 +- modules/mesh-delivery-assigner/module.json | 1 + 6 files changed, 110 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index b7d75eb..7c232b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -73,6 +73,7 @@ var readBuckets = []string{ delivery.StepsBucket(goBuildModule), takeoverBucket, provisions.ProvisionsBucket, + keptBucket, } // takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said @@ -159,6 +160,11 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == keptBucket && err != nil { + // mesh-config's held values (ADR 0565 §5): without them a composition lays none, said in the journal. + logf("the bucket %s cannot be opened (%v): no held value is laid on a composition until it is bound", name, err) + continue + } if name == provisions.ProvisionsBucket && err != nil { // The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made; // until then no want waits for a provision, as before. @@ -322,6 +328,7 @@ var watched = []string{ delivery.AssignerWantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, + keptBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -418,7 +425,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket || name == keptBucket) { continue // not on the bus at this start (open said so) } started++ @@ -448,7 +455,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket} { + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket} { if kvs[name] == nil && js != nil { name := name process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 258e526..8495a3d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -432,6 +432,11 @@ func (r worldReader) Served(ctx context.Context, node, module string, keys []str func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) { ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}} + heads := map[string]string{} + for _, p := range c.Pulls { + heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head + } + ch.Held = assign.HeldOf(w.read(keptBucket), heads) definedIn := map[string]map[string]bool{} for _, p := range c.Pulls { if p.Repository == assign.SDKRepository { @@ -675,3 +680,6 @@ func (w *world) runningCommits(ctx context.Context, get getter, module string) [ sort.Strings(out) return out } + +// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5). +var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState diff --git a/modules/mesh-delivery-assigner/internal/assign/compose.go b/modules/mesh-delivery-assigner/internal/assign/compose.go index 65c1481..35cb850 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose.go @@ -5,6 +5,7 @@ package assign import ( + "encoding/json" "fmt" "slices" "sort" @@ -59,6 +60,63 @@ type Change struct { // Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623). Digests map[string]string Members map[string]string + // Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR + // 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named. + Held []delivery.KeptValue +} + +// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another +// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key. +func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue { + var out []delivery.KeptValue + for _, raw := range kept { + var k delivery.KeptValue + if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" { + continue + } + if head, ok := heads[k.Pull]; ok && head == k.Head { + out = append(out, k) + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key + }) + return out +} + +// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers +// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the +// declaration refuses, or a key it does not declare, is a failure. +func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) { + for _, k := range held { + var v any + if err := json.Unmarshal(k.Value, &v); err != nil { + said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err)) + continue + } + d, declared := in.Catalogue[k.Module].Settings[k.Key] + if !declared { + fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull)) + continue + } + if problems := catalogue.ValueProblems(d, v); len(problems) > 0 { + fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module, + k.Node, k.Pull, strings.Join(problems, "; "))) + continue + } + at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node }) + if at < 0 { + said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node)) + continue + } + if in.Nodes[at].Settings == nil { + in.Nodes[at].Settings = map[string][]catalogue.Layer{} + } + in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}}) + said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved)) + } + return said, fails } // Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or @@ -171,6 +229,11 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit delete(changed, name) } after := input(changed, m.Assigned) + laid, heldFails := layHeld(&after, c.Held) + v.Warnings = append(v.Warnings, laid...) + for _, f := range heldFails { + failAt("", f) + } for _, node := range machines { _, before := compose.Resolve(trunk, node) _, now := compose.Resolve(after, node) diff --git a/modules/mesh-delivery-assigner/internal/assign/compose_test.go b/modules/mesh-delivery-assigner/internal/assign/compose_test.go index 96c46ab..86e217b 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose_test.go @@ -1,7 +1,9 @@ package assign import ( + "encoding/json" "slices" + "strconv" "strings" "testing" @@ -38,6 +40,32 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) { if _, err := delivery.EncodeCompositionVerdict(v); err != nil { t.Errorf("the verdict cannot be written: %v", err) } + // 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged + // against the head's declaration; one of another pull request or another head is not laid. + web, err := catalogue.ParseManifest([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`)) + if err != nil { + t.Fatal(err) + } + kept := map[string][]byte{} + for i, k := range []delivery.KeptValue{ + {Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"}, + {Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)}, + {Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)}, + } { + kept[strconv.Itoa(i)], _ = json.Marshal(k) + } + held := HeldOf(kept, map[string]string{"novox/x#1": "h1"}) + if len(held) != 1 || string(held[0].Value) != "81" { + t.Fatalf("held = %+v, want #1's at its head alone", held) + } + c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held} + if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") { + t.Errorf("composition = %+v", v) + } + c.Held[0].Value = json.RawMessage(`"eighty"`) + if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") { + t.Errorf("a held value the declaration refuses: %+v", v) + } } // 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module. diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 0003bae..84bee99 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -33,7 +33,7 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", + want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index ecbdc4b..ea1b4b7 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -28,6 +28,7 @@ "reads": [ "mesh-build-go.steps", "mesh-conditions.takeover", + "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", From 8a91871de66c4bd2baed24db85ba53036173c47c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:01:03 +0100 Subject: [PATCH 20/35] Watchman: raise machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed and directory-used-as-found from each node's own health report, under the controller's keys, so these alerts outlive the controller's lease (issue 774) Pins mesh-sdk 94d0de5. --- .../cmd/mesh-delivery-watchman/main.go | 24 +- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 +- .../internal/watch/findings_test.go | 53 +++ .../internal/watch/mirror.go | 13 +- .../internal/watch/nodehealth.go | 358 ++++++++++++++++++ .../internal/watch/signals.go | 1 + .../internal/watch/watch.go | 6 +- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 1 + 10 files changed, 457 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-watchman/internal/watch/nodehealth.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 62b84a8..660ea74 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -321,6 +321,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }() }) } + // Each node's own health report (the node-engine's state health): read before the findings are reconciled when the + // bucket is there, bound once it appears when it is not; each change is heard on the watch, never polled. + healthEach := func(e jetstream.KeyValueEntry) { w.NodeHealth(e.Key(), e.Value(), opOf(e)) } + var healthWatch jetstream.KeyWatcher + if kv, err := js.KeyValue(ctx, delivery.NodeHealthBucket); err == nil { + if healthWatch, err = settle(ctx, kv, healthEach); err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + } + } else { + process.AwaitBucket(ctx, js, conn, delivery.NodeHealthBucket, "no node's own health is said", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + hw, err := settle(ctx, kv, healthEach) + if err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + return + } + go func() { _ = follow(ctx, hw, healthEach) }() + }) + } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} w.SetRecount(dead.count) dead.count() @@ -332,7 +351,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } - errs := make(chan error, 8) + errs := make(chan error, 16) + if healthWatch != nil { + go func() { errs <- follow(ctx, healthWatch, healthEach) }() + } if provWatch != nil { go func() { errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 8842356..db64d6f 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index b279ec6..d016872 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 182c5ca..4fbf380 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -173,6 +173,59 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) w.Provision("ace.web.oidc", nil, Delete) }, map[string]string{}}, + {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + report := func(h delivery.NodeHealth) []byte { + b, err := delivery.EncodeNodeHealth(h) + if err != nil { + t.Fatal(err) + } + return b + } + w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{ + {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, + {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, + {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, + Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, + Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, + Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put) + for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, + "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} { + if o.findings[key].Severity != sev { + t.Errorf("%s = %+v", key, o.findings[key]) + } + } + if f := o.findings["machine.novox.network"]; strings.Contains(f.Summary, "names") { + t.Errorf("the names failing through the rewritten file are said twice: %s", f.Summary) + } + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox"}), Put) // all well again + }, map[string]string{}}, + {"774: a node two nodes' tunnels no longer see is unreachable, said once there; not while it is silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for node, key := range map[string]string{"ace": "kA", "g14": "kG", "lap": "kL"} { + b, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, TunnelKey: key}}) + w.Have(node+".node", b, Put) + } + peers := func(from string, seen bool) []byte { + b, _ := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: from, Peers: []delivery.PeerSeen{{Key: "kL", Seen: seen}}}) + return b + } + w.NodeHealth("ace", peers("ace", false), Put) + if len(o.findings) != 0 { + t.Fatalf("one node alone said it: %v", o.findings) + } + w.NodeHealth("g14", peers("g14", false), Put) + if f := o.findings["machine.lap.unreachable"]; len(f.Also) != 2 { + t.Fatalf("findings = %v", o.findings) + } + w.Connect("node.lap") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + if _, ok := o.findings["machine.lap.unreachable"]; ok { + t.Error("a silent node is also said unreachable") + } + w.Connect("node.lap") + w.NodeHealth("g14", peers("g14", true), Put) + }, map[string]string{}}, {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index 4eee236..fe9f262 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -106,11 +106,22 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { // fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. func (w *Watchman) Have(key string, value []byte, op Op) { node, module, isNode, err := delivery.ParseHaveKey(key) - if err != nil || isNode { + if err != nil { return } w.mu.Lock() defer w.mu.Unlock() + if isNode { + // The node's own record: its tunnel key, whose peers' sight of it is network-unreachable. + var record *delivery.NodeRecord + if op == Put { + if hn, err := delivery.DecodeHaveNode(value); err == nil { + record = hn.Record + } + } + w.tunnelKey(node, record) + return + } switch op { case Expired: return diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go new file mode 100644 index 0000000..7cb6f91 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -0,0 +1,358 @@ +package watch + +import ( + "fmt" + "maps" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A node's own health report (novox/hq ADR 0241, issue 315, the operator's alert table of 2026-10-29): each node's engine +// writes `mesh-host_health.` when what it says changes, and the watchman raises from it what the controller raised +// from the passing health message, under the same keys and words (mesh-control cmd/mesh-controller machine_network.go, +// machine_units.go, module_health.go at e04846e9): +// - machine-network `machine..network`: the machine's own unhealthy parts; urgent on the control node or for the bus; +// - network-rewritten `machine.[.].rewritten`: the resolver file rewritten by another program; +// - network-unreachable `machine..unreachable`: two or more nodes' tunnels no longer see it (their peers' handshake), +// said once there, never while its own network or its silence says it; +// - machine-units `machine..units`: failed units no module places; +// - relogin-needed `module...relogin-needed` and directory-used-as-found +// `module...directory-used-as-found`. +// +// What the controller held at another machine by the addresses its parts point at (Toward) needs each node's private +// address and the hub, which no report carries: a part pointing elsewhere is the machine's own here, and the machine all +// point at is said by its peers instead. + +// The kinds a node's health report raises. +const ( + KindMachineNetwork = "machine-network" + KindNetworkRewritten = "network-rewritten" + KindNetworkUnreachable = "network-unreachable" + KindMachineUnits = "machine-units" + KindReloginNeeded = "relogin-needed" + KindUsedAsFound = "directory-used-as-found" +) + +// The parts and states a report names (mesh-control internal/link). +const ( + partResolvConf = "resolv-conf" + partNames = "names" + partTunnel = "tunnel" + partBus = "bus" + stateUnhealthy = "unhealthy" +) + +func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } + +// NodeHealth is a change of a node's report: written, or deleted (value nil). +func (w *Watchman) NodeHealth(node string, value []byte, op Op) { + w.mu.Lock() + defer w.mu.Unlock() + if op == Put { + h, err := delivery.DecodeNodeHealth(value) + if err != nil { + w.say("the health report of %s cannot be read, and is left: %v", node, err) + return + } + w.health[node] = h + } else { + delete(w.health, node) + } + w.nodeFindings(node) + w.unreachable() +} + +// TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it. +func (w *Watchman) tunnelKey(node string, record *delivery.NodeRecord) { + key := "" + if record != nil { + key = record.TunnelKey + } + if w.tunnels[node] == key { + return + } + if key == "" { + delete(w.tunnels, node) + } else { + w.tunnels[node] = key + } + w.unreachable() +} + +// nodeFindings writes and deletes one node's own findings from its report. +func (w *Watchman) nodeFindings(node string) { + h, ok := w.health[node] + want := map[string]bool{} + if ok { + for _, f := range w.networkOf(node, h) { + want[f.Key()] = true + w.find(f) + } + if f, raise := unitsOf(node, h.Units); raise { + want[f.Key()] = true + w.find(f) + } + for _, f := range reloginOf(node, h.Relogin) { + want[f.Key()] = true + w.find(f) + } + for _, f := range foundOf(node, h.Found) { + want[f.Key()] = true + w.find(f) + } + } + for key := range w.findings { + if !want[key] && w.ownsHealthKey(key, node) { + w.unfind(key) + } + } +} + +// ownsHealthKey says a finding key is one of node's own report findings. +func (w *Watchman) ownsHealthKey(key, node string) bool { + f, err := DecodeFinding(w.findings[key]) + if err != nil || f.Machine != node || f.Source != "health-report" { + return false + } + return f.Kind != KindNetworkUnreachable +} + +// networkOf is a node's own network findings: the resolver file rewritten, and the rest of its unhealthy parts. +func (w *Watchman) networkOf(m string, h delivery.NodeHealth) []Finding { + if h.Network == nil { + return nil + } + var parts []delivery.NetworkPart + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + parts = append(parts, p) + } + } + var out []Finding + var own []delivery.NetworkPart + var rewritten *delivery.NetworkPart + for i, p := range parts { + if p.Part == partResolvConf { + rewritten = &parts[i] + continue + } + own = append(own, p) + } + if rewritten != nil { + out = append(out, w.rewrittenOf(m, *rewritten, parts)) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + var words, said []string + severity := Warning + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = Urgent + } + } + if w.control[m] { + severity = Urgent + } + out = append(out, Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, + Severity: severity, Source: "health-report", Since: h.Network.Since, + Summary: fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")), Said: strings.Join(said, " | ")}) + } + return out +} + +func (w *Watchman) rewrittenOf(m string, p delivery.NetworkPart, all []delivery.NetworkPart) Finding { + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names still resolve through what it wrote" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == partNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + id := m + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ + "back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + id = m + "." + p.Owner + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ + "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + severity := Warning + if w.control[m] { + severity = Urgent + } + return Finding{Scope: ScopeMachine, ID: id, Token: "rewritten", Kind: KindNetworkRewritten, Machine: m, Severity: severity, + Source: "health-report", Since: p.Since, Summary: summary, + Said: fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))} +} + +// unitsOf is a machine's failed units no module places, one finding. +func unitsOf(node string, u *delivery.UnitsHealth) (Finding, bool) { + if u == nil || len(u.Failed) == 0 { + return Finding{}, false + } + var names, said []string + since := u.Failed[0].Since + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + scope, result := f.Scope, f.Result + if scope == "" { + scope = "system" + } + if result == "" { + result = "no reason said" + } + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + said = append(said, line+", since "+stampSec(f.Since)) + if f.Since.Before(since) { + since = f.Since + } + } + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + return Finding{Scope: ScopeMachine, ID: node, Token: "units", Kind: KindMachineUnits, Machine: node, Severity: Warning, + Source: "health-report", Since: since, + Summary: fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ + "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), + Said: strings.Join(said, "; "), + Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node, + Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node), + Resolved: "No failed services on " + node + " any more"}, true +} + +// reloginOf is one finding per module whose accounts wait for a new login. +func reloginOf(node string, accounts []delivery.NodeAccount) []Finding { + by := map[string][]delivery.NodeAccount{} + for _, a := range accounts { + by[a.Module] = append(by[a.Module], a) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var names []string + since := by[module][0].Since + for _, a := range by[module] { + names = append(names, a.Account) + if a.Since.Before(since) { + since = a.Since + } + } + sort.Strings(names) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ + "before it was; log out of %[1]s completely and log in again, or restart it", node, orModule(module), strings.Join(names, ", ")) + out = append(out, Finding{Scope: ScopeModule, ID: orModule(module) + "." + node, Kind: KindReloginNeeded, Machine: node, + Severity: Warning, Source: "health-report", Since: since, Summary: said, Said: said}) + } + return out +} + +// foundOf is one finding per module with a directory used as found. +func foundOf(node string, dirs []delivery.NodeDirectory) []Finding { + by := map[string][]delivery.NodeDirectory{} + for _, d := range dirs { + by[d.Module] = append(by[d.Module], d) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var ids, why []string + since := by[module][0].Since + for _, d := range by[module] { + ids = append(ids, d.Resource) + why = append(why, d.Reason) + if d.Since.Before(since) { + since = d.Since + } + } + m := orModule(module) + said := fmt.Sprintf("%s on %s uses %s as found: %s", m, node, strings.Join(ids, ", "), strings.Join(why, "; ")) + out = append(out, Finding{Scope: ScopeModule, ID: m + "." + node, Kind: KindUsedAsFound, Machine: node, Severity: Warning, + Source: "health-report", Since: since, Summary: said, Said: said, + Headline: fmt.Sprintf("%s waits for a directory on %s", m, node), + Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ + "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m), + Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type.", + Resolved: fmt.Sprintf("%s's directory on %s is the mesh's", m, node)}) + } + return out +} + +func orModule(m string) string { + if m == "" { + return "_" + } + return m +} + +// unreachable is network-unreachable across every report: a node whose tunnel key two or more other nodes' peers list +// as not seen, said once there, unless its own network finding or its silence says it. +func (w *Watchman) unreachable() { + byKey := map[string]string{} + for node, key := range w.tunnels { + byKey[key] = node + } + notSeen := map[string][]string{} + for from, h := range w.health { + for _, p := range h.Peers { + x, ok := byKey[p.Key] + if !ok || x == from || p.Seen { + continue + } + notSeen[x] = append(notSeen[x], from) + } + } + want := map[string]bool{} + for x, from := range notSeen { + if len(from) < 2 { + continue + } + if _, own := w.findings[FindingKey(ScopeMachine, x, "network")]; own { + continue + } + if _, silent := w.findings[FindingKey(ScopeMachine, x, KindSilent)]; silent { + continue + } + sort.Strings(from) + severity := Warning + if w.control[x] { + severity = Urgent + } + f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from, + Severity: severity, Source: "health-report", + Summary: fmt.Sprintf("%s cannot be reached from %s: the tunnel to it", x, strings.Join(from, ", ")), + Said: fmt.Sprintf("%s: tunnel: no handshake with %s within five minutes", strings.Join(from, ", "), x)} + want[f.Key()] = true + w.find(f) + } + for key := range w.findings { + if strings.HasSuffix(key, ".unreachable") && strings.HasPrefix(key, ScopeMachine+".") && !want[key] { + w.unfind(key) + } + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 45c2881..217f896 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -240,6 +240,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { if s.Bucket == delivery.PresenceBucket { if node, part, _, err := delivery.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { key := FindingKey(ScopeMachine, node, KindSilent) + defer w.unreachable() // a silent machine is said by its silence, never also as unreachable if s.Moved || w.away[node] { w.unfind(key) w.unsignal("S1", node) diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 84e1ef0..cc4dcc3 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -86,6 +86,9 @@ type Watchman struct { recount func() // waiting is, by `.`, each of its provisions in waits, with its words. waiting map[string]map[string]string + // health is each node's own health report, and tunnels each node's tunnel key from its record. + health map[string]delivery.NodeHealth + tunnels map[string]string // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -96,7 +99,8 @@ func New(out Out, now func() time.Time) *Watchman { pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, - control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}} + control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, + health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 21a1ad5..a57ddfe 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -91,7 +91,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 reads := slices.Clone(m.Reads) slices.Sort(reads) if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease", - "mesh-delivery-assigner.wants", "mesh-provisioner.provisions", "node-engine.have"}) { + "mesh-delivery-assigner.wants", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 291908c..fcc3077 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -21,6 +21,7 @@ "mesh-build-node.steps", "mesh-controller.lease", "mesh-delivery-assigner.wants", + "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have" ], From ee2f1efe78a2fefbd15b1f0b0dd7781279a7167b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:06:19 +0100 Subject: [PATCH 21/35] Watchman: hold a failing network part at the machines it points at, as the controller does, so a machine others cannot reach is said once there and not as each reporter's own (issue 774) Pins mesh-sdk 6eae076. --- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 +- .../internal/watch/findings_test.go | 39 +++ .../internal/watch/nodehealth.go | 242 ++++++++++++------ .../internal/watch/signals.go | 2 +- 5 files changed, 201 insertions(+), 88 deletions(-) diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index db64d6f..57dac09 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index d016872..a9c7447 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 h1:4qfwx2bv2vosFM6tEbBZUeaQ/pfRauXZuNI3whul30g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 4fbf380..03f8cac 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -200,6 +200,45 @@ func TestTheWatchmansFindings(t *testing.T) { } w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox"}), Put) // all well again }, map[string]string{}}, + {"774: a failing part is held at the machines it points at, as the controller holds it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + toward := func(node, part string, targets ...string) []byte { + b, _ := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: node, Network: &delivery.NetworkHealth{State: "unhealthy", + Parts: []delivery.NetworkPart{{Part: part, State: "unhealthy", Reason: part + " fails", Said: "x", Targets: targets}}}}) + return b + } + // own part: one machine alone failing toward a healthy one is its own. + w.NodeHealth("ace", toward("ace", "names", "dns1"), Put) + if _, ok := o.findings["machine.ace.network"]; !ok || len(o.findings) != 1 { + t.Fatalf("own part: findings %v", o.findings) + } + // two resolvers' machines: a second machine failing toward dns1 holds both there, said once as unreachable. + w.NodeHealth("g14", toward("g14", "names", "dns1"), Put) + if f, ok := o.findings["machine.dns1.unreachable"]; !ok || len(f.Also) != 2 || len(o.findings) != 1 { + t.Fatalf("two toward one resolver: findings %v", o.findings) + } + w.NodeHealth("ace", toward("ace", "names", "dns1", "dns2"), Put) // toward two resolvers, one not down: its own + if _, ok := o.findings["machine.ace.network"]; !ok { + t.Fatalf("two resolvers, one up: findings %v", o.findings) + } + w.NodeHealth("ace", nil, Delete) + w.NodeHealth("g14", nil, Delete) + // the hub silent on the record: a tunnel toward it is held there, and its silence says it. + w.Connect("node.hub") + w.Disconnect("node.hub") + expire(w, c, pulseKeyOf(t, presenceNS, "hub"), 120*time.Second) + w.NodeHealth("lap", toward("lap", "tunnel", "hub"), Put) + if _, own := o.findings["machine.lap.network"]; own { + t.Errorf("a tunnel toward the silent hub is the reporter's own: %v", o.findings) + } + if _, ok := o.findings["machine.hub.unreachable"]; ok { + t.Error("the silent hub is said unreachable as well") + } + w.Connect("node.hub") // back: lap's tunnel failing toward a hub up is its own again + if _, own := o.findings["machine.lap.network"]; !own { + t.Errorf("hub back: findings %v", o.findings) + } + w.NodeHealth("lap", nil, Delete) + }, map[string]string{}}, {"774: a node two nodes' tunnels no longer see is unreachable, said once there; not while it is silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { for node, key := range map[string]string{"ace": "kA", "g14": "kG", "lap": "kL"} { b, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, TunnelKey: key}}) diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go index 7cb6f91..d3b6dd3 100644 --- a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -23,9 +23,10 @@ import ( // - relogin-needed `module...relogin-needed` and directory-used-as-found // `module...directory-used-as-found`. // -// What the controller held at another machine by the addresses its parts point at (Toward) needs each node's private -// address and the hub, which no report carries: a part pointing elsewhere is the machine's own here, and the machine all -// point at is said by its peers instead. +// A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held +// there, as the controller holds it: when every machine it points at is down on the record (silent, its own network +// unhealthy, or two machines failing toward it), it is that machine's `unreachable`, said once there, and no finding of +// the reporter's. A tunnel peer two or more nodes no longer see points at its node the same way. // The kinds a node's health report raises. const ( @@ -63,7 +64,7 @@ func (w *Watchman) NodeHealth(node string, value []byte, op Op) { delete(w.health, node) } w.nodeFindings(node) - w.unreachable() + w.networks() } // TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it. @@ -80,7 +81,7 @@ func (w *Watchman) tunnelKey(node string, record *delivery.NodeRecord) { } else { w.tunnels[node] = key } - w.unreachable() + w.networks() } // nodeFindings writes and deletes one node's own findings from its report. @@ -88,10 +89,6 @@ func (w *Watchman) nodeFindings(node string) { h, ok := w.health[node] want := map[string]bool{} if ok { - for _, f := range w.networkOf(node, h) { - want[f.Key()] = true - w.find(f) - } if f, raise := unitsOf(node, h.Units); raise { want[f.Key()] = true w.find(f) @@ -118,58 +115,7 @@ func (w *Watchman) ownsHealthKey(key, node string) bool { if err != nil || f.Machine != node || f.Source != "health-report" { return false } - return f.Kind != KindNetworkUnreachable -} - -// networkOf is a node's own network findings: the resolver file rewritten, and the rest of its unhealthy parts. -func (w *Watchman) networkOf(m string, h delivery.NodeHealth) []Finding { - if h.Network == nil { - return nil - } - var parts []delivery.NetworkPart - for _, p := range h.Network.Parts { - if p.State == stateUnhealthy { - parts = append(parts, p) - } - } - var out []Finding - var own []delivery.NetworkPart - var rewritten *delivery.NetworkPart - for i, p := range parts { - if p.Part == partResolvConf { - rewritten = &parts[i] - continue - } - own = append(own, p) - } - if rewritten != nil { - out = append(out, w.rewrittenOf(m, *rewritten, parts)) - kept := own[:0] - for _, p := range own { - if p.Part != partNames { - kept = append(kept, p) - } - } - own = kept - } - if len(own) > 0 { - var words, said []string - severity := Warning - for _, p := range own { - words = append(words, p.Reason) - said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) - if p.Part == partBus { - severity = Urgent - } - } - if w.control[m] { - severity = Urgent - } - out = append(out, Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, - Severity: severity, Source: "health-report", Since: h.Network.Since, - Summary: fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")), Said: strings.Join(said, " | ")}) - } - return out + return f.Kind != KindNetworkUnreachable && f.Kind != KindMachineNetwork && f.Kind != KindNetworkRewritten } func (w *Watchman) rewrittenOf(m string, p delivery.NetworkPart, all []delivery.NetworkPart) Finding { @@ -310,48 +256,176 @@ func orModule(m string) string { return m } -// unreachable is network-unreachable across every report: a node whose tunnel key two or more other nodes' peers list -// as not seen, said once there, unless its own network finding or its silence says it. -func (w *Watchman) unreachable() { +// pointed is one machine's failing part that points at another machine. +type pointed struct { + from string + part delivery.NetworkPart +} + +// networks is every machine's network findings from every report together (the controller's networkObservations): the +// resolver file rewritten, each machine's own unhealthy parts, and the machine others point at, held there. +func (w *Watchman) networks() { + machines := slices.Sorted(maps.Keys(w.health)) + unhealthy := func(m string) []delivery.NetworkPart { + h := w.health[m] + if h.Network == nil { + return nil + } + var out []delivery.NetworkPart + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + out = append(out, p) + } + } + return out + } + targets := func(m string, p delivery.NetworkPart) ([]string, bool) { + var out []string + for _, x := range p.Targets { + if x == "" || x == m { + return nil, false + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out, len(out) > 0 + } + pointing := map[string][]pointed{} + for _, m := range machines { + for _, p := range unhealthy(m) { + if xs, ok := targets(m, p); ok { + for _, x := range xs { + pointing[x] = append(pointing[x], pointed{from: m, part: p}) + } + } + } + } + // A tunnel peer a node no longer sees points at that peer's node, as a tunnel part toward it would. byKey := map[string]string{} for node, key := range w.tunnels { byKey[key] = node } - notSeen := map[string][]string{} - for from, h := range w.health { - for _, p := range h.Peers { - x, ok := byKey[p.Key] - if !ok || x == from || p.Seen { - continue + for _, m := range machines { + for _, p := range w.health[m].Peers { + if x, ok := byKey[p.Key]; ok && x != m && !p.Seen { + pointing[x] = append(pointing[x], pointed{from: m, part: delivery.NetworkPart{Part: partTunnel, State: stateUnhealthy, + Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}}) } - notSeen[x] = append(notSeen[x], from) } } + from := func(x string) []string { + var out []string + for _, pt := range pointing[x] { + if !slices.Contains(out, pt.from) { + out = append(out, pt.from) + } + } + sort.Strings(out) + return out + } + silent := func(x string) bool { + _, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)] + return ok + } + down := func(x string) bool { return silent(x) || len(unhealthy(x)) > 0 || len(from(x)) >= 2 } + allDown := func(xs []string) bool { + for _, x := range xs { + if !down(x) { + return false + } + } + return len(xs) > 0 + } want := map[string]bool{} - for x, from := range notSeen { - if len(from) < 2 { + saysOwn := map[string]bool{} + for _, m := range machines { + parts := unhealthy(m) + if len(parts) == 0 { continue } - if _, own := w.findings[FindingKey(ScopeMachine, x, "network")]; own { + var own []delivery.NetworkPart + var rewritten *delivery.NetworkPart + for i, p := range parts { + if p.Part == partResolvConf { + rewritten = &parts[i] + continue + } + if xs, ok := targets(m, p); ok && allDown(xs) { + continue // held at the machines it points at + } + own = append(own, p) + } + if rewritten != nil { + f := w.rewrittenOf(m, *rewritten, parts) + want[f.Key()] = true + w.find(f) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) == 0 { continue } - if _, silent := w.findings[FindingKey(ScopeMachine, x, KindSilent)]; silent { - continue - } - sort.Strings(from) + var words, said []string severity := Warning - if w.control[x] { + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = Urgent + } + } + if w.control[m] { severity = Urgent } - f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from, + summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) + if waiting := from(m); len(waiting) > 0 { + severity = Urgent + summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) + } + f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Severity: severity, + Source: "health-report", Since: w.health[m].Network.Since, Summary: summary, Said: strings.Join(said, " | ")} + want[f.Key()] = true + saysOwn[m] = true + w.find(f) + } + for _, x := range slices.Sorted(maps.Keys(pointing)) { + if !down(x) || saysOwn[x] || silent(x) { + continue + } + var what, said []string + for _, pt := range pointing[x] { + wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[pt.part.Part] + if wd == "" { + wd = pt.part.Part + } + if !slices.Contains(what, wd) { + what = append(what, wd) + } + said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) + } + severity := Warning + if w.control[x] || slices.Contains(what, "the bus on it") { + severity = Urgent + } + f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from(x), Severity: severity, Source: "health-report", - Summary: fmt.Sprintf("%s cannot be reached from %s: the tunnel to it", x, strings.Join(from, ", ")), - Said: fmt.Sprintf("%s: tunnel: no handshake with %s within five minutes", strings.Join(from, ", "), x)} + Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from(x), ", "), strings.Join(what, ", ")), + Said: strings.Join(said, " | ")} want[f.Key()] = true w.find(f) } - for key := range w.findings { - if strings.HasSuffix(key, ".unreachable") && strings.HasPrefix(key, ScopeMachine+".") && !want[key] { + for key, value := range w.findings { + f, err := DecodeFinding(value) + if err != nil || f.Source != "health-report" { + continue + } + if (f.Kind == KindMachineNetwork || f.Kind == KindNetworkRewritten || f.Kind == KindNetworkUnreachable) && !want[key] { w.unfind(key) } } diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 217f896..214ee3f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -240,7 +240,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { if s.Bucket == delivery.PresenceBucket { if node, part, _, err := delivery.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { key := FindingKey(ScopeMachine, node, KindSilent) - defer w.unreachable() // a silent machine is said by its silence, never also as unreachable + defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence if s.Moved || w.away[node] { w.unfind(key) w.unsignal("S1", node) From e5192f55ee9a88f217f9b64fbb042d22a309c98c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:08:46 +0100 Subject: [PATCH 22/35] =?UTF-8?q?View:=20read=20no=20build=20record=20and?= =?UTF-8?q?=20no=20composition=20of=20the=20controller's:=20fingerprints?= =?UTF-8?q?=20against=20the=20builder's=20done=20keys=20alone,=20a=20move'?= =?UTF-8?q?s=20judgement=20from=20the=20node=20agent's=20have=20(issue=201?= =?UTF-8?q?955,=20ADR=200571=20=C2=A71,=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mesh-delivery-view/integration_test.go | 8 +- .../cmd/mesh-delivery-view/main.go | 2 +- .../internal/view/fingerprints_test.go | 103 ++---------------- .../mesh-delivery-view/internal/view/view.go | 82 ++------------ .../internal/view/view_test.go | 11 -- .../internal/view/wants_test.go | 11 +- modules/mesh-delivery-view/manifest_test.go | 2 +- modules/mesh-delivery-view/module.json | 2 - 8 files changed, 34 insertions(+), 187 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 140707e..3cbf53e 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -48,7 +48,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -82,8 +82,6 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 } put(delivery.PlacementBucket, "placement.a", placeValue("a", "x", "y")) put(delivery.PlacementBucket, "placement.b", placeValue("b")) - put(view.BuildsBucket, "x.build.b1", []byte(`{"id":"b1","module":"x"}`)) - put(view.BuildsBucket, "x.newest", []byte("b1")) put(delivery.HaveBucket, "a.x", haveValue("b1", delivery.PhaseChecking, delivery.HealthWaits)) put(delivery.HaveBucket, "b.x", haveValue("b1", delivery.PhaseRunning, delivery.HealthOK)) @@ -161,7 +159,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -247,7 +245,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index dfdd0c5..a9953e7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -106,7 +106,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) - for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { + for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { go follow(ctx, js, bucket, false, model, logger) } // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). diff --git a/modules/mesh-delivery-view/internal/view/fingerprints_test.go b/modules/mesh-delivery-view/internal/view/fingerprints_test.go index d4faa9e..3004d0d 100644 --- a/modules/mesh-delivery-view/internal/view/fingerprints_test.go +++ b/modules/mesh-delivery-view/internal/view/fingerprints_test.go @@ -50,30 +50,6 @@ func modEntry(t *testing.T, name string, m delivery.Module) Entry { return Entry{Bucket: delivery.ModulesBucket, Key: name, Value: b, At: t0} } -// trunkBuild is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is -// the captured shape: id, state, repository, ref, module, commit, path, source-fingerprint, source-inputs, at). -func trunkBuild(module, id, ref, state, fp, at string) []Entry { - // an outcome has no state key; a failed one has "failed" (mesh-controller internal/chain/records.go:57-59, :109) - rec := `{"id":"` + id + `",` - switch state { - case "": - case "failed": - rec += `"failed":"the build failed",` - default: - rec += `"state":"` + state + `",` - } - rec += `"repository":"novox/mesh-delivery","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/` + module + `","at":"` + at + `"` - if fp != "" { - // the controller writes the inputs whenever it writes the fingerprint (mesh-sdk testdata/build-record-build-agent.json) - rec += `,"source-fingerprint":"` + fp + `","source-inputs":{"Module":"` + module + `","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":""}` - } - rec += `}` - return []Entry{ - {Bucket: BuildsBucket, Key: module + ".build." + id, Value: []byte(rec), At: t0}, - {Bucket: BuildsBucket, Key: module + ".newest", Value: []byte(id), At: t0}, - } -} - func fpRow(t *testing.T, s Snapshot, module string) FingerprintRow { t.Helper() for _, f := range s.Fingerprints { @@ -111,62 +87,27 @@ func TestSnapshotListsTheCandidateGroups(t *testing.T) { // 1241/AC1 } } -func TestFingerprintAgainstTheNewestTrunkBuild(t *testing.T) { // 1241/AC2 +// 1955/L7 (ADR 0571 §1): the view reads no build record of the controller's: a module with no fingerprint of the new +// form is said so, never compared with the controller's builds. +func TestAModuleOfTheOldFormIsComparedWithNoBuild(t *testing.T) { m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...) - fill(m, trunkBuild("d", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...) fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "d", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "b", delivery.Module{None: "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4", Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "e", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA})) + modEntry(t, "b", delivery.Module{None: "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4", Repository: "novox/mesh-delivery", Commit: hA})) s := m.Snapshot() - if r := fpRow(t, s, "a"); r.State != FPSame || r.Indexer != fF || r.Build != fF { - t.Errorf("a = %+v, want same", r) - } - if r := fpRow(t, s, "d"); r.State != FPDiffers || r.Indexer != fG || r.Build != fF { - t.Errorf("d = %+v, want differs with both", r) - } - if r := fpRow(t, s, "b"); r.State != FPNone || !strings.Contains(r.Reason, "release 4") || r.Indexer != "" { - t.Errorf("b = %+v, want none with the reason", r) - } - if r := fpRow(t, s, "e"); r.State != FPNoBuild || r.Indexer != fG || r.Build != "" { - t.Errorf("e = %+v, want no build with the indexer's alone", r) - } -} - -func TestOnlyTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1241/AC2 (the newest key is only a hint) - m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fG, "2026-10-19T09:00:00Z")...) - fill(m, trunkBuild("a", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...) - fill(m, trunkBuild("a", "build-3", "main", "failed", fG, "2026-10-19T11:00:00Z")...) // newer, not built - fill(m, trunkBuild("a", "build-4", "feat/x", "", fG, "2026-10-19T12:00:00Z")...) // newer, not trunk; also the newest key - fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "a"); r.State != FPSame || r.Build != fF { - t.Errorf("a = %+v, want same with the 10:00 trunk build's fingerprint", r) - } -} - -func TestBuildFingerprintIsReadWithTheSDKRecord(t *testing.T) { // 1241/AC3 - // The record as the controller's encoder wrote it (mesh-sdk go/delivery/testdata/build-record-build-agent.json), whole, - // with its source-inputs under their Go field names: the view reads source-fingerprint from it. - const rec = `{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","manifest-digest":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","source-inputs":{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""},"made":[{"name":"server","kind":"image","reference":"registry.example/build-agent@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"}],"at":"2026-10-19T09:38:00Z"}` - const fp = "src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7" - m := New() - fill(m, Entry{Bucket: BuildsBucket, Key: delivery.BuildKey("build-agent", "build-1792395447088936520"), Value: []byte(rec), At: t0}, - Entry{Bucket: BuildsBucket, Key: delivery.NewestKey("build-agent"), Value: []byte("build-1792395447088936520"), At: t0}, - modEntry(t, "build-agent", delivery.Module{FP: fp, Repository: "novox/mesh-controller", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "build-agent"); r.State != FPSame || r.Build != fp { - t.Errorf("build-agent = %+v, want same with %s", r, fp) + for _, name := range []string{"a", "b"} { + if r := fpRow(t, s, name); r.State != FPNone || !(strings.Contains(r.Reason, "new form") || strings.Contains(r.Reason, "release 4")) || r.Build != "" { + t.Errorf("%s = %+v, want none: no build of the builder's is compared", name, r) + } } } func shown(t *testing.T) *Model { m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...) fill(m, candidate(t, "feat/x", pullOf("novox/mesh-delivery", 12, hA, "a"), pullOf("novox/mesh-catalog", 300, hB, "c")), - modEntry(t, "a", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}), + modEntry(t, "a", delivery.Module{FP2: fp2A, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "a", fp2A, delivery.BuilderBuild{State: delivery.BuildFailed, Error: "the compile failed"}), modEntry(t, "b", delivery.Module{None: "no tree", Repository: "novox/mesh-delivery", Commit: hA})) return m } @@ -175,14 +116,14 @@ func TestTableAndPagePrintTheGroupsAndTheComparison(t *testing.T) { // 1241/AC6 m := shown(t) var out bytes.Buffer Table(&out, m.Snapshot()) - for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "novox/mesh-catalog#300", "FINGERPRINT", "differs", "indexer " + fG, "build " + fF, "no tree"} { + for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "novox/mesh-catalog#300", "FINGERPRINT", "the compile failed", fp2A, "no tree"} { if !strings.Contains(out.String(), want) { t.Errorf("the table lacks %q:\n%s", want, out.String()) } } rec := httptest.NewRecorder() Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "FINGERPRINTS", "differs", fG, fF, "no tree"} { + for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "FINGERPRINTS", "the compile failed", fp2A, "no tree"} { if !strings.Contains(rec.Body.String(), want) { t.Errorf("the page lacks %q:\n%s", want, rec.Body.String()) } @@ -198,23 +139,3 @@ func TestUnreadableIndexerValuesAreNamed(t *testing.T) { // never dropped silent t.Errorf("snapshot = %+v", s) } } - -func TestAPlanAskedBuildWithNoRefIsATrunkBuild(t *testing.T) { // 1281/AC5 - // The build record of module mesh-delivery from mesh-catalog, build-1792435539606203626, as `mesh-controller.command - // builds` showed it on 2026-10-19 at about 21:20 local: built on ace at 20:45 local, the repository the clone URL, the - // source fingerprint below, and no ref (a delivery plan asked it after a merge; the listing shows no "at main"). - // No reading command shows the whole record: its path is the module's directory in the catalogue, and its - // source-inputs are mailu's live ones (mesh-sdk go/delivery/fingerprint_test.go) with the module's name put in. - const live = "src1:f79e2eedc2689cf6b3591ebfdb81ec65899131122fb2fe3dd9e384091d8a2a6d" - const rec = `{"id":"build-1792435539606203626","repository":"http://novox.internal:20000/novox/mesh-catalog.git",` + - `"module":"mesh-delivery","commit":"9efdd87a","path":"modules/mesh-delivery","at":"2026-10-19T18:45:00Z","source-fingerprint":"` + live + `",` + - `"source-inputs":{"Module":"mesh-delivery","Tree":"cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,"Toolchains":null,"Unpinned":""}}` - m := New() - fill(m, - Entry{Bucket: BuildsBucket, Key: "mesh-delivery.build.build-1792435539606203626", Value: []byte(rec), At: t0}, - Entry{Bucket: BuildsBucket, Key: "mesh-delivery.newest", Value: []byte("build-1792435539606203626"), At: t0}, - modEntry(t, "mesh-delivery", delivery.Module{FP: live, Repository: "novox/mesh-catalog", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "mesh-delivery"); r.State != FPSame || r.Build != live { - t.Errorf("mesh-delivery = %+v, want same as the no-ref build's fingerprint", r) - } -} diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index 05391bd..a447fe7 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -103,8 +103,8 @@ type WantRow struct { Newest string `json:"newest,omitempty"` Rule int `json:"rule,omitempty"` Words string `json:"words,omitempty"` - // Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or - // cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14). + // Judgement is the node agent's judgement of the move that waits for a person's yes (its have in waiting-yes, ADR 0547 + // §3), and Judged its words; empty otherwise. The controller's composed copy is no longer read (ADR 0571 §6). Judgement string `json:"judgement,omitempty"` Judged string `json:"judged,omitempty"` } @@ -589,13 +589,6 @@ func (m *Model) apply(e Entry) { } w, err := delivery.DecodeWant(e.Value) m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At} - case delivery.ControllerComposedBucket: - if e.Deleted { - delete(m.composed, e.Key) - return - } - d, err := delivery.DecodeComposedDigest(e.Value) - m.composed[e.Key] = composedEntry{value: d, err: errText(err)} case delivery.TakenBucket: if e.Deleted { delete(m.taken, e.Key) @@ -643,34 +636,6 @@ func (m *Model) apply(e Entry) { return } m.keys[e.Key] = keyEntry{value: b} - case BuildsBucket: - module, rest, ok := strings.Cut(e.Key, ".") - if !ok { - return - } - switch { - case rest == "newest": - if e.Deleted { - delete(m.newest, module) - return - } - m.newest[module] = strings.TrimSpace(string(e.Value)) - case strings.HasPrefix(rest, "build."): - if e.Deleted { - delete(m.builds, e.Key) - return - } - var b build - if json.Unmarshal(e.Value, &b) != nil { - return - } - // The source fingerprint is read with the SDK's record, not with the view's own struct. - var rec delivery.BuildRecord - if json.Unmarshal(e.Value, &rec) != nil { - return - } - m.builds[e.Key] = buildEntry{build: b, rec: rec} - } } } @@ -869,38 +834,22 @@ func (m *Model) groupRows() []GroupRow { // successful trunk build (delivery.BuildRecord.IsTrunkBuild: an outcome, not failed, on main or with no ref) that recorded one. The newest key is any state and ref, so it is not // used: the records are. func (m *Model) fingerprintRows() []FingerprintRow { - newest := map[string]buildEntry{} - for _, b := range m.builds { - if b.rec.Module == "" || !b.rec.IsTrunkBuild() || b.rec.SourceFingerprint == "" { - continue - } - if old, ok := newest[b.rec.Module]; !ok || b.rec.At.After(old.rec.At) || (b.rec.At.Equal(old.rec.At) && b.rec.ID > old.rec.ID) { - newest[b.rec.Module] = b - } - } out := []FingerprintRow{} for name, mod := range m.modules { if mod.err != "" { continue } row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit} - if mod.value.FP2 != "" || mod.value.None2 != "" { - // The new form (R4e): the indexer's fp2 against the builder's key of it. - out = append(out, m.newFormRow(name, row, mod.value)) + if mod.value.FP2 == "" && mod.value.None2 == "" { + // The builder builds the new form alone, and its done keys are the build history the view reads (ADR 0571 §1). + row.State, row.Reason = FPNone, mod.value.None + if row.Reason == "" { + row.Reason = "the indexer holds no fingerprint of the new form for it, so no build of the builder's is compared" + } + out = append(out, row) continue } - b, hasBuild := newest[name] - switch { - case mod.value.None != "": - row.State, row.Reason = FPNone, mod.value.None - case !hasBuild: - row.State, row.Indexer = FPNoBuild, mod.value.FP - case b.rec.SourceFingerprint == mod.value.FP: - row.State, row.Indexer, row.Build = FPSame, mod.value.FP, b.rec.SourceFingerprint - default: - row.State, row.Indexer, row.Build = FPDiffers, mod.value.FP, b.rec.SourceFingerprint - } - out = append(out, row) + out = append(out, m.newFormRow(name, row, mod.value)) } sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) return out @@ -959,11 +908,6 @@ func (s Snapshot) Counted() int { return n } -// The buckets the view reads besides the have bucket (which mesh-sdk's delivery package names). -const ( - BuildsBucket = "mesh-controller_builds" -) - // The statuses of a row. const ( StatusAssigned = "assigned" @@ -1089,10 +1033,8 @@ func (m *Model) wantRows() []WantRow { if has && h.err == "" { row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health) } - if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed { - if c, ok := m.composed[ckey]; ok && c.err == "" { - row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words - } + if has && h.err == "" && h.have.Phase == delivery.PhaseWaitingYes && !v.Removed { + row.Judgement, row.Judged = string(h.have.Judgement), h.have.Detail } switch { case v.Removed: diff --git a/modules/mesh-delivery-view/internal/view/view_test.go b/modules/mesh-delivery-view/internal/view/view_test.go index 9980d92..a68ad20 100644 --- a/modules/mesh-delivery-view/internal/view/view_test.go +++ b/modules/mesh-delivery-view/internal/view/view_test.go @@ -45,14 +45,6 @@ func place(t *testing.T, node string, modules ...string) Entry { return Entry{Bucket: delivery.PlacementBucket, Key: key, Value: v, At: t0} } -func buildRec(module, id, extra string) []Entry { - rec := `{"id":"` + id + `","module":"` + module + `","commit":"c"` + extra + `}` - return []Entry{ - {Bucket: BuildsBucket, Key: module + ".build." + id, Value: []byte(rec), At: t0}, - {Bucket: BuildsBucket, Key: module + ".newest", Value: []byte(id), At: t0}, - } -} - func find(t *testing.T, s Snapshot, node, module string) Row { t.Helper() for _, r := range s.Rows { @@ -78,9 +70,6 @@ func base(t *testing.T) []Entry { haveE(t, "b", "x", have(t, "b1", delivery.PhaseChecking, delivery.HealthWaits, "starting"), t0.Add(2*time.Minute)), haveE(t, "b", "z", have(t, "b3", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(3*time.Minute)), } - es = append(es, buildRec("x", "b1", "")...) - es = append(es, buildRec("y", "b2", "")...) - es = append(es, buildRec("z", "b3", "")...) return es } diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go index 80759a7..07ad6dc 100644 --- a/modules/mesh-delivery-view/internal/view/wants_test.go +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -143,16 +143,15 @@ func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) { // 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words. func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) { + // 1955/L7 (ADR 0571 §6): the judgement is the node agent's, in its have waiting for a yes; the controller's composed copy + // is not read. m := New() - key, err := delivery.ComposedKey("ace", "web", fpNew) + h, err := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK, + Judgement: delivery.JudgedCannot, Waits: "sha256:" + strings.Repeat("c", 64), Detail: "the composition it runs is gone"}) if err != nil { t.Fatal(err) } - v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}) - if err != nil { - t.Fatal(err) - } - fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0}) + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: h, At: t0}) rows := m.Snapshot().Wants if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") { t.Fatalf("rows = %+v", rows) diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 6aea715..4c36dc6 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index b82fde5..4d27585 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -41,8 +41,6 @@ "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", - "mesh-controller.builds", - "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", From 229a3e98553d314f012b66012c0849078117ce71 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:10:31 +0100 Subject: [PATCH 23/35] =?UTF-8?q?Assigner:=20from=20the=20lease=20write=20?= =?UTF-8?q?rule=207=20reads=20a=20placed=20node's=20own=20composition=20di?= =?UTF-8?q?gest=20from=20its=20have,=20an=20unplaced=20node=20gets=20no=20?= =?UTF-8?q?want=20and=20is=20said,=20and=20the=20controller's=20compositio?= =?UTF-8?q?ns=20are=20read=20only=20until=20then=20(issue=201955,=20ADR=20?= =?UTF-8?q?0571=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-assigner/pass.go | 48 +++++++++++++++++- .../mesh-delivery-assigner/seat_tools_test.go | 41 +++++++++++++++ .../cmd/mesh-delivery-assigner/world.go | 50 +++++++++++++++++++ .../internal/assign/policy.go | 20 ++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 9c841f7..3ba3270 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -36,6 +36,8 @@ type passer struct { failedProvisions map[string]bool // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. missing, missingSaid map[string]bool + // unplaced are the nodes whose unplaced finding this process wrote. + unplaced map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -142,6 +144,7 @@ func (p *passer) wants(ctx context.Context) { provisionsOf := p.provisionsOf() missing := map[string]bool{} defer func() { p.provisionsMissing(ctx, missing) }() + defer p.unplacedFindings() defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -171,6 +174,9 @@ func (p *passer) wants(ctx context.Context) { m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { + if placed[node] && p.w.handedAway() { + return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6) + } if placed[node] { // The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want // names the build alone, and the node judges the move itself. @@ -972,7 +978,11 @@ func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen ma now = p.now } split := map[string]bool{} - for _, s := range assign.Splits(manifests, chosen, assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))) { + theirs := map[string][]string{} + if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6) + theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket)) + } + for _, s := range assign.Splits(manifests, chosen, theirs) { split[s.Module] = true if p.splits[s.Module] { continue @@ -1140,3 +1150,39 @@ func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) } } } + +// unplacedFindings writes, from the lease write, the finding `machine..unplaced` for each node that runs but no +// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed. +func (p *passer) unplacedFindings() { + if p.findings.Put == nil { + return + } + now := []string(p.w.unplacedNodes()) + is := map[string]bool{} + for _, node := range now { + is[node] = true + if p.unplaced[node] { + continue + } + value, err := assign.UnplacedFinding(node, p.clock()) + if err == nil { + err = p.findings.Put(assign.UnplacedKey(node), value) + } + if err != nil { + p.logf("the unplaced node %s is not said: %v", node, err) + continue + } + if p.unplaced == nil { + p.unplaced = map[string]bool{} + } + p.unplaced[node] = true + } + for node := range p.unplaced { + if is[node] { + continue + } + if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil { + delete(p.unplaced, node) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index af24d5f..ea1b3ef 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -160,6 +160,47 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("the seat's verbs as served: %+v", tools) } t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) + t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) { + w := newWorld() + fp := delivery.SourcePrefix2 + strings.Repeat("2", 64) + ckey, _ := delivery.ComposedKey("ace", "web", fp) + cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither, + Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault}) + w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now()) + pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}}) + pkey, _ := delivery.PlacementKey("ace") + w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now()) + for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} { + hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest}) + hk, _ := delivery.HaveNodeKey(node) + w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now()) + } + if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 { + t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes()) + } + w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now()) + if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) { + t.Errorf("after the lease write, ace: %+v %v", d, ok) + } + if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) { + t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes()) + } + findings := map[string][]byte{} + p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + p.unplacedFindings() + if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 { + t.Fatalf("findings = %v", findings) + } + pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}) + pkey2, _ := delivery.PlacementKey("g14") + w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now()) + p.unplacedFindings() + if len(findings) != 0 { + t.Errorf("placed again: findings %v", findings) + } + }) t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { findings := map[string][]byte{} p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{ diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 8495a3d..175bf7a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -604,7 +604,17 @@ func (w *world) built(module, fp string) bool { // composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it // (mesh-controller_composed `..`), and whether it exists and can be read. +// +// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a +// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets +// no want (and the pass says so, unplacedFindings). func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) { + if w.handedAway() { + if !w.placedNodes()[node] { + return delivery.ComposedDigest{}, false + } + return w.ownComposed(node) + } key, err := delivery.ComposedKey(node, module, fp) if err != nil { return delivery.ComposedDigest{}, false @@ -622,6 +632,9 @@ func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool // nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `.node`). func (w *world) nodeComposed() map[string]delivery.ComposedDigest { out := map[string]delivery.ComposedDigest{} + if w.handedAway() { + return out // the controller composes no node from the lease write (ADR 0571 §6) + } for key, value := range w.read(delivery.ControllerComposedBucket) { node, _, _, isNode, err := delivery.ParseComposedKey(key) if err != nil || !isNode { @@ -683,3 +696,40 @@ func (w *world) runningCommits(ctx context.Context, get getter, module string) [ // keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5). var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState + +// ownComposed is a placed node's own composition, as the digest its own have names (`.node` of the haves, Composed): +// what rule 7 reads from the lease write. A node whose have names none is not composed yet. +func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) { + key, err := delivery.HaveNodeKey(node) + if err != nil { + return delivery.ComposedDigest{}, false + } + raw, ok := w.read(delivery.HaveBucket)[key] + if !ok { + return delivery.ComposedDigest{}, false + } + h, err := delivery.DecodeHaveNode(raw) + if err != nil || h.Composed == "" { + return delivery.ComposedDigest{}, false + } + return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither, + Words: "the node composes its own"}, true +} + +// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no +// want, since nothing composes them. +func (w *world) unplacedNodes() []string { + if !w.handedAway() { + return nil + } + placed := w.placedNodes() + var out []string + for key := range w.read(delivery.HaveBucket) { + node, _, isNode, err := delivery.ParseHaveKey(key) + if err == nil && isNode && !placed[node] { + out = append(out, node) + } + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index c2c9aa8..e877ba7 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -186,3 +186,23 @@ func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]b Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()}) } + +// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6). +const KindUnplaced = "unplaced" + +// UnplacedKey is its finding's key, `machine..unplaced`. +func UnplacedKey(node string) string { + return health.FindingKey(health.ScopeMachine, node, KindUnplaced) +} + +// UnplacedFinding is the finding of a node no placement names since the lease write. +func UnplacedFinding(node string, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+ + "so the assigner writes it no want", node) + return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced, + Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary, + Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.", + Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".", + Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()}) +} From b44f01f89ce734988bb66229d7aca855b2e5f2cd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:13:32 +0100 Subject: [PATCH 24/35] =?UTF-8?q?Builder:=20write=20each=20build's=20start?= =?UTF-8?q?=20and=20end=20to=20its=20own=20log=20on=20the=20bus,=20and=20a?= =?UTF-8?q?sk=20the=20seat=20artifact-store's=20held=20and=20hear=20its=20?= =?UTF-8?q?artifacts-gone,=20the=20controller's=20only=20until=20the=20lea?= =?UTF-8?q?se=20write,=20so=20build=20history=20and=20the=20store's=20swee?= =?UTF-8?q?p=20leave=20the=20controller=20(issue=201955,=20ADR=200571=20?= =?UTF-8?q?=C2=A71,=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins mesh-sdk f9a496d. --- .../cmd/mesh-delivery-builder/main.go | 8 ++++ .../cmd/mesh-delivery-builder/reasks.go | 23 +++++++++ .../cmd/mesh-delivery-builder/stored.go | 16 ++++++- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 +- .../internal/build/run.go | 48 +++++++++++++++++++ .../internal/build/run_test.go | 19 ++++++++ .../mesh-delivery-builder/manifest_test.go | 26 +++++++--- modules/mesh-delivery-builder/module.json | 13 ++++- 9 files changed, 146 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 7fb207c..13b7dcb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -120,6 +120,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Manifests: manifestReader{manifests}, Logf: logger.Printf, Stored: storedVia(calls.Call), + Node: cred.Node, } // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision @@ -129,6 +130,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} trunks.start() b.Trunk = trunks.head + // The builder's log (ADR 0571 §1): each build's start and end, on its own stream, read by the view, the indexer, the + // assigner and mesh-catalog. + b.Log = func(e delivery.BuildLogEntry) { + if err := emitLog(context.WithoutCancel(ctx), js, cred, e); err != nil { + logger.Printf("%v", err) + } + } b.Said = func(e build.AskedAgain) { if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { logger.Printf("%v", err) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 311ba12..def0bd1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -138,3 +139,25 @@ func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) } } + +// emitLog says one entry of the builder's log on its stream `log` (ADR 0571 §1), under an id the stream keeps one of. +func emitLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error { + data, err := delivery.EncodeBuildLogEntry(e) + if err != nil { + return err + } + msg := nats.NewMsg(process.ModuleStreamSubject(cred.Module, e.Event)) + id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", e.At.Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err) + } + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go index 3f2bbef..fab1825 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go @@ -9,7 +9,9 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "mesh-delivery-builder/internal/build" ) @@ -18,17 +20,27 @@ import ( // consumer of artifacts-gone (mesh-controller internal/broker EventsStream). const eventsStream = "EVENTS" +// SeatHeld is the seat artifact-store's verb held (ADR 0571 §2). +const SeatHeld = "seat:" + catalogue.ArtifactStoreSeat + ".held" + // toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call). type toolCaller func(key string, body any) (json.RawMessage, error) // storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue // 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: // the key is then left done, and the controller's take-in still refuses a gone blob. +// +// It asks the seat artifact-store's `held` (novox/hq ADR 0571 §2), and the controller's while that seat has no holder that +// answers (a named exception of clause 10, until the lease write). func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) { return func(_ context.Context, references []string) ([]string, error) { - raw, err := call(delivery.HeldVerb, map[string]any{"references": references}) + body := map[string]any{"references": references} + raw, err := call(SeatHeld, body) + if handacts.NoResponder(err) { + raw, err = call(delivery.HeldVerb, body) + } if err != nil { - return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err) + return nil, fmt.Errorf("asking held: %w", err) } a, err := delivery.DecodeHeldAnswer(raw, references) if err != nil { diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 1031a9e..38e5c36 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 043e05f..ecec070 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 h1:OsA1v6n736a7RqEV8Nkip8pf5ILMbzj0PWuKqcBOhK0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 935860b..aeb84b2 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "sort" + "strconv" "strings" "sync" "time" @@ -68,6 +69,11 @@ type Builder struct { Askings int // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. Said func(AskedAgain) + // Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody. + Log func(delivery.BuildLogEntry) + // Now is the clock of the log; nil is time.Now. Node is the node the builder runs on, which asked the build. + Now func() time.Time + Node string once sync.Once sem chan struct{} @@ -429,6 +435,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } b.handClaimed(key, cl.Rev) + b.logBuild(LogStarted(module, m, b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -478,6 +485,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { b.logf("%s: the build could not be written: %v", cl.Key, err) return } + b.logBuild(LogEnded(module, m, b.Node, b.now(), result)) if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } @@ -872,3 +880,43 @@ func needsFetch(tasks []Task) bool { } return false } + +func (b *Builder) now() time.Time { + if b.Now != nil { + return b.Now() + } + return time.Now() +} + +// logBuild tells the log an entry, said in the journal when it cannot be one. +func (b *Builder) logBuild(e delivery.BuildLogEntry) { + if b.Log == nil { + return + } + if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err) + return + } + b.Log(e) +} + +// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the +// source revision, where it is built from (repository, directory, commit), the node of the builder that asked it and the time. +func LogStarted(module string, m delivery.Module, by string, at time.Time) delivery.BuildLogEntry { + dir := m.Path + if dir == "" { + dir = "." + } + return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10), + Repository: m.Repository, Directory: dir, Commit: m.Commit, Node: by, At: at.UTC()} +} + +// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued) and each artifact's digest. +func LogEnded(module string, m delivery.Module, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry { + e := LogStarted(module, m, by, at) + e.Event, e.Outcome = delivery.BuildEndedEvent, result.State + for _, a := range result.Artifacts { + e.Artifacts = append(e.Artifacts, delivery.BuiltDigest{Name: a.Name, Digest: a.Digest}) + } + return e +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 6cbb41c..9a4a10f 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -221,7 +221,26 @@ var ( func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) + var logged []delivery.BuildLogEntry + var lmu sync.Mutex + r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() } v := r.consider(t, "modules/m") + // 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read + // from the two. + if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent || + logged[1].Outcome != delivery.BuildDone || len(logged[1].Artifacts) != 3 || logged[0].Repository != "novox/mesh-catalog" || + logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" || + logged[0].FP != fp1 || logged[0].Module != "m" { + t.Fatalf("logged = %+v", logged) + } + for _, e := range logged { + if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + t.Errorf("an entry the log refuses: %v", err) + } + } + if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok { + t.Error("no duration from the two entries") + } if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index cf0cac7..a43e1f9 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -40,10 +40,15 @@ type manifest struct { Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` - Reads []string `json:"reads"` - Uses []string `json:"uses"` - Invokes []string `json:"invokes"` - Config map[string]struct { + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + } `json:"streams"` + Reads []string `json:"reads"` + Uses []string `json:"uses"` + Invokes []string `json:"invokes"` + Config map[string]struct { Type string `json:"type"` Description string `json:"description"` Pattern string `json:"pattern"` @@ -101,12 +106,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. - if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 { + // 1955/L7 (ADR 0571 §2): the seat artifact-store's held, the controller's only while the seat has no holder (clause 10). + if want := []string{"seat:artifact-store.held", "seat:mesh-controller.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it // starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4). - if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // 1955/L7: and the seat artifact-store's, the controller's until the lease write (its take-in sweeps until then). + if want := []string{"artifact-store.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -122,7 +129,12 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } - for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + // 1955/L7 (ADR 0571 §1): its log, each build's start and end, kept 30 days. + if len(m.Streams) != 1 || m.Streams[0].Name != "log" || m.Streams[0].MaxAgeDays != 30 || + !reflect.DeepEqual(m.Streams[0].Events, []string{"build.started", "build.ended"}) { + t.Errorf("streams = %+v, want the log", m.Streams) + } + for _, k := range []string{"logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index faf5694..98c1b0b 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -13,11 +13,13 @@ "build.asked-again" ], "consumes": [ + "artifact-store.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone" ], "invokes": [ + "seat:artifact-store.held", "seat:mesh-controller.held" ], "state": [ @@ -27,6 +29,16 @@ { "name": "queues", "history": 1 }, { "name": "reasks", "history": 1 } ], + "streams": [ + { + "name": "log", + "events": [ + "build.started", + "build.ended" + ], + "max-age-days": 30 + } + ], "reads": [ "mesh-build-go.steps", "mesh-build-go.toolchains", @@ -44,7 +56,6 @@ "node-build-image", "node-build-node" ], - "own-secrets": { "broker": "${dir:state}/broker" }, From c045057ca0b378acf27bc73146fc43e747c6bf79 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:14:28 +0100 Subject: [PATCH 25/35] =?UTF-8?q?Assigner:=20write=20who=20holds=20which?= =?UTF-8?q?=20seat=20from=20placement=20into=20seat-holders,=20only=20the?= =?UTF-8?q?=20keys=20a=20change=20moves,=20and=20answer=20it=20from=20plac?= =?UTF-8?q?ement's=20seats,=20so=20readers=20no=20longer=20ask=20the=20con?= =?UTF-8?q?troller's=20seats=20(issue=201955,=20ADR=200571=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins mesh-sdk f9a496d. --- .../cmd/mesh-delivery-assigner/main.go | 14 ++++-- .../cmd/mesh-delivery-assigner/pass.go | 45 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/seat_tools.go | 33 ++++++++++++++ .../mesh-delivery-assigner/seat_tools_test.go | 45 +++++++++++++++++++ modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../mesh-delivery-assigner/manifest_test.go | 2 +- modules/mesh-delivery-assigner/module.json | 3 +- 8 files changed, 140 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7c232b3..737cee7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -88,7 +88,7 @@ const goBuildModule = "mesh-build-go" // (ADR 0567 §1), named after the assigner as every module state is. var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket, - delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket} + delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket} // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). @@ -112,6 +112,7 @@ var keptStates = []announce.State{ {Name: "wants", Readable: true}, {Name: health.FindingsState, Readable: true}, {Name: delivery.AssignerPoliciesState, Readable: true}, + {Name: delivery.SeatHoldersState, Readable: true}, } // buckets is every bucket the assigner opened, by name. @@ -160,6 +161,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == delivery.SeatHoldersBucket && err != nil { + // The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who + // holds which seat here, and the controller's seats still answer. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } if name == keptBucket && err != nil { // mesh-config's held values (ADR 0565 §5): without them a composition lays none, said in the journal. logf("the bucket %s cannot be opened (%v): no held value is laid on a composition until it is bound", name, err) @@ -329,6 +336,7 @@ var watched = []string{ assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, + delivery.SeatHoldersBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -425,7 +433,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket || name == keptBucket) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket || name == keptBucket || name == delivery.SeatHoldersBucket) { continue // not on the bus at this start (open said so) } started++ @@ -455,7 +463,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket} { + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} { if kvs[name] == nil && js != nil { name := name process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 3ba3270..93a28ff 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "maps" "slices" @@ -145,6 +146,7 @@ func (p *passer) wants(ctx context.Context) { missing := map[string]bool{} defer func() { p.provisionsMissing(ctx, missing) }() defer p.unplacedFindings() + defer p.seatHolders(ctx) defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -1186,3 +1188,46 @@ func (p *passer) unplacedFindings() { } } } + +// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per +// seat held once, ``, and one per node of a node seat, `.`, written only where it differs, and a key no +// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no +// timer writes it. +func (p *passer) seatHolders(ctx context.Context) { + kv := p.kvs[delivery.SeatHoldersBucket] + if kv == nil { + return + } + placements := map[string]delivery.Placement{} + for key, value := range p.w.read(delivery.PlacementBucket) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + continue + } + if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node { + placements[node] = pl + } + } + if len(placements) == 0 { + return // no placement read yet: nothing is known of who holds what, and nothing is deleted + } + want := delivery.SeatHoldersOf(placements) + have := p.w.read(delivery.SeatHoldersBucket) + for key, sh := range want { + value, err := json.Marshal(sh) + if err != nil || string(have[key]) == string(value) { + continue + } + p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil }) + } + for key := range have { + if _, ok := want[key]; ok { + continue + } + if err := kv.Delete(ctx, key); err != nil { + p.logf("the seat holders %s are not deleted: %v", key, err) + continue + } + p.w.put(delivery.SeatHoldersBucket, key, nil) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 3c683f7..46f1530 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strings" "time" @@ -375,6 +376,9 @@ type historyEntry struct { // doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the // changes in order from the revisions the state keeps. func (s *seatTools) doPlacement(args map[string]any) (any, error) { + if arg(args, "seats") == "true" { + return s.seats(arg(args, "node")) + } all, revs, err := s.placements() if err != nil { return nil, err @@ -538,3 +542,32 @@ func (s *seatTools) tools() []stdio.Tool { } return out } + +// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders +// holds it; with node, the keys of seats held on that node. +func (s *seatTools) seats(node string) (any, error) { + keys, err := s.st.keys(delivery.SeatHoldersState) + if err != nil { + return nil, fmt.Errorf("seat-holders cannot be listed: %w", err) + } + sort.Strings(keys) + out := []delivery.SeatHolders{} + for _, key := range keys { + raw, _, found, err := s.st.get(delivery.SeatHoldersState, key) + if err != nil { + return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err) + } + if !found { + continue + } + var sh delivery.SeatHolders + if json.Unmarshal(raw, &sh) != nil { + continue + } + if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) { + continue + } + out = append(out, sh) + } + return map[string]any{"seats": out, "holds": len(keys)}, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index ea1b3ef..87c0017 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -201,6 +201,40 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("placed again: findings %v", findings) } }) + t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) { + writes := 0 + kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes} + p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}} + place := func(node string, holds ...delivery.Holding) { + pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds}) + if err != nil { + t.Fatal(err) + } + k, _ := delivery.PlacementKey(node) + p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now()) + } + place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"}) + place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"}) + ctx := context.Background() + p.seatHolders(ctx) + if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil { + t.Fatalf("%d writes: %v", writes, kv.values) + } + p.seatHolders(ctx) + if writes != 3 { + t.Errorf("an unchanged placement wrote %d keys", writes-3) + } + place("g14") // g14 holds nothing now + p.seatHolders(ctx) + if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil { + t.Errorf("after g14's change: %d writes, %v", writes, kv.values) + } + st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}} + out, err := newSeatTools(st, nil).seats("ace") + if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 { + t.Errorf("seats on ace = %+v, %v", out, err) + } + }) t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { findings := map[string][]byte{} p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{ @@ -367,3 +401,14 @@ func readManifest() (catalogue.Manifest, error) { } return catalogue.ParseManifest(raw) } + +// countingKV is a memKV that counts its puts. +type countingKV struct { + memKV + writes *int +} + +func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) { + *c.writes++ + return c.memKV.Put(ctx, key, value) +} diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index a9b4fbe..0744ac9 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 326dffd..ecec070 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 h1:OsA1v6n736a7RqEV8Nkip8pf5ILMbzj0PWuKqcBOhK0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 84bee99..b939b72 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -44,7 +44,7 @@ func TestManifest(t *testing.T) { states[s.Name] = s.History } if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20, - "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20}; !reflect.DeepEqual(states, want) { + "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index ea1b4b7..9dae8b6 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -82,7 +82,8 @@ { "name": "puts", "history": 20 }, { "name": "wants", "history": 20 }, { "name": "findings", "history": 5 }, - { "name": "policies", "history": 20 } + { "name": "policies", "history": 20 }, + { "name": "seat-holders", "history": 5 } ], "streams": [ { From 17c0734fb27d049a2e6b077e6fa7a0db5deb156a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:48:14 +0100 Subject: [PATCH 26/35] =?UTF-8?q?Builder:=20keep=20each=20build's=20start?= =?UTF-8?q?=20and=20end=20in=20its=20own=20module=20log=20build-log,=20and?= =?UTF-8?q?=20answer=20builder=5Fbuilds=5Fsince=20from=20it,=20so=20reader?= =?UTF-8?q?s=20and=20mesh-catalog=20read=20the=20builder's=20history=20wit?= =?UTF-8?q?hout=20a=20bare=20stream=20(issue=201955,=20ADR=200571=20=C2=A7?= =?UTF-8?q?1,=20=C2=A79)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-builder-tools/main.go | 18 ++++++- .../mesh-delivery-builder/integration_test.go | 37 ++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 6 +-- .../cmd/mesh-delivery-builder/reasks.go | 20 ++++---- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 +- .../internal/build/run.go | 49 +++++++++++++------ .../internal/build/run_test.go | 21 +++++++- .../internal/build/since.go | 43 ++++++++++++++++ .../mesh-delivery-builder/manifest_test.go | 22 ++++----- modules/mesh-delivery-builder/module.json | 12 ++--- 11 files changed, 179 insertions(+), 55 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/since.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index dd9835d..49f5ddd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -26,7 +26,7 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for // the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer. -func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool { +func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: build.StuckTool, @@ -94,6 +94,20 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer, "next": build.StuckTool + " shows the key and its asks while they wait"}, nil }}, + {Name: delivery.BuildsSinceTool, + Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " + + "fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + + "queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " + + "log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{ + "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}}, + Run: func(args map[string]any) (any, error) { + at, err := time.Parse(time.RFC3339Nano, str(args, "since")) + if err != nil { + return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err) + } + return build.BuildsSince(since, at) + }}, {Name: build.AnnounceAgainTool, Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " + "again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " + @@ -250,7 +264,7 @@ func main() { a, err := build.DecodeReAsk(v) return err == nil && !a.Answered.IsZero() }) - if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil { + if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index b944b16..bae4fd6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -268,6 +269,12 @@ func startBedWith(t *testing.T, closed bool) *bed { mk(delivery.ManifestsBucket, delivery.ManifestsHistory) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) + // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. + logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog) + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, + AllowDirect: true}); err != nil { + t.Fatal(err) + } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.closed.Store(closed) b.fake.serve(t, "mesh-build-go", "n1", "n2") @@ -280,6 +287,29 @@ func startBedWith(t *testing.T, closed bool) *bed { return b } +// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user. +func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry { + t.Helper() + conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"}) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module) + if err != nil { + t.Fatal(err) + } + var out []delivery.BuildLogEntry + for _, le := range entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + // report writes a node's toolchains key of a build module and says so on the module's stream, as the module does. func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() @@ -406,6 +436,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } + // 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a + // reader with the log's read reads them, the end naming the manifest and what was made. + ended := b.logged(t, "netcheck") + if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent || + ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" { + t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended) + } for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 13b7dcb..cef81c7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -130,10 +130,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} trunks.start() b.Trunk = trunks.head - // The builder's log (ADR 0571 §1): each build's start and end, on its own stream, read by the view, the indexer, the - // assigner and mesh-catalog. + // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the + // indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since. b.Log = func(e delivery.BuildLogEntry) { - if err := emitLog(context.WithoutCancel(ctx), js, cred, e); err != nil { + if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil { logger.Printf("%v", err) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index def0bd1..f2b4f37 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" @@ -140,20 +141,21 @@ func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e } } -// emitLog says one entry of the builder's log on its stream `log` (ADR 0571 §1), under an id the stream keeps one of. -func emitLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error { +// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key +// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential. +// The id makes a repeated append of one entry one entry. +func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error { data, err := delivery.EncodeBuildLogEntry(e) if err != nil { return err } - msg := nats.NewMsg(process.ModuleStreamSubject(cred.Module, e.Event)) - id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", cred.Module) - msg.Header.Set("x-node", cred.Node) - msg.Header.Set("x-time", e.At.Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") + key, err := delivery.BuildLogKey(e.Module) + if err != nil { + return err + } + msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key) msg.Data = data + id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) pub, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 38e5c36..ab1a8a6 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index ecec070..07e0ff7 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 h1:OsA1v6n736a7RqEV8Nkip8pf5ILMbzj0PWuKqcBOhK0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index aeb84b2..3687c21 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -435,7 +435,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } b.handClaimed(key, cl.Rev) - b.logBuild(LogStarted(module, m, b.Node, b.now())) + b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -485,7 +485,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { b.logf("%s: the build could not be written: %v", cl.Key, err) return } - b.logBuild(LogEnded(module, m, b.Node, b.now(), result)) + b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result)) if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } @@ -900,23 +900,40 @@ func (b *Builder) logBuild(e delivery.BuildLogEntry) { b.Log(e) } -// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the -// source revision, where it is built from (repository, directory, commit), the node of the builder that asked it and the time. -func LogStarted(module string, m delivery.Module, by string, at time.Time) delivery.BuildLogEntry { - dir := m.Path - if dir == "" { - dir = "." +// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's. +func (b *Builder) refOf(module, fp string) string { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "" } - return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10), - Repository: m.Repository, Directory: dir, Commit: m.Commit, Node: by, At: at.UTC()} + b.mu.Lock() + defer b.mu.Unlock() + return b.via[key].Ref } -// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued) and each artifact's digest. -func LogEnded(module string, m delivery.Module, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry { - e := LogStarted(module, m, by, at) - e.Event, e.Outcome = delivery.BuildEndedEvent, result.State - for _, a := range result.Artifacts { - e.Artifacts = append(e.Artifacts, delivery.BuiltDigest{Name: a.Name, Digest: a.Digest}) +// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the +// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder +// that asked it and the time. +func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry { + dir := m.Path + if dir == "" { + dir = delivery.RootPath } + return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10), + Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()} +} + +// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it +// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from. +func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry { + e := LogStarted(module, m, ref, by, at) + e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest + if result.Manifest != "" { + e.Manifest = result.Manifest + } + if m.Inputs2 != nil { + e.Against = append([]string(nil), m.Inputs2.Bases...) + } + e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...) return e } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 9a4a10f..a348fdc 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strconv" "strings" "sync" @@ -13,9 +12,11 @@ import ( "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the @@ -228,7 +229,8 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 // 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read // from the two. if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent || - logged[1].Outcome != delivery.BuildDone || len(logged[1].Artifacts) != 3 || logged[0].Repository != "novox/mesh-catalog" || + logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" || + logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" || logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" || logged[0].FP != fp1 || logged[0].Module != "m" { t.Fatalf("logged = %+v", logged) @@ -241,6 +243,21 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok { t.Error("no duration from the two entries") } + // builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more. + var page stdio.LogPage + for i, e := range logged { + raw, _ := delivery.EncodeBuildLogEntry(e) + page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At}) + } + page.Next = 3 + since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At) + if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { + t.Errorf("builds since = %+v, %v, want the one end and more", since, err) + } + page.Entries[0].Value = []byte(`{"v":0}`) + if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil { + t.Error("an entry that does not read was skipped, want it refused") + } if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } diff --git a/modules/mesh-delivery-builder/internal/build/since.go b/modules/mesh-delivery-builder/internal/build/since.go new file mode 100644 index 0000000..968e597 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/since.go @@ -0,0 +1,43 @@ +package build + +import ( + "fmt" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more. +const BuildsSinceLimit = 500 + +// LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since. +type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) + +// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, +// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again +// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that +// skipped it would register too little. +func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) { + out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}} + page, err := read(since, "", BuildsSinceLimit) + if err != nil { + return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err) + } + if page == nil { + return out, nil + } + var entries []delivery.BuildLogEntry + for _, le := range page.Entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err) + } + entries = append(entries, e) + } + if ended := delivery.EndedSince(entries, since); ended != nil { + out.Builds = ended + } + out.More = page.Next != 0 + return out, nil +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index a43e1f9..8914a77 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -10,6 +10,8 @@ import ( "slices" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, @@ -40,11 +42,10 @@ type manifest struct { Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` - Streams []struct { - Name string `json:"name"` - Events []string `json:"events"` - MaxAgeDays int `json:"max-age-days"` - } `json:"streams"` + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` Reads []string `json:"reads"` Uses []string `json:"uses"` Invokes []string `json:"invokes"` @@ -123,18 +124,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } - // 1955/L7 (ADR 0571 §1): its log, each build's start and end, kept 30 days. - if len(m.Streams) != 1 || m.Streams[0].Name != "log" || m.Streams[0].MaxAgeDays != 30 || - !reflect.DeepEqual(m.Streams[0].Events, []string{"build.started", "build.ended"}) { - t.Errorf("streams = %+v, want the log", m.Streams) + // 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream. + if len(m.Logs) != 1 || m.Logs[0].Name != delivery.BuilderLog || m.Logs[0].MaxAgeDays != delivery.BuilderLogDays { + t.Errorf("logs = %+v, want %s kept %d days", m.Logs, delivery.BuilderLog, delivery.BuilderLogDays) } - for _, k := range []string{"logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 98c1b0b..5606863 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -6,6 +6,7 @@ "builder_announce_again", "builder_ask_again", "builder_build_again", + "builder_builds_since", "builder_seat_queue", "builder_stuck" ], @@ -29,15 +30,8 @@ { "name": "queues", "history": 1 }, { "name": "reasks", "history": 1 } ], - "streams": [ - { - "name": "log", - "events": [ - "build.started", - "build.ended" - ], - "max-age-days": 30 - } + "logs": [ + { "name": "build-log", "max-age-days": 30 } ], "reads": [ "mesh-build-go.steps", From 06b58bc83bc6b305cb089c7ed806a7286f3cc97c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:48:14 +0100 Subject: [PATCH 27/35] =?UTF-8?q?Indexer:=20take=20directories=20and=20out?= =?UTF-8?q?side-built=20registrations=20from=20the=20builder's=20log,=20wa?= =?UTF-8?q?tched=20from=20its=20start,=20so=20no=20build=20record=20of=20t?= =?UTF-8?q?he=20controller's=20is=20read=20(issue=201955,=20ADR=200571=20?= =?UTF-8?q?=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mesh-delivery-indexer/integration_test.go | 95 ++++++++----------- .../cmd/mesh-delivery-indexer/main.go | 67 ++++++------- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 +- .../internal/index/builds.go | 74 +++++++-------- .../internal/index/index_test.go | 16 +--- .../internal/index/live_builds_test.go | 12 +-- .../internal/index/modules_test.go | 55 ++++++----- .../internal/index/registered_test.go | 4 +- .../internal/index/toolchains_test.go | 13 +-- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 2 +- 12 files changed, 154 insertions(+), 192 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index e27b51f..3262321 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -19,6 +19,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -44,21 +45,10 @@ accounts { // manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and // no mesh-delivery-gate_source bucket. func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - // Build records in the controller's layout (mesh-sdk go/delivery/testdata/build-record-build-agent.json, captured from - // the controller's encoder), under delivery.BuildKey, with the newest pointer beside them. - for _, b := range []struct{ repo, module, path, id string }{ - {"novox/mesh-delivery", "a", "modules/a", "build-1"}, {"novox/mesh-catalog", "c", "modules/c", "build-2"}, - } { - rec, _ := json.Marshal(map[string]any{"id": b.id, "repository": b.repo, "ref": "main", "module": b.module, - "commit": "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", "path": b.path, "at": "2026-10-19T09:38:00Z"}) - if _, err := builds.Put(ctx, delivery.BuildKey(b.module, b.id), rec); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey(b.module), []byte(b.id)); err != nil { - t.Fatal(err) - } - } + js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { + // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. + builds.end(t, "novox/mesh-delivery", "a", "modules/a") + builds.end(t, "novox/mesh-catalog", "c", "modules/c") }) h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef" for _, f := range []delivery.Forge{ @@ -99,7 +89,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 // startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill // mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created. -func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -162,11 +152,12 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil { t.Fatal(err) } - builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5}) - if err != nil { + // The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog), + Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - put(ctx, builds) + put(ctx, buildLog{ctx: ctx, js: js}) for _, b := range before { b(ctx, js) } @@ -185,15 +176,32 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b return js, ctx, candidates, modules } +// buildLog appends entries to the builder's log as the builder does. +type buildLog struct { + ctx context.Context + js jetstream.JetStream +} + +// end appends the end of a done trunk build of module from repo's directory path. +func (l buildLog) end(t *testing.T, repo, module, path string) { + t.Helper() + raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + module, + Repository: repo, Directory: path, Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Node: "n1", + At: time.Date(2026, 10, 19, 9, 38, 0, 0, time.UTC), Outcome: delivery.BuildDone}) + if err != nil { + t.Fatal(err) + } + if _, err := l.js.Publish(l.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil { + t.Fatal(err) + } +} + // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) // The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1). - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}` - if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, @@ -229,15 +237,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -287,15 +288,8 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -362,16 +356,9 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` const trunk, head = "5555555555555555555555555555555555555555", "0123456789abcdef0123456789abcdef01234567" - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }, func(ctx context.Context, js jetstream.JetStream) { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: PreviewsStream, Subjects: []string{previewWrittenSubject}}); err != nil { t.Fatal(err) @@ -451,7 +438,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) { } objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} - js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) + js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}) publish := func(merge, root string, objects map[string]string) { data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70, Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root}, @@ -518,7 +505,7 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" digest := delivery.ManifestDigest(manifest) - _, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) { + _, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { mkey, err := delivery.ManifestsKey(digest) if err != nil { t.Fatal(err) @@ -605,7 +592,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } } - js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) { + js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 05e6a27..e02d915 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -10,19 +10,20 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "io" "log" "os" "os/signal" + "sync/atomic" "syscall" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" @@ -277,28 +278,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The control records (ADR 0565 §2), new with this delivery and bound when they appear: until then nothing is written to them. predefined := process.AwaitBucket(ctx, js, conn, index.PredefinedBucket, "no predefined list is written", logger.Printf) configCandidates := process.AwaitBucket(ctx, js, conn, index.ConfigCandidatesBucket, "no pull request's config declarations are written", logger.Printf) - // The controller's build records only name the directory of a module whose key names none (fillPaths): without them - // such a module waits for its path, every other is indexed (issue 1945). - builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) + // The builder's log (ADR 0571 clause 1), read on the indexer's own credential (the calls connection) through its declared read: every entry it + // keeps at the start, then each new one as the builder appends it (no timer). It names the directory of a module whose key + // names none (fillPaths) and the places a registered module was built from; no build moves a fingerprint (ADR 0530 §1). + // Where the log cannot be read such a module waits for its path, every other is indexed (issue 1945), and it is said. known := index.NewBuilds() - // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any - // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). - each := func(e jetstream.KeyValueEntry) { - if opOf(e) { - known.Put(e.Key(), e.Value()) - } else { - known.Delete(e.Key()) - } - } errs := make(chan error, 1) - var buildsWatch jetstream.KeyWatcher - if builds != nil { - if w, err := settle(ctx, builds, each); err != nil { - process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again") - } else { - buildsWatch = w - } - } // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) @@ -310,6 +295,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call)} + // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each + // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). + var started atomic.Bool + if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error { + if err := known.Put(le.Value); err != nil { + logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err) + return nil + } + if started.Load() { + if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil { + logger.Printf("the registered module %s is not indexed: %v", le.Key, err) + } + } + return nil + }); err != nil { + logger.Printf("%s is not read (%v): a module whose key names no directory is not given one, and no registered module is "+ + "indexed from it, until the indexer starts again", delivery.BuildLogRead, err) + } + started.Store(true) // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. @@ -360,28 +364,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.RecomputeAtStart(ctx); err != nil { logger.Printf("computing the new-form fingerprints at start: %v", err) } - // A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records - // name it and the bucket holds none (registered.go); the records that follow are heard below. + // A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names + // it and the bucket holds none (registered.go); the entries that follow are heard above. if err := ix.IndexRegisteredAll(ctx); err != nil { logger.Printf("indexing the registered modules at start: %v", err) } - if buildsWatch != nil { - go func() { - errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) { - each(e) - if !opOf(e) { - return - } - var r delivery.BuildRecord - if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" { - return - } - if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil { - logger.Printf("the registered module %s is not indexed: %v", r.Module, err) - } - }) - }() - } // A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched, // as every other input. A module the forge could not read is also read again by itself on the next source revision and // merged message (index.RetryUnread). diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 52694c1..e98c8bd 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 685613a..07e0ff7 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 7c3eacd..02a7a63 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -1,72 +1,63 @@ package index import ( - "encoding/json" + "fmt" "maps" "slices" "strings" "sync" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Builds holds the controller's build records (mesh-controller_builds) for the two things the indexer still reads from -// them until release R4g, when build-agent goes (the lead's decision of R4e, 2026-10-22): -// - the Builder input of the builder's own module, which build-agent builds (TrunkOf; novox/hq ADR 0453 clause 5, -// issue 1727); -// - the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no fingerprint. +// Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads +// of past builds: the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no +// fingerprint, and the places a module registered from its own repository was built from (RegistrationsOf). It is filled +// from the log's entries at the indexer's start (every one the log keeps, 30 days) and then each new one as the builder +// appends it; no entry is ever taken out, since only age takes one from the log. // // It lists no module and maps no directory to a module: the indexer's modules are those its own messages and bucket name // (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once. type Builds struct { mu sync.Mutex - records map[string]delivery.BuildRecord // by bucket key + records map[string]delivery.BuildRecord // by the entry's event, module, fingerprint and time } // NewBuilds is an empty Builds. func NewBuilds() *Builds { return &Builds{records: map[string]delivery.BuildRecord{}} } -// Put takes a key of the builds bucket and its value. Only a build record of a module (key `.build.`, -// delivery.BuildKey) counts; the newest-pointer keys and requests that knew no module are not records of one. -func (b *Builds) Put(key string, value []byte) { - if !strings.Contains(key, ".build.") { - return +// Put takes one entry of the builder's log. A build's end is an outcome: done registers, failed or queued does not; its +// start is a request, which registers nothing either. An entry that does not read, or names no module or repository, is +// refused and said. +func (b *Builds) Put(value []byte) error { + e, err := delivery.DecodeBuildLogEntry(value) + if err != nil { + return err } - var r delivery.BuildRecord - if err := json.Unmarshal(value, &r); err != nil || r.Module == "" || r.Repository == "" { - return + if e.Module == "" || e.Repository == "" { + return fmt.Errorf("a build log entry naming no module or repository: %s", value) + } + r := delivery.BuildRecord{ID: e.At.UTC().Format(time.RFC3339Nano) + "." + e.FP, Repository: e.Repository, Ref: e.Ref, Module: e.Module, + Path: e.Directory, Commit: e.Commit, At: e.At} + switch { + case e.Event != delivery.BuildEndedEvent: + r.State = e.Event + case e.Outcome == delivery.BuildFailed: + r.Failed = "the build failed" + case e.Outcome != delivery.BuildDone: + r.State = e.Outcome } b.mu.Lock() defer b.mu.Unlock() - b.records[key] = r -} - -// Delete takes a key out. -func (b *Builds) Delete(key string) { - b.mu.Lock() - defer b.mu.Unlock() - delete(b.records, key) + b.records[e.Event+"."+e.Module+"."+r.ID] = r + return nil } func newer(a, b delivery.BuildRecord) bool { return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID) } -// TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when -// there is none (issue 1727). -func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { - var best *delivery.BuildRecord - b.mu.Lock() - defer b.mu.Unlock() - for _, r := range b.records { - if r.Module == module && r.IsTrunkBuild() && (best == nil || newer(r, *best)) { - c := r - best = &c - } - } - return best -} - // PathOf is the directory of a module of the repository as its newest build record of that repository names it, its // newest successful trunk build's first ("." for the repository's root), and false when no record of the repository // names the module (issue 1614). @@ -79,7 +70,7 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { if r.Module != module || !delivery.SameRepository(r.Repository, repository) { continue } - t := r.IsTrunkBuild() + t := registered(r) && trunkLike[r.Ref] if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) { c := r best, trunk = &c, t @@ -91,15 +82,14 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { return pathOf(strings.Trim(best.Path, "/")), true } -// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record +// Registration is a module registered in the catalogue from its own repository, as the builder log.s newest end of a build // of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was // built at. type Registration struct { Module, Repository, Path, Commit, Ref string } -// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it -// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a +// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a // repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were // built from nox-mesh, and every record of them names that ref). func registered(r delivery.BuildRecord) bool { diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 3dd794d..e6f9da4 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -2,7 +2,6 @@ package index import ( "context" - "encoding/json" "strings" "testing" @@ -85,14 +84,7 @@ func (m *memStore) Delete(_ context.Context, key string, rev uint64) error { // the controller's encoder; copied here, the vendored SDK does not carry testdata) has it, with the repository, module and path changed. func buildRecord(t *testing.T, repo, module, path, at string) []byte { t.Helper() - raw := []byte(`{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`) - var m map[string]any - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - m["repository"], m["module"], m["path"], m["at"] = repo, module, path, at - b, _ := json.Marshal(m) - return b + return logged(t, "build-"+module, module, repo, path, "", "", at) } func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge { @@ -104,9 +96,9 @@ func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) { t.Helper() st, logs := newMem(), &strings.Builder{} b := NewBuilds() - b.Put(delivery.BuildKey("a", "build-1"), buildRecord(t, "novox/mesh-delivery", "a", "modules/a", "2026-10-19T09:00:00Z")) - b.Put(delivery.BuildKey("c", "build-2"), buildRecord(t, "novox/mesh-catalog", "c", "modules/c", "2026-10-19T09:00:00Z")) - b.Put(delivery.BuildKey("odd", "build-3"), buildRecord(t, "novox/mesh-delivery", "odd-name", "modules/odd", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-delivery", "a", "modules/a", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-catalog", "c", "modules/c", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-delivery", "odd-name", "modules/odd", "2026-10-19T09:00:00Z")) return &Indexer{Store: st, Builds: b, Logf: func(f string, a ...any) { logs.WriteString(strings.TrimSpace(strings.ReplaceAll(f, "\n", " ")) + "\n") _ = a diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index 7aacda2..fe8c43a 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -2,6 +2,7 @@ package index import ( "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -23,13 +24,10 @@ const ( ) func liveRecord(id, module, repository, ref, commit, path, at, source string) [2]any { - raw := `{"id":"` + id + `","repository":"` + repository + `",` - if ref != "" { - raw += `"ref":"` + ref + `",` - } - raw += `"module":"` + module + `","commit":"` + commit + `","path":"` + path + `","at":"` + at + `","source-fingerprint":"` + source + - `","source-inputs":{"Module":"` + module + `",` + liveInputsTail + `}` - return [2]any{delivery.BuildKey(module, id), []byte(raw)} + when, _ := time.Parse(time.RFC3339, at) + raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: source, Repository: repository, + Directory: path, Commit: commit, Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone}) + return [2]any{module, raw} } func liveBuilds() [][2]any { diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index e65bc31..d090f08 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -2,12 +2,12 @@ package index import ( "context" - "encoding/json" "errors" "fmt" "path" "strings" "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -15,29 +15,38 @@ import ( const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -// record is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is -// the captured shape): SourceInputs under its Go field names, no json keys. inputs is that object's JSON. -func record(t *testing.T, id, module, repo, path, ref, state, at, inputs string) (string, []byte) { +// logged is one entry of the builder's log (mesh-sdk go/delivery BuildLogEntry, as the builder appends it): a build of +// module from repo's directory path at ref ("" for the trunk), its outcome ("" done, "failed", or "asked" for a build only +// started), at the time at; id makes its fingerprint. +func logged(t *testing.T, id, module, repo, path, ref, outcome, at string) []byte { t.Helper() - // an outcome has no state key (mesh-controller internal/chain/records.go:57-59); state asked is a request - raw := `{"id":"` + id + `",` - switch state { - case "": - case "failed": // a failure is an outcome with failed set and no state (mesh-controller internal/chain/records.go:109) - raw += `"failed":"the build failed",` - default: - raw += `"state":"` + state + `",` - } - raw += `"repository":"` + repo + `","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"` + path + `","at":"` + at + `"` - if inputs != "" { - raw += `,"source-fingerprint":"src1:ignored","source-inputs":` + inputs - } - raw += `}` - var check delivery.BuildRecord - if err := json.Unmarshal([]byte(raw), &check); err != nil { + when, err := time.Parse(time.RFC3339, at) + if err != nil { t.Fatal(err) } - return delivery.BuildKey(module, id), []byte(raw) + if path == "" { + path = delivery.RootPath + } + e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + id, Repository: repo, Directory: path, + Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone} + switch outcome { + case "": + case "failed": + e.Outcome = delivery.BuildFailed + default: + e.Event, e.Outcome = delivery.BuildStartedEvent, "" + } + raw, err := delivery.EncodeBuildLogEntry(e) + if err != nil { + t.Fatal(err) + } + return raw +} + +// record is logged under the module's log key, as the tests' tables carry it. +func record(t *testing.T, id, module, repo, path, ref, state, at, _ string) (string, []byte) { + t.Helper() + return module, logged(t, id, module, repo, path, ref, state, at) } func modulesIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore) { @@ -45,7 +54,9 @@ func modulesIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore) { mods := newMem() ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()} for _, r := range recs { - ix.Builds.Put(r[0].(string), r[1].([]byte)) + if err := ix.Builds.Put(r[1].([]byte)); err != nil { + t.Fatal(err) + } } return ix, mods } diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index 78101a4..d5751cd 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -302,14 +302,14 @@ func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""), rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""), } { - b.Put(r[0].(string), r[1].([]byte)) + _ = b.Put(r[1].([]byte)) } got := b.RegistrationsOf("photos") if len(got) != 1 || got[0].Ref != "nox-mesh" { t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got) } k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "") - b.Put(k, v) // a trunk's ref beats a branch of its own + _, _ = k, b.Put(v) // a trunk's ref beats a branch of its own if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" { t.Fatalf("registrations = %+v", got) } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index b9d01be..39b890c 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -46,13 +46,10 @@ const ( ) func agentBuild(id, src, at string) [2]any { - commit := "cada7e3c" - if id == "build-1792587792086548572" { - commit = "f10fbda1" - } - raw := `{"id":"` + id + `","repository":"` + agentCloneURL + `","ref":"main","module":"build-agent","commit":"` + commit + `",` + - `"path":"modules/build-agent","at":"` + at + `","source-fingerprint":"` + src + `","source-inputs":` + aInputs + `}` - return [2]any{delivery.BuildKey("build-agent", id), []byte(raw)} + when, _ := time.Parse(time.RFC3339, at) + raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "build-agent", FP: src, Repository: agentCloneURL, + Directory: "modules/build-agent", Commit: "cada7e3c", Node: "n1", At: when, Outcome: delivery.BuildDone}) + return [2]any{"build-agent", raw} } type mod struct { @@ -184,7 +181,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2) } a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z") - ix.Builds.Put(a[0].(string), a[1].([]byte)) + _ = ix.Builds.Put(a[1].([]byte)) if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 603cb46..161bf20 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -89,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-assigner.placement", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index faeb9f8..6b26c33 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -24,8 +24,8 @@ "reads": [ "mesh-build-go.toolchains", "mesh-build-node.toolchains", - "mesh-controller.builds", "mesh-delivery-assigner.placement", + "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], From e82ea89c2344282e320275838ac2796427f84af7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:48:14 +0100 Subject: [PATCH 28/35] =?UTF-8?q?Assigner:=20read=20the=20running=20commit?= =?UTF-8?q?=20of=20a=20reader=20of=20the=20manifests=20from=20the=20builde?= =?UTF-8?q?r's=20log,=20and=20hold=20the=20buckets=20it=20starts=20without?= =?UTF-8?q?=20in=20one=20list=20so=20the=20start=20test=20skips=20them=20(?= =?UTF-8?q?issue=201955,=20ADR=200571=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-assigner/asks.go | 6 +- .../compose_integration_test.go | 2 +- .../integration_test.go | 23 +++++++- .../cmd/mesh-delivery-assigner/main.go | 42 ++++++++++++-- .../cmd/mesh-delivery-assigner/main_test.go | 10 +++- .../cmd/mesh-delivery-assigner/world.go | 56 ++++++++++--------- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 2 +- 10 files changed, 109 insertions(+), 42 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go index 6fbe713..2ceefe7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go @@ -119,11 +119,11 @@ func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Cha if h := ch.Members[repository]; h != "" { commits, member = []string{h}, true } else { - commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module) + commits = p.w.runningCommits(module) } if len(commits) == 0 { - notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+ - "commit on record, and the group holds no member of "+repository) + notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+ + "trunk build of it, and the group holds no member of "+repository) continue } for _, commit := range commits { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go index 8bc0662..43030fa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go @@ -106,7 +106,7 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) { b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`)) running := strings.Repeat("3", 40) b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`)) - b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`)) + b.built(t, "mesh-controller", "novox/mesh-controller", running) sdkHead := strings.Repeat("4", 40) heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}} hash, _ := delivery.HeadsHash(heads) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 234b56a..5cce388 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -21,6 +21,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose/judge" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -82,6 +83,11 @@ func startBed(t *testing.T, leaveOut string) *bed { } b.buckets[name] = kv } + // The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog), + Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { + t.Fatal(err) + } if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { t.Fatal(err) } @@ -177,8 +183,8 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) { // 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket. func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { for _, missing := range readBuckets { - if missing == delivery.PlacementBucket || sourceOfGrants(missing) { - continue // it starts without it, below (the grants' source: main_test.go) + if startsWithout(missing) { + continue // it starts without it (the placement and the grants' source: main_test.go; the puts: below) } b := startBed(t, missing) select { @@ -264,6 +270,19 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { } } +// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does. +func (b *bed) built(t *testing.T, module, repository, commit string) { + t.Helper() + raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: delivery.SourcePrefix2 + strings.Repeat("c", 64), + Repository: repository, Directory: delivery.RootPath, Commit: commit, Node: "novox", At: time.Now().UTC(), Outcome: delivery.BuildDone}) + if err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(b.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil { + t.Fatal(err) + } +} + // compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the // move to it, in mesh-controller_composed. func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 737cee7..a3fd2f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -22,6 +22,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -58,7 +59,6 @@ func main() { var readBuckets = []string{ buscompose.ResolvedBucket, provider.AssignmentsBucket, - delivery.BuildsBucket, delivery.ControllerGrantsBucket, delivery.PartGrantsBucket, buscompose.LeaseBucket, @@ -76,6 +76,10 @@ var readBuckets = []string{ keptBucket, } +// readLogs are the logs of other parts the assigner reads, each named in its manifest's reads: the builder's log, whose +// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1). +var readLogs = []string{delivery.BuildLogRead} + // takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said // conditions go straight to the router or only as findings (issue 1952 WM6). var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState) @@ -204,6 +208,17 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any return out, nil } +// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's +// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source. +func startsWithout(name string) bool { + switch name { + case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, takeoverBucket, + delivery.AssignerPutsBucket: + return true + } + return slices.Contains(seatBuckets, name) || sourceOfGrants(name) +} + // sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the // controller's lease: neither stops the assigner when it is not there (delivery.GrantSource). func sourceOfGrants(name string) bool { @@ -239,7 +254,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro thisNode = account.Node logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) findings := findingsOf(ctx, kvs, logger.Printf) - return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger) + buildLog := func(handle func(stdio.LogEntry) error) error { + return said.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, handle) + } + return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger) } // findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the @@ -341,7 +359,7 @@ var watched = []string{ // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, buildLog logWatch, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -350,6 +368,22 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, default: } } + // The builder's log (ADR 0571 clause 1): every entry it keeps before the first pass, then each new one as it is appended; + // a module's newer done trunk build makes a pass, as a composition waiting on its commit is judged then. Where it cannot be + // read, a composition names no running reader of the manifests until the assigner starts again, and it is said. + if buildLog != nil { + var first atomic.Bool + if err := buildLog(func(le stdio.LogEntry) error { + if w.putBuilt(le.Value) && first.Load() { + wake() + } + return nil + }); err != nil { + logger.Printf("%s is not read (%v): no composition is judged against a running reader of the manifests until the assigner starts again", + delivery.BuildLogRead, err) + } + first.Store(true) + } errs := make(chan error, len(watched)+2) settled := make(chan struct{}, len(watched)+2) watch := func(name string, opts ...jetstream.WatchOpt) error { @@ -433,7 +467,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket || name == keptBucket || name == delivery.SeatHoldersBucket) { + if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) { continue // not on the bus at this start (open said so) } started++ diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index d2a1a83..54f98ec 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -42,8 +42,16 @@ func TestOpensWhatItsManifestNames(t *testing.T) { } var reads []string for _, r := range m.Reads { + if slices.Contains(readLogs, r) { + continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket + } reads = append(reads, strings.Replace(r, ".", "_", 1)) } + for _, l := range readLogs { + if !slices.Contains(m.Reads, l) { + t.Errorf("the program reads the log %s, the manifest does not name it", l) + } + } var states []string for _, s := range m.State { states = append(states, m.Module+"_"+s.Name) @@ -112,7 +120,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}} kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote - go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0)) + go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0)) select { case key := <-wrote: if key != "want.ace.node" { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 175bf7a..1acdc2b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" @@ -51,6 +52,8 @@ type world struct { changes map[string]uint64 // meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry). meta map[string]entryMeta + // trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt). + trunkBuilt map[string]delivery.BuildLogEntry } // entryMeta is a value's revision in its bucket and when it was written. @@ -68,7 +71,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, - meta: map[string]entryMeta{}} + meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -666,32 +669,35 @@ func (w *world) changed(bucket, key string) uint64 { return w.changes[bucket+"|"+key] } -// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's -// build record (`mesh-controller_builds` `.build.`), distinct and sorted: the commits a reader of the -// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out. -func (w *world) runningCommits(ctx context.Context, get getter, module string) []string { - _, all := w.assigned() - seen := map[string]bool{} - for _, a := range all { - if a.Module != module || a.Build == "" { - continue - } - value, err := get(ctx, delivery.BuildKey(module, a.Build)) - if err != nil { - continue - } - var rec delivery.BuildRecord - if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 { - continue - } - seen[rec.Commit] = true +// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch). +type logWatch func(handle func(stdio.LogEntry) error) error + +// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the +// module's last, becomes the module's running commit. It answers whether the commit changed. +func (w *world) putBuilt(raw []byte) bool { + e, err := delivery.DecodeBuildLogEntry(raw) + if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 { + return false } - out := make([]string, 0, len(seen)) - for c := range seen { - out = append(out, c) + w.mu.Lock() + defer w.mu.Unlock() + have, ok := w.trunkBuilt[e.Module] + if ok && !e.At.After(have.At) { + return false } - sort.Strings(out) - return out + w.trunkBuilt[e.Module] = e + return !ok || have.Commit != e.Commit +} + +// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a +// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build. +func (w *world) runningCommits(module string) []string { + w.mu.Lock() + defer w.mu.Unlock() + if e, ok := w.trunkBuilt[module]; ok { + return []string{e.Commit} + } + return nil } // keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5). diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 0744ac9..df09256 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index ecec070..07e0ff7 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121 h1:OsA1v6n736a7RqEV8Nkip8pf5ILMbzj0PWuKqcBOhK0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220947-f9a496d3e121/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index b939b72..f722280 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -33,8 +33,8 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", - "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", + want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 9dae8b6..91ad85a 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -31,10 +31,10 @@ "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", - "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", From a54d1acf45505533f526583fbc6f2f0cf195334a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:48:14 +0100 Subject: [PATCH 29/35] =?UTF-8?q?View:=20answer=20builder=5Fhistory=20from?= =?UTF-8?q?=20the=20builder's=20log,=20each=20build's=20end=20with=20its?= =?UTF-8?q?=20duration=20from=20its=20start=20(issue=201955,=20ADR=200571?= =?UTF-8?q?=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-view-tools/main.go | 24 +++++- .../cmd/mesh-delivery-view-tools/main_test.go | 23 +++++ modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/read.go | 85 +++++++++++++++++++ modules/mesh-delivery-view/manifest_test.go | 4 +- modules/mesh-delivery-view/module.json | 2 + 7 files changed, 138 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index e0bbef6..2af9092 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -145,6 +145,27 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { limit, _ := args["limit"].(float64) return r.BuilderBuilds(module, state, int(limit)) }}, + {Name: view.BuildHistoryTool, + Description: "Every build that ended since a time (default the last 24 hours), oldest first, from the builder's log build-log (kept 30 " + + "days): its module, fingerprint, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + + "queued), the manifest, the bases it was built against, each artifact made, and seconds, how long it took from its start (absent " + + "when the start is older than since). With module, that module's builds only; more says the log held more: ask again from the " + + "last build's time. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's builds"}, + "since": map[string]any{"type": "string", "description": "RFC 3339 time; default 24 hours ago"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + since := time.Now().Add(-view.DefaultHistorySince) + if raw, _ := args["since"].(string); strings.TrimSpace(raw) != "" { + at, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)) + if err != nil { + return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", raw, err) + } + since = at + } + return r.BuildHistory(module, since) + }}, {Name: view.BuilderBuildReadTool, Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " + "at the fingerprint the indexer holds for it now, joined through modules.; with commit, only if the indexer holds the module at that commit (a module's value names the commit its fingerprint first appeared at, and stays there through merges that leave the module alone). Read-only.", @@ -264,7 +285,8 @@ func main() { ConditionsState: read(stdio.State(view.StatusConditionsState)), AttemptsState: read(stdio.State(view.StatusAttemptsState)), PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState)), PredefinedState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerPredefinedState)), - ConfigCandidatesState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState))} + ConfigCandidatesState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState)), + BuildLog: stdio.Log(delivery.BuildLogRead).Since} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index b103e85..0e230ce 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-view/internal/view" @@ -177,6 +178,28 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) { t.Errorf("%s filter = %+v, want %s", args, got, want) } } + // 1955/L7 (ADR 0571 clause 1): builder_history answers each end from the builder's log with its duration from its start, + // the start of a later build of the same fingerprint not counted, and narrows by module through the log's key. + t0 := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + entry := func(event string, at time.Duration) []byte { + e := delivery.BuildLogEntry{Event: event, Module: "netcheck", FP: fp2A, Repository: "novox/mesh-catalog", Directory: "modules/netcheck", + Commit: strings.Repeat("7", 40), Node: "ace", At: t0.Add(at)} + if event == delivery.BuildEndedEvent { + e.Outcome = delivery.BuildDone + } + raw, _ := delivery.EncodeBuildLogEntry(e) + return raw + } + var asked string + r.BuildLog = func(at time.Time, keys string, limit int) (*stdio.LogPage, error) { + asked = keys + return &stdio.LogPage{Entries: []stdio.LogEntry{{Sequence: 1, Value: entry(delivery.BuildStartedEvent, 0)}, + {Sequence: 2, Value: entry(delivery.BuildEndedEvent, 90*time.Second)}, {Sequence: 3, Value: entry(delivery.BuildStartedEvent, time.Hour)}}}, nil + } + out, err = run(t, r, view.BuildHistoryTool, map[string]any{"module": "netcheck", "since": t0.Format(time.RFC3339)}) + if h, _ := out.(view.HistoryAnswer); err != nil || len(h.Builds) != 1 || h.Builds[0].Seconds != 90 || h.Builds[0].Outcome != delivery.BuildDone || asked != "netcheck" || h.More { + t.Errorf("history = %+v, %v (keys %q), want the one end at 90 s", out, err, asked) + } } // 1723/AC2: builder_build answers one key by module and fp2, and by module alone through the indexer's fp2 (joined at its commit). diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 96438b1..6340773 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 685613a..07e0ff7 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index cc9b8f8..fc9ac7c 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -52,8 +53,13 @@ type Reader struct { ConditionsState, AttemptsState, PresenceState, StuckState StateRead // PredefinedState and ConfigCandidatesState are the indexer's control records (ADR 0565 §2, §15); unset reads none. PredefinedState, ConfigCandidatesState StateRead + // BuildLog reads the builder's log from a time (stdio.Log(delivery.BuildLogRead).Since); unset reads none. + BuildLog LogSince } +// LogSince is one page of a log from a time on, at most limit entries of keys (stdio.KeptLog.Since). +type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) + // The read tools of the indexer's control records (ADR 0565 §15): the indexer serves no tool of its own (ADR 0437 §4). const ( PredefinedReadTool = "indexer_predefined" @@ -782,3 +788,82 @@ func (r Reader) Trusts() (TrustsAnswer, error) { } return out, nil } + +// BuildHistoryTool is the view's tool of the builds that ended, read from the builder's log (novox/hq ADR 0571 clause 1). +const BuildHistoryTool = "builder_history" + +// The bounds of builder_history: how far back it reads when no since is given, and the entries it reads of the log. +const ( + DefaultHistorySince = 24 * time.Hour + MaxHistoryEntries = 1000 +) + +// HistoryRow is one ended build of the builder's log, with how long it took from its start (none when its start is not in +// what was read). +type HistoryRow struct { + delivery.BuildLogEntry + Seconds float64 `json:"seconds,omitempty"` +} + +// HistoryAnswer is builder_history's answer: every build that ended since, oldest first, and More when the log held more +// entries than one read takes (ask again from the last build's time). +type HistoryAnswer struct { + Since time.Time `json:"since"` + Builds []HistoryRow `json:"builds"` + More bool `json:"more,omitempty"` +} + +// BuildHistory is every build of module (every module when empty) that ended at or after since, oldest first, each with its +// duration from the start the log holds of the same module and fingerprint before it (delivery.BuildDuration). An entry that +// does not read is refused, naming its sequence. +func (r Reader) BuildHistory(module string, since time.Time) (HistoryAnswer, error) { + out := HistoryAnswer{Since: since.UTC(), Builds: []HistoryRow{}} + if r.BuildLog == nil { + return out, fmt.Errorf("the builder's log cannot be read here") + } + keys := "" + if module = strings.TrimSpace(module); module != "" { + k, err := delivery.BuildLogKey(module) + if err != nil { + return out, err + } + keys = k + } + page, err := r.BuildLog(since, keys, MaxHistoryEntries) + if err != nil { + return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuildLogRead, since.Format(time.RFC3339), err) + } + if page == nil { + return out, nil + } + var entries []delivery.BuildLogEntry + for _, le := range page.Entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuildLogRead, le.Sequence, err) + } + entries = append(entries, e) + } + starts := map[string][]delivery.BuildLogEntry{} + for _, e := range entries { + if e.Event == delivery.BuildStartedEvent { + starts[e.Module+" "+e.FP] = append(starts[e.Module+" "+e.FP], e) + } + } + for _, e := range delivery.EndedSince(entries, since) { + row := HistoryRow{BuildLogEntry: e} + // the newest start of the same build at or before its end: a fingerprint built again later starts again after it + var start delivery.BuildLogEntry + for _, s := range starts[e.Module+" "+e.FP] { + if !s.At.After(e.At) && s.At.After(start.At) { + start = s + } + } + if d, ok := delivery.BuildDuration(start, e); ok { + row.Seconds = d.Seconds() + } + out.Builds = append(out.Builds, row) + } + out.More = page.Next != 0 + return out, nil +} diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 4c36dc6..72c912c 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 4d27585..6f16694 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -13,6 +13,7 @@ "indexer_config_candidates", "builder_build", "builder_builds", + "builder_history", "assigner_wants", "assigner_want", "assigner_differences", @@ -45,6 +46,7 @@ "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", From e564eaf75f304e4d6920af2a3b23b48d7802845f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:51:48 +0100 Subject: [PATCH 30/35] Grants: write the hearings it withholds to its state withheld on change, so the self-check's probe D-heard reads them from the bus and not the controller's store (issue 1955) --- modules/mesh-grants/grants_test.go | 28 +++++ modules/mesh-grants/internal/grants/part.go | 14 ++- .../mesh-grants/internal/grants/withheld.go | 101 ++++++++++++++++++ modules/mesh-grants/module.json | 4 + 4 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-grants/internal/grants/withheld.go diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 53b8e9b..7a9725e 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -584,6 +584,34 @@ func TestGrants(t *testing.T) { } }) + t.Run("withheld: a hearing of a restricted stream in the tool runner is written on change only, and deleted once heard (issue 1955 L7)", func(t *testing.T) { + const mail = `{"module":"mesh-mail","version":"1","streams":[{"name":"mail","events":["mail.*.message.added"],"max-age-days":7,` + + `"grant":{"config":"grants","events":"mail.{key}.**"}}],"resources":[{"id":"state","type":"directory","mode":"0700","place":"."},` + + `{"id":"grants","trusted":true,"type":"file","path":"${dir:state}/grants.json","mode":"0600","merge":"json","takes-settings":["grants"],"content":"{\"grants\": {}}"}]}` + b := fixture() + b.manifestOf("mesh-mail", mail) + b.manifestOf("reader", `{"module":"reader","version":"1","consumes":["mesh-mail.mail.box.message.added"]}`) + b.assign("a", "mesh-mail") + b.assign("a", "reader") + b.put(grants.ReadResolved, "a.mesh-mail.grants", map[string]any{"node": "a", "module": "mesh-mail", "key": "grants", "value": map[string]any{"box": []string{"reader"}}}) + p := partOn(b) + res := p.Compose() + const key = "reader.mail.a" + var h grants.WithheldHearing + if res.Withheld == nil || !slices.Equal(res.Withheld.Written, []string{key}) || json.Unmarshal(b.own[grants.StateWithheld][key], &h) != nil || + h.Module != "reader" || h.Node != "a" || h.Stream != "MAIL" || h.Emitter != "mesh-mail" || !strings.Contains(h.Said, "runs-as") { + t.Fatalf("withheld = %+v, stored %+v (errors %v), want %s written naming the remedy", res.Withheld, h, res.Errors, key) + } + if again := p.Compose(); again.Withheld == nil || len(again.Withheld.Written) != 0 || again.Withheld.Withheld != 1 { + t.Errorf("a second pass wrote %+v, want nothing (on change only)", again.Withheld) + } + b.manifestOf("reader", `{"module":"reader","version":"1","runs-as":"reader","resources":[{"id":"account","type":"user","name":"reader","shell":"/usr/bin/nologin","home":"/var/lib/reader"}],`+ + `"own-secrets":{"broker":"/var/lib/r/broker"},"secrets-owner":"reader","consumes":["mesh-mail.mail.box.message.added"]}`) + if heard := p.Compose(); heard.Withheld == nil || !slices.Equal(heard.Withheld.Deleted, []string{key}) || b.own[grants.StateWithheld][key] != nil { + t.Errorf("once reader runs as its own account the pass did %+v (errors %v), want %s deleted", heard.Withheld, heard.Errors, key) + } + }) + t.Run("root-free: the controller's word fills the composition, failing closed (issue 1945)", func(t *testing.T) { const router = `{"module":"messenger","version":"1","runs-as":"messenger","resources":[{"id":"account","type":"user","name":"messenger","shell":"/usr/bin/nologin","home":"/var/lib/messenger"}],"own-secrets":{"broker":"/var/lib/m/broker"},"secrets-owner":"messenger",` + `"seats":[{"name":"operator-channel","scope":"mesh","serves":["open"]},{"name":"channel","scope":"mesh","kinded":true,"accepts":["show"]}],` + diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index ca23375..38da1f4 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -34,7 +34,7 @@ const ( ) // States is every state the part keeps. -var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList} +var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList, StateWithheld} // The events the part says. const ( @@ -125,9 +125,11 @@ type Result struct { // Accounts is what the pass wrote of the wanted accounts the vault mints from (ADR 0561 §2). Accounts *AccountsWritten `json:"accounts,omitempty"` // Grants is what the pass wrote of its grants (grants.go). - Grants *GrantsWritten `json:"grants,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + Grants *GrantsWritten `json:"grants,omitempty"` + // Withheld is what the pass wrote of the hearings it withholds (withheld.go). + Withheld *WithheldWritten `json:"withheld,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -279,6 +281,10 @@ func (p *Part) Compose() Result { if p.Store != nil { p.writeAccounts(&res, in, listRecs) } + // The hearings withheld of what runs, for the self-check's probe D-heard (withheld.go). + if p.Store != nil { + p.writeWithheld(&res, in, recs) + } list, n, leftOut, err := ComposeList(listRecs, in.Hashes) // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over // one that adds a permission only once the bus loaded a list that carries it (issue 774). diff --git a/modules/mesh-grants/internal/grants/withheld.go b/modules/mesh-grants/internal/grants/withheld.go new file mode 100644 index 0000000..6e7260b --- /dev/null +++ b/modules/mesh-grants/internal/grants/withheld.go @@ -0,0 +1,101 @@ +package grants + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The hearings the part withholds (novox/hq ADR 0376 §2; issue 1955 L7, the lead 2026-10-29): a module granted to hear a +// restricted stream that runs in its machine's tool runner (no runs-as) hears none of it, since nothing is composed for +// the account every session there shares. The part writes each such hearing to its state `withheld`, on change only, so the +// self-check's probe D-heard reads it from the bus in place of the controller's store. +// +// Layout: key `..` (the controller's condition id of it); value WithheldHearing. +const StateWithheld = "withheld" + +// WithheldHearing is one withheld hearing: the machine, the module, the restricted stream and its emitter, the plain +// words of it and the remedy (buscompose.Withheld.Said), and when the part first wrote it as it stands. +type WithheldHearing struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Stream string `json:"stream"` + Emitter string `json:"emitter"` + Said string `json:"said"` + At time.Time `json:"at"` +} + +// WithheldKey is a withheld hearing's key in StateWithheld. +func WithheldKey(w buscompose.Withheld) string { + return w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node +} + +// WithheldWritten is what one pass did to StateWithheld. +type WithheldWritten struct { + Withheld int `json:"withheld"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept is set when a read of the pass failed: no key of a hearing no longer withheld is deleted then. + Kept string `json:"kept,omitempty"` +} + +// writeWithheld writes each withheld hearing of what runs whose key differs (its At aside), and deletes the key of one no +// longer withheld when every input of the pass was read. +func (p *Part) writeWithheld(res *Result, in Inputs, recs buscompose.Records) { + now := buscompose.WithheldOf(recs) + done := &WithheldWritten{Withheld: len(now)} + res.Withheld = done + want := map[string]WithheldHearing{} + for _, w := range now { + want[WithheldKey(w)] = WithheldHearing{V: 1, Node: w.Node, Module: w.Module, Stream: w.Stream, Emitter: w.Emitter, Said: w.Said()} + } + for _, key := range sortedKeys(want) { + h := want[key] + raw, _, found, err := p.Store.Get(StateWithheld, key) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateWithheld, key, err)) + continue + } + if found { + var was WithheldHearing + if json.Unmarshal(raw, &was) == nil { + was.At = time.Time{} + if was == h { + continue + } + } + } + h.At = p.now() + data, _ := json.Marshal(h) + if _, err := p.Store.Put(StateWithheld, key, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateWithheld, key, err)) + continue + } + done.Written = append(done.Written, key) + } + held, err := p.Store.Keys(StateWithheld) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no hearing no longer withheld is deleted: %v", StateWithheld, err)) + return + } + if len(in.ReadFailures()) > 0 { + done.Kept = "a read of this pass failed, so no hearing no longer withheld is deleted" + return + } + sort.Strings(held) + for _, key := range held { + if _, still := want[key]; still { + continue + } + if err := p.Store.Delete(StateWithheld, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateWithheld, key, err)) + continue + } + done.Deleted = append(done.Deleted, key) + } +} diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 2d25d3a..822b9e3 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -78,6 +78,10 @@ { "name": "user-list", "history": 20 + }, + { + "name": "withheld", + "history": 5 } ], "own-secrets": { From ba502479e8966899f4591dd2712300bdad603e0c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:56:21 +0100 Subject: [PATCH 31/35] =?UTF-8?q?Builder:=20ask=20and=20hear=20the=20seat?= =?UTF-8?q?=20mesh-artifact-sweep,=20the=20name=20mesh-sdk=207bd0f6c=20giv?= =?UTF-8?q?es=20the=20store's=20sweep,=20so=20its=20held=20and=20artifacts?= =?UTF-8?q?-gone=20reach=20their=20holder=20(issue=201955,=20ADR=200571=20?= =?UTF-8?q?=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-builder/stored.go | 6 +++--- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-builder/manifest_test.go | 8 ++++---- modules/mesh-delivery-builder/module.json | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go index fab1825..2b0edb1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go @@ -20,8 +20,8 @@ import ( // consumer of artifacts-gone (mesh-controller internal/broker EventsStream). const eventsStream = "EVENTS" -// SeatHeld is the seat artifact-store's verb held (ADR 0571 §2). -const SeatHeld = "seat:" + catalogue.ArtifactStoreSeat + ".held" +// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2). +const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held" // toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call). type toolCaller func(key string, body any) (json.RawMessage, error) @@ -30,7 +30,7 @@ type toolCaller func(key string, body any) (json.RawMessage, error) // 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: // the key is then left done, and the controller's take-in still refuses a gone blob. // -// It asks the seat artifact-store's `held` (novox/hq ADR 0571 §2), and the controller's while that seat has no holder that +// It asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2), and the controller's while that seat has no holder that // answers (a named exception of clause 10, until the lease write). func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) { return func(_ context.Context, references []string) ([]string, error) { diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index ab1a8a6..29e4276 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 07e0ff7..17b37be 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 h1:LXFwlVh0fGcOTU41PNkZDknHF070F/mhVy8Te6wKr3Q= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 8914a77..ce7f764 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -107,14 +107,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. - // 1955/L7 (ADR 0571 §2): the seat artifact-store's held, the controller's only while the seat has no holder (clause 10). - if want := []string{"seat:artifact-store.held", "seat:mesh-controller.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { + // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's only while the seat has no holder (clause 10). + if want := []string{"seat:mesh-artifact-sweep.held", "seat:mesh-controller.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it // starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4). - // 1955/L7: and the seat artifact-store's, the controller's until the lease write (its take-in sweeps until then). - if want := []string{"artifact-store.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // 1955/L7: and the seat mesh-artifact-sweep's, the controller's until the lease write (its take-in sweeps until then). + if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 5606863..1631df0 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -14,13 +14,13 @@ "build.asked-again" ], "consumes": [ - "artifact-store.artifacts-gone", + "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone" ], "invokes": [ - "seat:artifact-store.held", + "seat:mesh-artifact-sweep.held", "seat:mesh-controller.held" ], "state": [ From 08ecf10ef3edb15d0f3acaea2edea120d01fc5bb Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:56:21 +0100 Subject: [PATCH 32/35] Grants: write withheld hearings in mesh-sdk's layout, so the self-check reads them by the sdk's names (issue 1955) --- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 +-- .../mesh-grants/internal/grants/withheld.go | 27 ++++++------------- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 361c8a5..040bbe2 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index e8df9ed..6f2151d 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 h1:LXFwlVh0fGcOTU41PNkZDknHF070F/mhVy8Te6wKr3Q= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/internal/grants/withheld.go b/modules/mesh-grants/internal/grants/withheld.go index 6e7260b..1c130cb 100644 --- a/modules/mesh-grants/internal/grants/withheld.go +++ b/modules/mesh-grants/internal/grants/withheld.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "sort" - "strings" "time" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -15,25 +14,15 @@ import ( // the account every session there shares. The part writes each such hearing to its state `withheld`, on change only, so the // self-check's probe D-heard reads it from the bus in place of the controller's store. // -// Layout: key `..` (the controller's condition id of it); value WithheldHearing. -const StateWithheld = "withheld" +// Layout (mesh-sdk buscompose): key `..` (the controller's condition id of it); value +// buscompose.WithheldHearing; readers name buscompose.GrantsWithheldRead. +const StateWithheld = buscompose.GrantsWithheldState -// WithheldHearing is one withheld hearing: the machine, the module, the restricted stream and its emitter, the plain -// words of it and the remedy (buscompose.Withheld.Said), and when the part first wrote it as it stands. -type WithheldHearing struct { - V int `json:"v"` - Node string `json:"node"` - Module string `json:"module"` - Stream string `json:"stream"` - Emitter string `json:"emitter"` - Said string `json:"said"` - At time.Time `json:"at"` -} +// WithheldHearing is one withheld hearing, as mesh-sdk lays it (buscompose.WithheldHearing). +type WithheldHearing = buscompose.WithheldHearing -// WithheldKey is a withheld hearing's key in StateWithheld. -func WithheldKey(w buscompose.Withheld) string { - return w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node -} +// WithheldKey is a withheld hearing's key in StateWithheld (buscompose.WithheldKey). +func WithheldKey(w buscompose.Withheld) string { return buscompose.WithheldKey(w) } // WithheldWritten is what one pass did to StateWithheld. type WithheldWritten struct { @@ -52,7 +41,7 @@ func (p *Part) writeWithheld(res *Result, in Inputs, recs buscompose.Records) { res.Withheld = done want := map[string]WithheldHearing{} for _, w := range now { - want[WithheldKey(w)] = WithheldHearing{V: 1, Node: w.Node, Module: w.Module, Stream: w.Stream, Emitter: w.Emitter, Said: w.Said()} + want[WithheldKey(w)] = WithheldHearing{V: buscompose.WithheldVersion, Node: w.Node, Module: w.Module, Stream: w.Stream, Emitter: w.Emitter, Said: w.Said()} } for _, key := range sortedKeys(want) { h := want[key] From ce7032c49b4b012bb71b430cb8ac2057e7057603 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:05:25 +0100 Subject: [PATCH 33/35] Pin mesh-sdk 007b6fd, the release's final head (issue 774) --- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 ++-- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 ++-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- modules/mesh-delivery-teller/go.mod | 2 +- modules/mesh-delivery-teller/go.sum | 4 ++-- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 ++-- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 ++-- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index df09256..84cad99 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 07e0ff7..22bcaae 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 29e4276..d26a563 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 17b37be..22bcaae 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 h1:LXFwlVh0fGcOTU41PNkZDknHF070F/mhVy8Te6wKr3Q= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 39195d1..247522d 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 685613a..22bcaae 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 418479d..3fe781a 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 685613a..22bcaae 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index e98c8bd..606f645 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 07e0ff7..22bcaae 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index 4135620..3ce54e0 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index 6245202..22bcaae 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 6340773..3b1d6e2 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 07e0ff7..22bcaae 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641 h1:nP/jrjm2Qt1wdmd1Bc+E6c4+PobAzW5o64SPArE+jaw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029222821-bb993dd60641/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 57dac09..714abc9 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index a9c7447..b30f892 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 h1:4qfwx2bv2vosFM6tEbBZUeaQ/pfRauXZuNI3whul30g= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 040bbe2..43047aa 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 6f2151d..9ac13fe 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2 h1:LXFwlVh0fGcOTU41PNkZDknHF070F/mhVy8Te6wKr3Q= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225351-7bd0f6c2bdf2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 7536d9d14e46756e6ee7f91e63e4e483da4515db Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:18:04 +0100 Subject: [PATCH 34/35] Pin mesh-sdk 6e4a301, the release's final head, which keeps the old provisioner client's needs grant for one release (issue 774) --- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 ++-- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 ++-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- modules/mesh-delivery-teller/go.mod | 2 +- modules/mesh-delivery-teller/go.sum | 4 ++-- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 ++-- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 ++-- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 84cad99..835c2f8 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index d26a563..ff23e12 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 247522d..1c57121 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 3fe781a..9541b68 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 606f645..45bd8a9 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index 3ce54e0..23adad1 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 3b1d6e2..445dcee 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 22bcaae..eb0efcb 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 714abc9..075e7c3 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index b30f892..c9a31c5 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 43047aa..0a5681f 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 9ac13fe..508cf24 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 8edc83950ceab6bddd013933c852d2c8281f9a86 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 01:12:57 +0100 Subject: [PATCH 35/35] Pin mesh-sdk 2da0de7, the release's sdk with main merged in (issue 774) --- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 ++-- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 ++-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- modules/mesh-delivery-teller/go.mod | 2 +- modules/mesh-delivery-teller/go.sum | 4 ++-- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 ++-- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 4 ++-- modules/mesh-grants/go.mod | 2 +- modules/mesh-grants/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 835c2f8..0ed6939 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index ff23e12..e9d789c 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 1c57121..2cb4fa6 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 9541b68..3889df4 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 45bd8a9..bf49249 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index 23adad1..0aef26d 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 445dcee..3e7e9f0 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index eb0efcb..a2b8e01 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 075e7c3..ea23a95 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index c9a31c5..cfa6f9b 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 0a5681f..93d9d03 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 508cf24..05084f7 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=