diff --git a/modules/assignments-readers b/modules/assignments-readers new file mode 100644 index 0000000..436863a --- /dev/null +++ b/modules/assignments-readers @@ -0,0 +1,7 @@ +# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with +# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry +# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's +# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here +# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it. +mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after +mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go index 6fbe713..2ceefe7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go @@ -119,11 +119,11 @@ func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Cha if h := ch.Members[repository]; h != "" { commits, member = []string{h}, true } else { - commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module) + commits = p.w.runningCommits(module) } if len(commits) == 0 { - notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+ - "commit on record, and the group holds no member of "+repository) + notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+ + "trunk build of it, and the group holds no member of "+repository) continue } for _, commit := range commits { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go index 8bc0662..43030fa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go @@ -106,7 +106,7 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) { b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`)) running := strings.Repeat("3", 40) b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`)) - b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`)) + b.built(t, "mesh-controller", "novox/mesh-controller", running) sdkHead := strings.Repeat("4", 40) heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}} hash, _ := delivery.HeadsHash(heads) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1323409..5cce388 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -21,6 +21,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose/judge" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -82,6 +83,11 @@ func startBed(t *testing.T, leaveOut string) *bed { } b.buckets[name] = kv } + // The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog), + Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { + t.Fatal(err) + } if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { t.Fatal(err) } @@ -177,8 +183,8 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) { // 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket. func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { for _, missing := range readBuckets { - if missing == delivery.PlacementBucket { - continue // it starts without it, below + if startsWithout(missing) { + continue // it starts without it (the placement and the grants' source: main_test.go; the puts: below) } b := startBed(t, missing) select { @@ -264,6 +270,19 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { } } +// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does. +func (b *bed) built(t *testing.T, module, repository, commit string) { + t.Helper() + raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: delivery.SourcePrefix2 + strings.Repeat("c", 64), + Repository: repository, Directory: delivery.RootPath, Commit: commit, Node: "novox", At: time.Now().UTC(), Outcome: delivery.BuildDone}) + if err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(b.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil { + t.Fatal(err) + } +} + // compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the // move to it, in mesh-controller_composed. func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 54791b2..a3fd2f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -8,25 +8,32 @@ package main import ( "context" + "errors" "fmt" "io" "log" "os" "os/signal" "slices" + "sync/atomic" "syscall" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" + + "mesh-delivery-assigner/internal/assign" ) func main() { @@ -52,8 +59,9 @@ func main() { var readBuckets = []string{ buscompose.ResolvedBucket, provider.AssignmentsBucket, - delivery.BuildsBucket, delivery.ControllerGrantsBucket, + delivery.PartGrantsBucket, + buscompose.LeaseBucket, delivery.ControllerComposedBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, @@ -63,8 +71,19 @@ var readBuckets = []string{ delivery.PresenceBucket, delivery.HaveBucket, delivery.StepsBucket(goBuildModule), + takeoverBucket, + provisions.ProvisionsBucket, + keptBucket, } +// readLogs are the logs of other parts the assigner reads, each named in its manifest's reads: the builder's log, whose +// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1). +var readLogs = []string{delivery.BuildLogRead} + +// takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said +// conditions go straight to the router or only as findings (issue 1952 WM6). +var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState) + // goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes // their steps in its steps state (novox/hq ADR 0482 clause 4). const goBuildModule = "mesh-build-go" @@ -73,7 +92,7 @@ const goBuildModule = "mesh-build-go" // (ADR 0567 §1), named after the assigner as every module state is. var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket, - delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket} + delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket} // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). @@ -83,7 +102,8 @@ var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposals // keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view // (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view // and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and -// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9). +// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by +// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3). var keptStates = []announce.State{ {Name: delivery.PlacementState, Readable: true}, {Name: delivery.PlacementProposalsState, Readable: true}, @@ -94,6 +114,9 @@ var keptStates = []announce.State{ {Name: "differences", Readable: true}, {Name: "puts", Readable: true}, {Name: "wants", Readable: true}, + {Name: health.FindingsState, Readable: true}, + {Name: delivery.AssignerPoliciesState, Readable: true}, + {Name: delivery.SeatHoldersState, Readable: true}, } // buckets is every bucket the assigner opened, by name. @@ -105,8 +128,33 @@ type buckets map[string]jetstream.KeyValue // without it and takes no put further until it starts again (ADR 0546). func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) { out := buckets{} + // The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names + // where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the + // controller holds it, as before. + away, why := false, "" + if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil { + logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err) + } else { + out[buscompose.LeaseBucket] = kv + e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey) + var raw []byte + if gerr == nil { + raw = e.Value() + } + if errors.Is(gerr, jetstream.ErrKeyNotFound) { + gerr = nil + } + away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr) + } for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { + if name == buscompose.LeaseBucket { + continue + } kv, err := js.KeyValue(ctx, name) + if name == provider.AssignmentsBucket && err != nil && away { + logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why) + continue + } if name == delivery.AssignerPutsBucket && err != nil { logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err) continue @@ -117,6 +165,41 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == delivery.SeatHoldersBucket && err != nil { + // The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who + // holds which seat here, and the controller's seats still answer. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } + if name == keptBucket && err != nil { + // mesh-config's held values (ADR 0565 §5): without them a composition lays none, said in the journal. + logf("the bucket %s cannot be opened (%v): no held value is laid on a composition until it is bound", name, err) + continue + } + if name == provisions.ProvisionsBucket && err != nil { + // The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made; + // until then no want waits for a provision, as before. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } + if name == assign.PoliciesBucket && err != nil { + // The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no + // person's policy is read, so the controller's beside each composition holds, as before. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } + if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil { + // New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the + // router and kept as no finding, as before. + logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err) + continue + } + if sourceOfGrants(name) && err != nil { + // The grants part's grants and the controller's lease choose where grants are read from (issue 774): without + // them the controller's grants are read, as before the lease is handed away. + logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err) + continue + } if err != nil { return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) } @@ -125,6 +208,23 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any return out, nil } +// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's +// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source. +func startsWithout(name string) bool { + switch name { + case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, takeoverBucket, + delivery.AssignerPutsBucket: + return true + } + return slices.Contains(seatBuckets, name) || sourceOfGrants(name) +} + +// sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the +// controller's lease: neither stops the assigner when it is not there (delivery.GrantSource). +func sourceOfGrants(name string) bool { + return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket +} + // run is the program: it returns when ctx ends or the bus cannot be used. func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags) @@ -153,7 +253,71 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } thisNode = account.Node logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) - return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger) + findings := findingsOf(ctx, kvs, logger.Printf) + buildLog := func(handle func(stdio.LogEntry) error) error { + return said.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, handle) + } + return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger) +} + +// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the +// condition keeper's take-over key, watched, so a condition is said straight to the router only until the take-over. A +// bucket it cannot open is said: without its findings the conditions are said straight, as before; without the take-over +// key they are said straight too, the controller being followed. +func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings { + var f assign.Findings + if kv := kvs[assign.FindingsBucket]; kv != nil { + f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err } + f.Delete = func(key string) error { return kv.Delete(ctx, key) } + f.Keys = func() ([]string, error) { + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err + } + } + var keeper atomic.Bool + f.ToKeeper = keeper.Load + takeover := takeoverBucket + kv := kvs[takeover] + if kv == nil { + return f + } + w, err := kv.Watch(ctx, health.TakeoverKey) + if err != nil { + process.NotWatched(logf, takeover, err, "the controller is followed: the conditions are said straight to the router") + return f + } + go func() { + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + logf("the watch of %s ended: the last source it named is followed until the assigner starts again", takeover) + return + } + if e == nil { + continue + } + var raw []byte + if e.Operation() == jetstream.KeyValuePut { + raw = e.Value() + } + t, err := health.DecodeTakeover(raw, raw != nil) + if err != nil { + logf("the take-over key cannot be read (%v): the controller is followed", err) + } + if was := keeper.Swap(t.Follow == health.FollowKeeper); was != keeper.Load() { + logf("the assigner's conditions are said by %s from now (the take-over key follows %s)", t.Seat(), t.Follow) + } + } + } + }() + return f } // thisNode is the node this process runs on, as its credential names it: said with each of the seat's events. @@ -170,6 +334,8 @@ var watched = []string{ buscompose.ResolvedBucket, provider.AssignmentsBucket, delivery.ControllerGrantsBucket, + delivery.PartGrantsBucket, + buscompose.LeaseBucket, delivery.ControllerComposedBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, @@ -185,11 +351,15 @@ var watched = []string{ delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket, + assign.PoliciesBucket, + provisions.ProvisionsBucket, + keptBucket, + delivery.SeatHoldersBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, buildLog logWatch, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -198,6 +368,22 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, default: } } + // The builder's log (ADR 0571 clause 1): every entry it keeps before the first pass, then each new one as it is appended; + // a module's newer done trunk build makes a pass, as a composition waiting on its commit is judged then. Where it cannot be + // read, a composition names no running reader of the manifests until the assigner starts again, and it is said. + if buildLog != nil { + var first atomic.Bool + if err := buildLog(func(le stdio.LogEntry) error { + if w.putBuilt(le.Value) && first.Load() { + wake() + } + return nil + }); err != nil { + logger.Printf("%s is not read (%v): no composition is judged against a running reader of the manifests until the assigner starts again", + delivery.BuildLogRead, err) + } + first.Store(true) + } errs := make(chan error, len(watched)+2) settled := make(chan struct{}, len(watched)+2) watch := func(name string, opts ...jetstream.WatchOpt) error { @@ -233,7 +419,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if e.Operation() == jetstream.KeyValuePut { value = e.Value() } + before := w.grantsBucket() w.putEntry(name, e.Key(), value, e.Revision(), e.Created()) + if name == buscompose.LeaseBucket && w.grantsBucket() == before { + // The controller renews its lease every few seconds: only a change of where grants are read from + // makes a pass (delivery.GrantSource). + continue + } wake() } } @@ -269,13 +461,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, return } if bus != nil { - p.acts = newPutActs(bus, wake, logger.Printf) + p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway) } logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket) } started := 0 for _, name := range watched { - if kvs[name] == nil && slices.Contains(seatBuckets, name) { + if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) { continue // not on the bus at this start (open said so) } started++ @@ -297,13 +489,21 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, name string kv jetstream.KeyValue } - placementAppeared := make(chan appeared, len(seatBuckets)) + placementAppeared := make(chan appeared, len(seatBuckets)+2) for _, name := range seatBuckets { if kvs[name] == nil && js != nil { process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it", logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} }) } } + // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} { + if kvs[name] == nil && js != nil { + name := name + process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", + logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} }) + } + } takePlacement := func(a appeared) error { if kvs[a.name] != nil { return nil // bound already, by the other of its two events @@ -321,13 +521,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, case <-settled: } } - p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}} + p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}} p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf) if bus != nil && kvs[delivery.AssignerPutsBucket] != nil { - p.acts = newPutActs(bus, wake, logger.Printf) + p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway) } if bus != nil { - p.seat = newSeatActs(bus, wake, logger.Printf) + p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway) } if js != nil { p.publish = streamPublisher(ctx, js, thisNode) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 70572f5..54f98ec 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -1,6 +1,8 @@ package main import ( + "mesh-delivery-assigner/internal/assign" + "context" "errors" "fmt" @@ -17,8 +19,10 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provider" ) // 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them @@ -38,8 +42,16 @@ func TestOpensWhatItsManifestNames(t *testing.T) { } var reads []string for _, r := range m.Reads { + if slices.Contains(readLogs, r) { + continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket + } reads = append(reads, strings.Replace(r, ".", "_", 1)) } + for _, l := range readLogs { + if !slices.Contains(m.Reads, l) { + t.Errorf("the program reads the log %s, the manifest does not name it", l) + } + } var states []string for _, s := range m.State { states = append(states, m.Module+"_"+s.Name) @@ -62,20 +74,30 @@ func TestOpensWhatItsManifestNames(t *testing.T) { } // ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens // the rest and starts, writing wants; any other bucket it cannot open stops the start, named. + // Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the + // assignments refused while the controller holds it stop the start, and once it is handed away they are not needed. + handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`) for _, c := range []struct { missing string err error starts bool + lease []byte + said string }{ - {delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true}, - {delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true}, - {delivery.AssignerPutsBucket, context.DeadlineExceeded, true}, - {delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false}, + {delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"}, + {delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"}, + {delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"}, + {delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""}, + // Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start. + {delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"}, + {buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"}, + {provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""}, + {provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"}, } { var logged string - kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) }) + kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) }) switch { - case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")): + case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)): t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged) case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err) @@ -98,7 +120,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}} kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote - go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0)) + go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0)) select { case key := <-wrote: if key != "want.ace.node" { @@ -114,13 +136,18 @@ type refusingJS struct { jetstream.JetStream missing string err error + lease []byte // the controller's lease, when it holds one } func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { if name == r.missing { return nil, r.err } - return &memKV{values: map[string][]byte{}}, nil + kv := &memKV{values: map[string][]byte{}} + if name == buscompose.LeaseBucket && r.lease != nil { + kv.values[buscompose.ControllerLeaseKey] = r.lease + } + return kv, nil } // watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index ad767b0..93a28ff 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "maps" "slices" @@ -11,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "git.novox.be/novox/mesh-sdk/go/sdkversion" "mesh-delivery-assigner/internal/assign" @@ -27,6 +29,16 @@ type passer struct { // pass says a condition once and a restart says it again (the router keeps one per key). say func(asks.Condition) error said map[string]bool + // findings is the assigner's findings bucket (its said conditions and policy-split, ADR 0564 §2, ADR 0565 §8); splits the + // modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left. + findings assign.Findings + splits map[string]bool + // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). + failedProvisions map[string]bool + // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. + missing, missingSaid map[string]bool + // unplaced are the nodes whose unplaced finding this process wrote. + unplaced map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -129,6 +141,13 @@ func (p *passer) wants(ctx context.Context) { } } wanted := p.w.wantsOf() + chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + provisionsOf := p.provisionsOf() + missing := map[string]bool{} + defer func() { p.provisionsMissing(ctx, missing) }() + defer p.unplacedFindings() + defer p.seatHolders(ctx) + defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() puts := p.putsByModule() @@ -149,12 +168,17 @@ func (p *passer) wants(ctx context.Context) { stale := map[string]neitherOne{} left := map[string]string{} defer func() { p.sayNeither(neither, stale, left) }() + // Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774). + grants := p.w.grantsBucket() for name := range names { fp := trunk[name].FP2 name := name - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { + if placed[node] && p.w.handedAway() { + return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6) + } if placed[node] { // The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want // names the build alone, and the node judges the move itself. @@ -164,7 +188,7 @@ func (p *passer) wants(ctx context.Context) { }, GrantedAt: func(fp string) bool { key, err := delivery.ControllerGrantKey(name, fp) - return err == nil && p.w.has(delivery.ControllerGrantsBucket, key) + return err == nil && p.w.has(grants, key) }, Restored: func(node string) bool { key, err := delivery.RestoreAskKey(node, name, fp) @@ -180,8 +204,8 @@ func (p *passer) wants(ctx context.Context) { hold, _ := delivery.HoldAskKey(name, fp) m.Released = p.answeredWith(hold, "release") m.Kept = p.answeredWith(hold, "keep") - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant) - if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil { + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant) + if raw := p.w.value(grants, grant); raw != nil { if g, err := delivery.DecodeControllerGrant(raw); err == nil { m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } } @@ -195,6 +219,11 @@ func (p *passer) wants(ctx context.Context) { last[node] = p.w.lastHealthy(node, name, fp) } out := assign.Wants(m, nodes, wanted[name], last) + for _, node := range m.Nodes { + for _, prov := range m.MissingProvisions(node) { + missing[node+"."+name+"."+prov] = true + } + } p.yeses(ctx, name, m, wanted[name], &out) gatherNeither(neither, name, out) leftNeither(left, m, wanted[name], out) @@ -930,3 +959,275 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo } } } + +// chosenOf is a person's choice for module, nil when none. +func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy { + if c, ok := chosen[module]; ok { + return &c + } + return nil +} + +// policySplits writes the finding policy-split for each module whose policy at the assigner differs from one the controller +// writes beside a composition of it, and deletes it once they agree (ADR 0565 §8): written when a split begins, never again +// while it stands. The first pass after a start deletes every one a previous process left that no longer splits. +func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) { + if p.findings.Put == nil { + return + } + now := time.Now + if p.now != nil { + now = p.now + } + split := map[string]bool{} + theirs := map[string][]string{} + if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6) + theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket)) + } + for _, s := range assign.Splits(manifests, chosen, theirs) { + split[s.Module] = true + if p.splits[s.Module] { + continue + } + value, err := assign.PolicySplitFinding(s, now()) + if err == nil { + err = p.findings.Put(assign.PolicySplitKey(s.Module), value) + } + if err != nil { + p.logf("the policy split of %s is not written: %v", s.Module, err) + delete(split, s.Module) + } + } + gone := p.splits + if gone == nil && p.findings.Keys != nil { + gone = map[string]bool{} + if keys, err := p.findings.Keys(); err == nil { + for _, k := range keys { + if m, ok := strings.CutSuffix(strings.TrimPrefix(k, "module."), "."+assign.KindPolicySplit); ok && strings.HasPrefix(k, "module.") { + gone[m] = true + } + } + } + } + for m := range gone { + if !split[m] { + if err := p.findings.Delete(assign.PolicySplitKey(m)); err != nil { + p.logf("the policy split of %s is not deleted: %v", m, err) + split[m] = true // tried again at the next pass + } + } + } + p.splits = split +} + +// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat +// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed +// provision is also said as the assigner's finding `provider..provision-failed` (a waiting one is the watchman's +// needs-operator), deleted once it is no longer failed. +func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision { + if p.kvs[provisions.ProvisionsBucket] == nil { + return func(string) func(string) map[string]provisions.Provision { return nil } + } + by := map[string]map[string]map[string]provisions.Provision{} // module, node, key + failed := map[string]provisions.Provision{} + for key, raw := range p.w.read(provisions.ProvisionsBucket) { + pr, err := provisions.Decode(raw) + if err != nil { + continue + } + node, module := pr.Consumer.Node, pr.Consumer.Module + if by[module] == nil { + by[module] = map[string]map[string]provisions.Provision{} + } + if by[module][node] == nil { + by[module][node] = map[string]provisions.Provision{} + } + by[module][node][key] = pr + if pr.State == provisions.Failed { + failed[key] = pr + } + } + p.provisionFindings(failed) + return func(module string) func(string) map[string]provisions.Provision { + return func(node string) map[string]provisions.Provision { return by[module][node] } + } +} + +// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed. +func (p *passer) provisionFindings(failed map[string]provisions.Provision) { + if p.findings.Put == nil { + return + } + now := time.Now + if p.now != nil { + now = p.now + } + for key, pr := range failed { + if p.failedProvisions[key] { + continue + } + value, err := assign.ProvisionFailedFinding(key, pr, now()) + if err == nil { + err = p.findings.Put(assign.ProvisionFailedKey(key), value) + } + if err != nil { + p.logf("the failed provision %s is not said: %v", key, err) + continue + } + if p.failedProvisions == nil { + p.failedProvisions = map[string]bool{} + } + p.failedProvisions[key] = true + } + for key := range p.failedProvisions { + if _, still := failed[key]; still { + continue + } + if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil { + p.logf("the failed provision %s is not cleared: %v", key, err) + continue + } + delete(p.failedProvisions, key) + } +} + +// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default +// when its manifest names none (ADR 0564 clause 13 (d)). +var MissingBound = delivery.DefaultDeliveryBound + +// missingPulse is the pulse key of a provision with no key, `provision-missing...`. +func missingPulse(key string) string { return "provision-missing." + key } + +// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is +// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding +// `provider....provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on +// the watch, is the pass that writes it: no timer. +func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) { + pulses := p.kvs[delivery.PlacementPulsesBucket] + if p.missing == nil { + p.missing = map[string]bool{} + } + for key := range missing { + switch { + case !p.missing[key]: + p.missing[key] = true + if pulses != nil { + p.pulse(ctx, missingPulse(key), MissingBound) + } + case pulses == nil || !p.pulsing(missingPulse(key)): + if p.missingSaid[key] || p.findings.Put == nil { + continue + } + value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock()) + if err == nil { + err = p.findings.Put(assign.ProvisionMissingKey(key), value) + } + if err != nil { + p.logf("the missing provision %s is not said: %v", key, err) + continue + } + if p.missingSaid == nil { + p.missingSaid = map[string]bool{} + } + p.missingSaid[key] = true + } + } + for key := range p.missing { + if missing[key] { + continue + } + delete(p.missing, key) + if pulses != nil && p.pulsing(missingPulse(key)) { + if err := pulses.Delete(ctx, missingPulse(key)); err == nil { + p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil) + } + } + if p.missingSaid[key] && p.findings.Delete != nil { + if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil { + p.logf("the missing provision %s is not cleared: %v", key, err) + p.missing[key] = true + continue + } + delete(p.missingSaid, key) + } + } +} + +// unplacedFindings writes, from the lease write, the finding `machine..unplaced` for each node that runs but no +// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed. +func (p *passer) unplacedFindings() { + if p.findings.Put == nil { + return + } + now := []string(p.w.unplacedNodes()) + is := map[string]bool{} + for _, node := range now { + is[node] = true + if p.unplaced[node] { + continue + } + value, err := assign.UnplacedFinding(node, p.clock()) + if err == nil { + err = p.findings.Put(assign.UnplacedKey(node), value) + } + if err != nil { + p.logf("the unplaced node %s is not said: %v", node, err) + continue + } + if p.unplaced == nil { + p.unplaced = map[string]bool{} + } + p.unplaced[node] = true + } + for node := range p.unplaced { + if is[node] { + continue + } + if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil { + delete(p.unplaced, node) + } + } +} + +// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per +// seat held once, ``, and one per node of a node seat, `.`, written only where it differs, and a key no +// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no +// timer writes it. +func (p *passer) seatHolders(ctx context.Context) { + kv := p.kvs[delivery.SeatHoldersBucket] + if kv == nil { + return + } + placements := map[string]delivery.Placement{} + for key, value := range p.w.read(delivery.PlacementBucket) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + continue + } + if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node { + placements[node] = pl + } + } + if len(placements) == 0 { + return // no placement read yet: nothing is known of who holds what, and nothing is deleted + } + want := delivery.SeatHoldersOf(placements) + have := p.w.read(delivery.SeatHoldersBucket) + for key, sh := range want { + value, err := json.Marshal(sh) + if err != nil || string(have[key]) == string(value) { + continue + } + p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil }) + } + for key := range have { + if _, ok := want[key]; ok { + continue + } + if err := kv.Delete(ctx, key); err != nil { + p.logf("the seat holders %s are not deleted: %v", key, err) + continue + } + p.w.put(delivery.SeatHoldersBucket, key, nil) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index e584c94..affbf2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provider" @@ -31,6 +32,11 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) return uint64(len(m.values)), nil } +func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + m.values[key] = value + return uint64(len(m.values)), nil +} + func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { delete(m.values, key) return nil @@ -135,6 +141,9 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin type memStore map[string]*memKV func (s memStore) get(state, key string) ([]byte, uint64, bool, error) { + if s[state] == nil { + return nil, 0, false, nil + } v, ok := s[state].values[key] return v, uint64(len(v)), ok, nil } @@ -183,7 +192,7 @@ func putOfTheBus(t *testing.T) { p := newPasser(kvs, now) bus := &fakeBus{calls: make(chan string, 8)} woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done - p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}) + p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil) p.acts.snapshot = func(context.Context, string, string) (string, error) { return "novox's restore point of nats (streams) taken 22:01", nil } @@ -284,7 +293,7 @@ func putOfTheBus(t *testing.T) { if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold { t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err) } - if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) { + if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat t.Errorf("the warrant was recorded as %q", call) } world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"]) @@ -873,6 +882,24 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) { if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" { t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread) } + + // Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules + // run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of + // the node. A lease that goes back to a controller brings the comparison back. + place("n2", "d") + assigned("g14", "e") + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)) + got, all := p.w.assigned() + if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 { + t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all) + } + if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 { + t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:]) + } + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`)) + if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] { + t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen) + } } // 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 7549bb9..2def647 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "mesh-delivery-assigner/internal/assign" ) @@ -46,14 +47,16 @@ type putActs struct { snapped map[string]string snapNo map[string]string warranted map[string]bool + // rec records the put's warrant in the hand-act log (ADR 0565 §11). + rec handacts.Recorder warrantNo map[string]string said map[string]string // snapshot takes the bus's snapshot on node; a variable so a test takes none. snapshot func(ctx context.Context, node, module string) (string, error) } -func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs { - a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{}, +func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs { + a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{}, said: map[string]string{}} a.snapshot = a.snapshotTheBus return a @@ -183,7 +186,10 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo w.Composed, w.Granted = p.composedFor(node, module, put.FP) if d := w.Composed; d != nil { policy := d.Policy - if policy == "" { + chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + if c, ok := chosen[module]; ok { + policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7) + } else if policy == "" { policy, _, _ = catalogue.DerivedUpgrade(manifest) } w.Recorded = policy == catalogue.PolicyRecord @@ -213,8 +219,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo // composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one // place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and -// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them -// from their new owner and nothing else changes. +// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket). func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) { var out *delivery.ComposedDigest if d, ok := p.w.composed(node, module, fp); ok { @@ -222,7 +227,7 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, } granted := false if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil { - if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil { + if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil { if g, err := delivery.DecodeControllerGrant(raw); err == nil { granted = g.NamesNode(node, module) } @@ -382,8 +387,9 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri } } -// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the -// router's record of the teller's ask says who chose and how (ADR 0274). +// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and +// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's +// record of the teller's ask says who chose and how (ADR 0274). func (a *putActs) warrant(id, ask string) { a.mu.Lock() if a.warranted[id] { @@ -393,10 +399,10 @@ func (a *putActs) warrant(id, ask string) { a.warranted[id] = true a.mu.Unlock() go func() { - if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil { + if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil { a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err) a.mu.Lock() - a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err) + a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err) a.mu.Unlock() a.wake() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 85d363d..7ba0b4d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/compose/judge" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" @@ -67,6 +68,8 @@ type seatActs struct { export map[string]exportCheck // calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to. calls map[string]string + // rec records the seat's acts in the hand-act log (ADR 0565 §11). + rec handacts.Recorder } // exportCheck is the export read again for one proposal: running, its answer, or why it was not read. @@ -76,8 +79,9 @@ type exportCheck struct { err string } -func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs { - return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}} +func newSeatActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *seatActs { + return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}, + rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}} } // exportOf is the export read again for a proposal, asked once in the background. @@ -114,26 +118,31 @@ func (a *seatActs) forget(id string) { a.mu.Unlock() } -// record records once, in the controller's hand-act log, that placement was written on the operator's warrant (§5): the -// router's record of the teller's ask says who chose and how (ADR 0274). +// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's +// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and +// the controller's while that seat has no holder and the controller holds its lease (ADR 0565 §11, handacts.Recorder). func (a *seatActs) record(warrant string) { if a == nil || warrant == "" { return } ask, _, _ := strings.Cut(warrant, "@") go func() { - if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil { + if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil { a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err) + } else { + a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log) } }() } -// handAct records a hand-back in the controller's hand-act log (§7). +// handAct records a hand-back in the hand-act log (§7), as record does. func (a *seatActs) handAct(prop delivery.PlacementProposal) { go func() { - if _, err := a.bus.Call(handActCall, map[string]any{"what": "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "), - "why": prop.Why, "cause": "asked by " + prop.By + ", approved by the operator"}); err != nil { + if taken, err := a.rec.Record(handacts.Request{What: "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "), + Why: prop.Why, Cause: "asked by " + prop.By + ", approved by the operator"}); err != nil { a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err) + } else { + a.logf("the hand-back %s is recorded in the log of %s", prop.ID, taken.Log) } }() } @@ -524,7 +533,7 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, held, recorded := p.lays(ctx) again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, - Seed: prop.Seed, ExportDigest: prop.ExportDigest}) + Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()}) if len(findings) > 0 { p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; ")) return @@ -661,7 +670,7 @@ func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.An } p.seat.record(answer.Warrant) p.sayChanged(prop, answer, changes) - if p.seat != nil && !assign.Leased && !prop.Seed { + if p.seat != nil && !p.w.handedAway() && !prop.Seed { p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant)) } } @@ -729,7 +738,7 @@ func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal now := p.clock() seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now, Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{}, - Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)}) + Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()}) if len(findings) > 0 { p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; ")) return diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index b599301..46f1530 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -4,11 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strings" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -214,7 +216,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { } now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()}) if len(findings) > 0 { return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; ")) } @@ -298,7 +300,7 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) { now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{}, - Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)}) + Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()}) if len(findings) > 0 { return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; ")) } @@ -308,14 +310,20 @@ func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) { return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil } -// handActCall records a hand act in the controller's hand-act log. -const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act" +// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease +// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or +// unreadable, is not handed away. +func (s *seatTools) leased() bool { + raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err) + return away +} // doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back // proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys, // records the hand act and says placement-changed; every node returns to the controller's compositions. func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) { - if assign.Leased { + if s.leased() { return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to") } why := arg(args, "why") @@ -368,6 +376,9 @@ type historyEntry struct { // doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the // changes in order from the revisions the state keeps. func (s *seatTools) doPlacement(args map[string]any) (any, error) { + if arg(args, "seats") == "true" { + return s.seats(arg(args, "node")) + } all, revs, err := s.placements() if err != nil { return nil, err @@ -531,3 +542,32 @@ func (s *seatTools) tools() []stdio.Tool { } return out } + +// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders +// holds it; with node, the keys of seats held on that node. +func (s *seatTools) seats(node string) (any, error) { + keys, err := s.st.keys(delivery.SeatHoldersState) + if err != nil { + return nil, fmt.Errorf("seat-holders cannot be listed: %w", err) + } + sort.Strings(keys) + out := []delivery.SeatHolders{} + for _, key := range keys { + raw, _, found, err := s.st.get(delivery.SeatHoldersState, key) + if err != nil { + return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err) + } + if !found { + continue + } + var sh delivery.SeatHolders + if json.Unmarshal(raw, &sh) != nil { + continue + } + if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) { + continue + } + out = append(out, sh) + } + return map[string]any{"seats": out, "holds": len(keys)}, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index fb22c73..87c0017 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "slices" @@ -9,8 +10,12 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" + + "mesh-delivery-assigner/internal/assign" ) func (s memStore) del(state, key string) error { @@ -142,9 +147,250 @@ func TestTheSeatsVerbs(t *testing.T) { if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 { t.Errorf("hand-back = %+v, %v", got, err) } + // Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and + // there is no controller to hand placement back to. + st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}} + if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil { + t.Errorf("a hold after the lease write: %v", err) + } + if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") { + t.Errorf("a hand-back after the lease write: %v", err) + } if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil { t.Errorf("the seat's verbs as served: %+v", tools) } + t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) + t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) { + w := newWorld() + fp := delivery.SourcePrefix2 + strings.Repeat("2", 64) + ckey, _ := delivery.ComposedKey("ace", "web", fp) + cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither, + Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault}) + w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now()) + pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}}) + pkey, _ := delivery.PlacementKey("ace") + w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now()) + for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} { + hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest}) + hk, _ := delivery.HaveNodeKey(node) + w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now()) + } + if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 { + t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes()) + } + w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now()) + if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) { + t.Errorf("after the lease write, ace: %+v %v", d, ok) + } + if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) { + t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes()) + } + findings := map[string][]byte{} + p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + p.unplacedFindings() + if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 { + t.Fatalf("findings = %v", findings) + } + pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}) + pkey2, _ := delivery.PlacementKey("g14") + w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now()) + p.unplacedFindings() + if len(findings) != 0 { + t.Errorf("placed again: findings %v", findings) + } + }) + t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) { + writes := 0 + kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes} + p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}} + place := func(node string, holds ...delivery.Holding) { + pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds}) + if err != nil { + t.Fatal(err) + } + k, _ := delivery.PlacementKey(node) + p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now()) + } + place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"}) + place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"}) + ctx := context.Background() + p.seatHolders(ctx) + if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil { + t.Fatalf("%d writes: %v", writes, kv.values) + } + p.seatHolders(ctx) + if writes != 3 { + t.Errorf("an unchanged placement wrote %d keys", writes-3) + } + place("g14") // g14 holds nothing now + p.seatHolders(ctx) + if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil { + t.Errorf("after g14's change: %d writes, %v", writes, kv.values) + } + st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}} + out, err := newSeatTools(st, nil).seats("ace") + if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 { + t.Errorf("seats on ace = %+v, %v", out, err) + } + }) + t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { + findings := map[string][]byte{} + p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now()) + p.kvs[provisions.ProvisionsBucket] = &memKV{} + of := p.provisionsOf() + if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed { + t.Fatalf("provisions = %+v", got) + } + if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 { + t.Fatalf("findings = %v", findings) + } + provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now()) + p.provisionsOf() + if len(findings) != 0 { + t.Errorf("findings after provided = %v", findings) + } + // A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed. + ctx := context.Background() + p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}} + gone := map[string]bool{"ace.photos.s3": true} + p.provisionsMissing(ctx, gone) + if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 { + t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings) + } + p.provisionsMissing(ctx, gone) + if len(findings) != 0 { + t.Fatal("said before its bound") + } + p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it + p.provisionsMissing(ctx, gone) + if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok { + t.Fatalf("past its bound: findings %v", findings) + } + p.provisionsMissing(ctx, map[string]bool{}) + if len(findings) != 0 { + t.Errorf("keyed again: findings %v", findings) + } + }) +} + +// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is +// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back; +// the bus is refused; compare lists the modules the controller writes another policy for; the process writes policy-split +// for a split and deletes it once they agree; bus names the node behind the indexer's newest. +func wantsVerbs(t *testing.T) { + fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64) + st := memStore{} + for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} { + st[state] = &memKV{values: map[string][]byte{}} + } + manifest := func(name, extra string) { + raw := `{"module":"` + name + `","version":"1"` + extra + `}` + digest := delivery.ManifestDigest([]byte(raw)) + st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw}) + st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"}) + } + manifest("web", "") + manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`) + manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`) + for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} { + st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + } + composed := func(module, policy string) { + key, _ := delivery.ComposedKey("anchor", module, fpNew) + st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64), + Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"}) + } + composed("web", catalogue.PolicyRecord) + composed("photos", catalogue.PolicyRecord) + var asked []string + var said []string + w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key) + return json.RawMessage(`{"ok":true}`), nil + }, func(event, id string, body any) error { said = append(said, event+" "+id); return nil }) + policyOf := func(module string) policyRow { + out, err := w.doUpgrade(map[string]any{"module": module}, "op") + if err != nil { + t.Fatal(err) + } + return out.(map[string]any)["modules"].([]policyRow)[0] + } + if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { + t.Errorf("photos = %+v", p) + } + if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus { + t.Errorf("nats = %+v", p) + } + // compare: web rolls at the assigner and records at the controller. + out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op") + if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil { + t.Errorf("splits = %+v", splits) + } + // The process says the split as its finding, and deletes it once they agree. + findings := map[string][]byte{} + p := &passer{w: newWorld(), logf: func(string, ...any) {}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }, + Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}} + for k, v := range st[composedState].values { + p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now()) + } + man := map[string]catalogue.Manifest{} + for _, name := range []string{"web", "photos"} { + raw, _, _, _ := st.get(modulesState, name) + mod, _ := delivery.DecodeModule(raw) + mraw, _, _, _ := st.get(manifestsState, mod.Manifest) + kept, _ := delivery.DecodeManifest(mraw) + man[name], _ = catalogue.ParseManifest([]byte(kept.JSON)) + } + findings["module.gone.policy-split"] = []byte("{}") + p.policySplits(man, nil) + if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 { + t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings) + } + // a person's record for web: written, recorded, said; the split is gone. + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil { + t.Fatal(err) + } + chosen, _ := assign.DecodePolicies(st[policiesState].values) + if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") || + len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") { + t.Errorf("chosen %+v, asked %v, said %v", c, asked, said) + } + p.policySplits(man, chosen) + if len(findings) != 0 { + t.Errorf("findings after the choice = %v", findings) + } + if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil { + t.Error("the bus was given a policy") + } + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil { + t.Error("a policy without why was set") + } + if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 { + t.Errorf("default left %v (%v)", st[policiesState].values, err) + } + // bus: the node behind the indexer's newest. + b, err := w.doBus(nil) + if err != nil { + t.Fatal(err) + } + nodes := b.(map[string]any)["nodes"].([]busNode) + if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") { + t.Errorf("bus = %+v, asked %v", b, asked) + } + if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil { + t.Errorf("the seat's verbs as served: %+v", tools) + } } // readManifest is the module's manifest, read as the controller reads it. @@ -155,3 +401,14 @@ func readManifest() (catalogue.Manifest, error) { } return catalogue.ParseManifest(raw) } + +// countingKV is a memKV that counts its puts. +type countingKV struct { + memKV + writes *int +} + +func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) { + *c.writes++ + return c.memKV.Put(ctx, key, value) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 0e47bb4..74a5180 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -351,7 +351,12 @@ func serveTools() error { time.Sleep(wait) } }() - return stdio.Serve("", append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)) + tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...) + tools = append(tools, newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error { + // The seat's event, said on the seat: a claimed seat's event is never the module's own. + return stdio.SeatPublish(wantsEventSubject(e), b, id) + }).tools()...) + return stdio.Serve("", tools) } // shortFP is a fingerprint's first twelve hex digits. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go new file mode 100644 index 0000000..945a959 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go @@ -0,0 +1,300 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "mesh-delivery-assigner/internal/assign" +) + +// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the +// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies` +// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from +// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads +// `policies` when it judges a move and writes the finding policy-split while a split stands. + +// The states the verbs read, as the runtime names them. +const ( + policiesState = delivery.AssignerPoliciesState + manifestsState = delivery.IndexerModule + ".manifests" + composedState = "mesh-controller.composed" + // PolicyChangedEvent is the seat's event of a person's policy set or taken back. + PolicyChangedEvent = "policy-changed" + // busStepCause is the cause the bus's step is recorded under in the hand-act log. + busStepCause = "bus-upgrade" +) + +// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit. +type wantsTools struct { + st seatStore + ask func(key string, body any) (json.RawMessage, error) + emit func(event, id string, body any) error + now func() time.Time +} + +// wantsEventSubject is the subject of one of the seat's events. +func wantsEventSubject(event string) string { + return "mesh.seat." + catalogue.WantsSeat + ".event." + event +} + +func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools { + return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now} +} + +// leased says the controller's lease is handed away. +func (s *wantsTools) leased() bool { + raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err) + return away +} + +// values reads every key of a state. +func (s *wantsTools) values(state string) (map[string][]byte, error) { + keys, err := s.st.keys(state) + if err != nil { + return nil, fmt.Errorf("%s cannot be listed: %w", state, err) + } + out := map[string][]byte{} + for _, k := range keys { + raw, _, found, err := s.st.get(state, k) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err) + } + if found { + out[k] = raw + } + } + return out, nil +} + +// manifests is every module the indexer holds, with its trunk manifest and fingerprint. +func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) { + mods, err := s.values(modulesState) + if err != nil { + return nil, nil, err + } + out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{} + for name, raw := range mods { + m, err := delivery.DecodeModule(raw) + if err != nil { + continue + } + held[name] = m + if m.Manifest == "" { + continue + } + mraw, _, found, err := s.st.get(manifestsState, m.Manifest) + if err != nil || !found { + continue + } + kept, err := delivery.DecodeManifest(mraw) + if err != nil { + continue + } + if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil { + out[name] = man + } + } + return out, held, nil +} + +// policyRow is one module's policy as upgrade answers it. +type policyRow struct { + Module string `json:"module"` + Policy string `json:"policy"` + From string `json:"from"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` +} + +// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it +// back with default; with compare list the splits. +func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) { + manifests, _, err := s.manifests() + if err != nil { + return nil, err + } + raw, err := s.values(policiesState) + if err != nil { + return nil, err + } + chosen, unreadable := assign.DecodePolicies(raw) + if arg(args, "compare") == "true" { + composed, err := s.values(composedState) + if err != nil { + return nil, err + } + splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed)) + if splits == nil { + splits = []assign.Split{} + } + out := map[string]any{"splits": splits, "empty": len(splits) == 0} + // The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15). + if found, err := s.values(health.FindingsState); err != nil { + out["findings-unread"] = err.Error() + } else { + keys := []string{} + for k := range found { + keys = append(keys, k) + } + sort.Strings(keys) + out["findings"] = keys + } + return out, nil + } + module, policy := arg(args, "module"), arg(args, "policy") + if policy != "" { + return s.setPolicy(module, policy, arg(args, "why"), by, manifests) + } + var rows []policyRow + for name, m := range manifests { + if module != "" && name != module { + continue + } + p, from, why, who := assign.PolicyOf(m, chosen) + rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who}) + } + if module != "" && len(rows) == 0 { + return nil, fmt.Errorf("the indexer holds no manifest of %s", module) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module }) + return map[string]any{"modules": rows, "unreadable": unreadable}, nil +} + +// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log +// and says policy-changed. The bus is never rolled: a choice for it is refused. +func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) { + if module == "" { + return nil, errors.New("a policy is set for one module: name it") + } + m, ok := manifests[module] + if !ok { + return nil, fmt.Errorf("the indexer holds no manifest of %s", module) + } + if catalogue.ProvidesBus(m) { + return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module) + } + if why == "" { + return nil, errors.New("a person's policy says why") + } + if by == "" || by == stdio.Unproven { + return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller") + } + what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy) + if policy == "default" { + if err := s.st.del(policiesState, module); err != nil { + return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err) + } + } else { + p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()} + if err := p.Check(); err != nil { + return nil, err + } + raw, _ := json.Marshal(p) + if err := s.st.put(policiesState, module, raw); err != nil { + return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err) + } + } + rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased} + taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"}) + recorded := taken.Log + if rerr != nil { + recorded = "not recorded: " + rerr.Error() + } + body := map[string]any{"module": module, "policy": policy, "why": why, "by": by} + said := "" + if s.emit != nil { + id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10) + if err := s.emit(PolicyChangedEvent, id, body); err != nil { + said = "not said: " + err.Error() + } + } + return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil +} + +// busNode is one node's bus as bus answers it. +type busNode struct { + Node string `json:"node"` + FP string `json:"fp"` + Behind bool `json:"behind"` +} + +// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind, +// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only. +func (s *wantsTools) doBus(map[string]any) (any, error) { + manifests, held, err := s.manifests() + if err != nil { + return nil, err + } + bus := "" + for name, m := range manifests { + if catalogue.ProvidesBus(m) { + bus = name + } + } + if bus == "" { + return nil, errors.New("the indexer holds no module that provides the bus") + } + newest := held[bus].FP2 + haves, err := s.values(haveState) + if err != nil { + return nil, err + } + var nodes []busNode + for key, raw := range haves { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode || module != bus { + continue + } + h, err := delivery.DecodeHave(raw) + if err != nil || h.Phase == delivery.PhaseRemoved { + continue + } + nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest}) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node }) + out := map[string]any{"module": bus, "newest": newest, "nodes": nodes} + if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil { + out["steps-unread"] = err.Error() + } else { + out["steps"] = steps + } + puts, err := s.values(putsState) + if err != nil { + out["puts-unread"] = err.Error() + } else { + mine := map[string]json.RawMessage{} + for k, v := range puts { + if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus { + mine[k] = v + } + } + out["puts"] = mine + } + return out, nil +} + +// tools are the seat's verbs as the runtime serves them, each named ., with the seat's own schemas. +func (s *wantsTools) tools() []stdio.Tool { + seat, _ := catalogue.SeatNamed(catalogue.WantsSeat) + run := map[string]func(map[string]any) (any, error){ + "upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) }, + "bus": s.doBus, + } + var out []stdio.Tool + for _, v := range seat.Serves { + out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]}) + } + return out +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 3dba2bc..1acdc2b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" @@ -51,6 +52,8 @@ type world struct { changes map[string]uint64 // meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry). meta map[string]entryMeta + // trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt). + trunkBuilt map[string]delivery.BuildLogEntry } // entryMeta is a value's revision in its bucket and when it was written. @@ -68,7 +71,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, - meta: map[string]entryMeta{}} + meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -195,21 +198,38 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed // assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order // (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module. +// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6). func (w *world) assigned() (map[string][]string, map[string]assignment) { - byAssignment, all := w.assignedByController() + // A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh: + // the lease write's own check refuses that order (ADR 0547 §11). placed, seeded := w.placed() - both, _ := assign.Placed(byAssignment, placed, seeded) + byAssignment, all := w.assignedByController() + if away := w.handedAway(); away && seeded { + both, _ := assign.Placed(byAssignment, placed, seeded, away) + return both, map[string]assignment{} + } + both, _ := assign.Placed(byAssignment, placed, seeded, false) return both, all } -// placementDiffers is every pair the controller's assignments and the seat's placement do not both name. +// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once +// the controller's lease is handed away, since the placement is then the one source. func (w *world) placementDiffers() []assign.Pair { - byAssignment, _ := w.assignedByController() placed, seeded := w.placed() - _, differ := assign.Placed(byAssignment, placed, seeded) + byAssignment, _ := w.assignedByController() + _, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway()) return differ } +// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway, +// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its +// bucket not granted or not there), a controller's, expired or unreadable, is not handed away. +func (w *world) handedAway() bool { + raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey] + away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil) + return away +} + // placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key // that does not read places nothing on its node and is said by the indexer's own read tool. func (w *world) placed() (map[string][]string, bool) { @@ -415,6 +435,11 @@ func (r worldReader) Served(ctx context.Context, node, module string, keys []str func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) { ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}} + heads := map[string]string{} + for _, p := range c.Pulls { + heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head + } + ch.Held = assign.HeldOf(w.read(keptBucket), heads) definedIn := map[string]map[string]bool{} for _, p := range c.Pulls { if p.Repository == assign.SDKRepository { @@ -546,6 +571,16 @@ func (w *world) value(bucket, key string) []byte { return w.values[bucket][key] } +// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's +// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other +// case, a lease not held here included. +func (w *world) grantsBucket() string { + raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil) + bucket, _ := delivery.GrantSource(away) + return bucket +} + // has says whether a bucket holds a key now. func (w *world) has(bucket, key string) bool { w.mu.Lock() @@ -572,7 +607,17 @@ func (w *world) built(module, fp string) bool { // composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it // (mesh-controller_composed `..`), and whether it exists and can be read. +// +// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a +// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets +// no want (and the pass says so, unplacedFindings). func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) { + if w.handedAway() { + if !w.placedNodes()[node] { + return delivery.ComposedDigest{}, false + } + return w.ownComposed(node) + } key, err := delivery.ComposedKey(node, module, fp) if err != nil { return delivery.ComposedDigest{}, false @@ -590,6 +635,9 @@ func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool // nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `.node`). func (w *world) nodeComposed() map[string]delivery.ComposedDigest { out := map[string]delivery.ComposedDigest{} + if w.handedAway() { + return out // the controller composes no node from the lease write (ADR 0571 §6) + } for key, value := range w.read(delivery.ControllerComposedBucket) { node, _, _, isNode, err := delivery.ParseComposedKey(key) if err != nil || !isNode { @@ -621,29 +669,72 @@ func (w *world) changed(bucket, key string) uint64 { return w.changes[bucket+"|"+key] } -// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's -// build record (`mesh-controller_builds` `.build.`), distinct and sorted: the commits a reader of the -// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out. -func (w *world) runningCommits(ctx context.Context, get getter, module string) []string { - _, all := w.assigned() - seen := map[string]bool{} - for _, a := range all { - if a.Module != module || a.Build == "" { - continue - } - value, err := get(ctx, delivery.BuildKey(module, a.Build)) - if err != nil { - continue - } - var rec delivery.BuildRecord - if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 { - continue - } - seen[rec.Commit] = true +// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch). +type logWatch func(handle func(stdio.LogEntry) error) error + +// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the +// module's last, becomes the module's running commit. It answers whether the commit changed. +func (w *world) putBuilt(raw []byte) bool { + e, err := delivery.DecodeBuildLogEntry(raw) + if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 { + return false } - out := make([]string, 0, len(seen)) - for c := range seen { - out = append(out, c) + w.mu.Lock() + defer w.mu.Unlock() + have, ok := w.trunkBuilt[e.Module] + if ok && !e.At.After(have.At) { + return false + } + w.trunkBuilt[e.Module] = e + return !ok || have.Commit != e.Commit +} + +// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a +// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build. +func (w *world) runningCommits(module string) []string { + w.mu.Lock() + defer w.mu.Unlock() + if e, ok := w.trunkBuilt[module]; ok { + return []string{e.Commit} + } + return nil +} + +// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5). +var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState + +// ownComposed is a placed node's own composition, as the digest its own have names (`.node` of the haves, Composed): +// what rule 7 reads from the lease write. A node whose have names none is not composed yet. +func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) { + key, err := delivery.HaveNodeKey(node) + if err != nil { + return delivery.ComposedDigest{}, false + } + raw, ok := w.read(delivery.HaveBucket)[key] + if !ok { + return delivery.ComposedDigest{}, false + } + h, err := delivery.DecodeHaveNode(raw) + if err != nil || h.Composed == "" { + return delivery.ComposedDigest{}, false + } + return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither, + Words: "the node composes its own"}, true +} + +// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no +// want, since nothing composes them. +func (w *world) unplacedNodes() []string { + if !w.handedAway() { + return nil + } + placed := w.placedNodes() + var out []string + for key := range w.read(delivery.HaveBucket) { + node, _, isNode, err := delivery.ParseHaveKey(key) + if err == nil && isNode && !placed[node] { + out = append(out, node) + } } sort.Strings(out) return out diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index eb4fa5b..7ac1d63 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -6,7 +6,9 @@ import ( "slices" "strings" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-assigner/internal/assign" @@ -125,4 +127,40 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) { if got := w.changed(delivery.HaveBucket, "gone"); got != 0 { t.Errorf("changed = %d for a key never seen", got) } + // Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant + // only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it + // holds it, a grant only the controller holds being the one that counts instead. + fp := delivery.SourcePrefix2 + strings.Repeat("d", 64) + gkey, _ := delivery.ControllerGrantKey("web", fp) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}}) + for _, row := range []struct { + name, lease string + inPart, inController bool + bucket string + granted bool + }{ + {"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true}, + {"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false}, + {"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false}, + {"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true}, + {"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false}, + {"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false}, + } { + p := newPasser(buckets{}, time.Now()) + if row.lease != "" { + p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease)) + } + if row.inPart { + p.w.put(delivery.PartGrantsBucket, gkey, grant) + } + if row.inController { + p.w.put(delivery.ControllerGrantsBucket, gkey, grant) + } + if got := p.w.grantsBucket(); got != row.bucket { + t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket) + } + if _, granted := p.composedFor("ace", "web", fp); granted != row.granted { + t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted) + } + } } diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 7eb042d..0ed6939 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 9dde7f0..a2b8e01 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index 0f0fbdb..231ea07 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -1,12 +1,14 @@ package assign import ( + "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at @@ -73,7 +75,13 @@ func TestTheRestoreAskIsValid(t *testing.T) { // 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own // name (ADR 0437 clause 9), and is cleared under the same key. -func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) { +func TestTheAssignersConditions(t *testing.T) { + t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords) + t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy) + t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings) +} + +func heldConditionNamesModuleNodeAndWords(t *testing.T) { c := HeldCondition("mail", fNew, "ace", "connection refused", true, true) if _, err := asks.EncodeCondition(c); err != nil { t.Fatalf("the condition is refused: %v", err) @@ -87,7 +95,7 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) { } // 1906/H2: a hold lifted by itself clears the held condition under its key, saying why. -func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) { +func liftedHoldClearsItsConditionSayingWhy(t *testing.T) { c := LiftedCondition("mail", fNew) if _, err := asks.EncodeCondition(c); err != nil { t.Fatalf("the condition is refused: %v", err) @@ -97,6 +105,48 @@ func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) { } } +// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under +// `..`, written at the raise and deleted at the clear; it is said straight to the router while the take-over +// key follows the controller, and only as a finding from the take-over. +func saidConditionsAreFindings(t *testing.T) { + at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + for _, c := range []struct { + raised, cleared asks.Condition + key string + }{ + {HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"}, + {NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"}, + {NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"}, + {PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"}, + {SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"}, + {BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"}, + } { + puts, deletes, said := map[string]health.Finding{}, []string{}, 0 + keeper := false + f := Findings{Put: func(k string, v []byte) error { + got, err := health.DecodeFinding(v) + puts[k] = got + return err + }, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }} + say := f.Say(func(asks.Condition) error { said++; return nil }) + if err := say(c.raised); err != nil { + t.Fatalf("%s: %v", c.key, err) + } + got, ok := puts[c.key] + if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule || + got.Subject.Machine != c.raised.Subject.Machine || said != 1 { + t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said) + } + keeper = true + if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 { + t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err) + } + } + if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil { + t.Error("a condition not the assigner's is a finding") + } +} + // 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in // the view, and name the move and the node. func TestTheRecordAndBackupAsksAreValid(t *testing.T) { diff --git a/modules/mesh-delivery-assigner/internal/assign/compose.go b/modules/mesh-delivery-assigner/internal/assign/compose.go index 65c1481..35cb850 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose.go @@ -5,6 +5,7 @@ package assign import ( + "encoding/json" "fmt" "slices" "sort" @@ -59,6 +60,63 @@ type Change struct { // Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623). Digests map[string]string Members map[string]string + // Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR + // 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named. + Held []delivery.KeptValue +} + +// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another +// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key. +func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue { + var out []delivery.KeptValue + for _, raw := range kept { + var k delivery.KeptValue + if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" { + continue + } + if head, ok := heads[k.Pull]; ok && head == k.Head { + out = append(out, k) + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key + }) + return out +} + +// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers +// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the +// declaration refuses, or a key it does not declare, is a failure. +func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) { + for _, k := range held { + var v any + if err := json.Unmarshal(k.Value, &v); err != nil { + said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err)) + continue + } + d, declared := in.Catalogue[k.Module].Settings[k.Key] + if !declared { + fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull)) + continue + } + if problems := catalogue.ValueProblems(d, v); len(problems) > 0 { + fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module, + k.Node, k.Pull, strings.Join(problems, "; "))) + continue + } + at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node }) + if at < 0 { + said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node)) + continue + } + if in.Nodes[at].Settings == nil { + in.Nodes[at].Settings = map[string][]catalogue.Layer{} + } + in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}}) + said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved)) + } + return said, fails } // Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or @@ -171,6 +229,11 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit delete(changed, name) } after := input(changed, m.Assigned) + laid, heldFails := layHeld(&after, c.Held) + v.Warnings = append(v.Warnings, laid...) + for _, f := range heldFails { + failAt("", f) + } for _, node := range machines { _, before := compose.Resolve(trunk, node) _, now := compose.Resolve(after, node) diff --git a/modules/mesh-delivery-assigner/internal/assign/compose_test.go b/modules/mesh-delivery-assigner/internal/assign/compose_test.go index 96c46ab..86e217b 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose_test.go @@ -1,7 +1,9 @@ package assign import ( + "encoding/json" "slices" + "strconv" "strings" "testing" @@ -38,6 +40,32 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) { if _, err := delivery.EncodeCompositionVerdict(v); err != nil { t.Errorf("the verdict cannot be written: %v", err) } + // 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged + // against the head's declaration; one of another pull request or another head is not laid. + web, err := catalogue.ParseManifest([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`)) + if err != nil { + t.Fatal(err) + } + kept := map[string][]byte{} + for i, k := range []delivery.KeptValue{ + {Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"}, + {Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)}, + {Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)}, + } { + kept[strconv.Itoa(i)], _ = json.Marshal(k) + } + held := HeldOf(kept, map[string]string{"novox/x#1": "h1"}) + if len(held) != 1 || string(held[0].Value) != "81" { + t.Fatalf("held = %+v, want #1's at its head alone", held) + } + c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held} + if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") { + t.Errorf("composition = %+v", v) + } + c.Held[0].Value = json.RawMessage(`"eighty"`) + if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") { + t.Errorf("a held value the declaration refuses: %+v", v) + } } // 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module. diff --git a/modules/mesh-delivery-assigner/internal/assign/findings.go b/modules/mesh-delivery-assigner/internal/assign/findings.go new file mode 100644 index 0000000..d6e2ea9 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/findings.go @@ -0,0 +1,105 @@ +package assign + +import ( + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says +// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair +// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its +// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router +// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the +// assigner no longer says it straight (never both). + +// FindingsBucket is the assigner's findings. +var FindingsBucket = health.FindingsBucket(delivery.AssignerModule) + +// FindingOf is a said condition as a finding: its key `..`, from the condition's own key +// `mesh-delivery-assigner..` (the subject's id when the key names none), its words unchanged. raised is false for +// a clear, whose key is to be deleted; value is nil then. +func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) { + rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".") + if !ok || rest == "" { + return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key) + } + kind, id, _ := strings.Cut(rest, ".") + if id == "" { + id = c.Subject.ID + } + scope := c.Subject.Scope + if scope == "" { + scope = health.ScopeModule + } + key = health.FindingKey(scope, id, kind) + if c.State == asks.StateCleared { + return key, nil, false, nil + } + if c.State != asks.StateRaised { + return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key) + } + summary := c.Summary + if summary == "" { + summary = c.Headline + } + var actions []health.Action + for _, a := range c.Actions { + actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)}) + } + value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine}, + Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, + Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind, + Observer: delivery.AssignerModule, At: at.UTC()}) + return key, value, true, err +} + +// Findings is where the assigner's findings go: the bucket's put and delete, and whether the keeper's conditions are the +// ones said (the take-over key names the keeper). +type Findings struct { + Put func(key string, value []byte) error + Delete func(key string) error + Keys func() ([]string, error) + ToKeeper func() bool + Now func() time.Time +} + +// Say is say with each condition also written as a finding (or its finding deleted), and said straight to the router only +// while the take-over key follows the controller. A finding that cannot be written is the error, beside the say's. +func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error { + return func(c asks.Condition) error { + var errs []error + if f.Put != nil { + now := time.Now + if f.Now != nil { + now = f.Now + } + key, value, raised, err := FindingOf(c, now()) + switch { + case err != nil: + errs = append(errs, err) + case raised: + if err := f.Put(key, value); err != nil { + errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err)) + } + default: + if err := f.Delete(key); err != nil { + errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err)) + } + } + } + if f.ToKeeper == nil || !f.ToKeeper() { + if say != nil { + if err := say(c); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/placement.go b/modules/mesh-delivery-assigner/internal/assign/placement.go index 9f31c35..131512d 100644 --- a/modules/mesh-delivery-assigner/internal/assign/placement.go +++ b/modules/mesh-delivery-assigner/internal/assign/placement.go @@ -2,6 +2,7 @@ package assign import ( "fmt" + "slices" "sort" "strings" @@ -15,7 +16,8 @@ import ( // removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal // the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not // yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key -// is written the controller's assignments are the one source, as before. +// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease +// handed away) the placement is the one source. // Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or // OnlyPlacement. @@ -32,8 +34,17 @@ const ( ) // Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then -// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. -func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) { +// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's +// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the +// assignments are then a source of nothing. +func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) { + if handedAway && seeded { + out := make(map[string][]string, len(placed)) + for node, modules := range placed { + out[node] = slices.Sorted(slices.Values(modules)) + } + return out, nil + } if !seeded { return assigned, nil } diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go new file mode 100644 index 0000000..e877ba7 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -0,0 +1,208 @@ +package assign + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one +// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's +// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled. +// Until the lease write the controller still writes a policy beside each composition's digest; where it differs from the +// assigner's, the assigner says so as its finding `policy-split`, naming mesh-wants.upgrade as the remedy. + +// PoliciesBucket is the assigner's state `policies`. +var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState + +// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named. +func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) { + out := map[string]catalogue.PersonPolicy{} + var bad []string + for key, raw := range values { + var p catalogue.PersonPolicy + if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key { + bad = append(bad, key) + continue + } + out[key] = p + } + sort.Strings(bad) + return out, bad +} + +// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one. +func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) { + var c *catalogue.PersonPolicy + if p, ok := chosen[m.Module]; ok { + c = &p + } + return catalogue.Upgrade(m, c) +} + +// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it. +func ControllerPolicies(composed map[string][]byte) map[string][]string { + out := map[string][]string{} + for key, raw := range composed { + _, module, _, isNode, err := delivery.ParseComposedKey(key) + if err != nil || isNode { + continue + } + d, err := delivery.DecodeComposedDigest(raw) + if err != nil || d.Policy == "" { + continue + } + if !contains(out[module], d.Policy) { + out[module] = append(out[module], d.Policy) + sort.Strings(out[module]) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// Split is a module whose policy at the assigner differs from one the controller writes beside a composition of it. +type Split struct { + Module string `json:"module"` + Policy string `json:"policy"` + From string `json:"from"` + Controller []string `json:"controller"` +} + +// Splits is every module of manifests the controller writes a policy for that differs from the assigner's, sorted. +func Splits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy, controller map[string][]string) []Split { + var out []Split + for module, theirs := range controller { + m, ok := manifests[module] + if !ok { + continue + } + policy, from, _, _ := PolicyOf(m, chosen) + for _, p := range theirs { + if p != policy { + out = append(out, Split{Module: module, Policy: policy, From: from, Controller: theirs}) + break + } + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// KindPolicySplit is the assigner's finding of a split policy (ADR 0565 §8). +const KindPolicySplit = "policy-split" + +// PolicySplitKey is a module's policy-split finding's key. +func PolicySplitKey(module string) string { + return health.FindingKey(health.ScopeModule, module, KindPolicySplit) +} + +// PolicySplitFinding is the finding of one split, in mesh-sdk's layout. +func PolicySplitFinding(s Split, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s's upgrade policy is %s at the assigner (%s) and %s at the controller", s.Module, s.Policy, s.From, + strings.Join(s.Controller, " and ")) + return health.EncodeFinding(health.Finding{Key: PolicySplitKey(s.Module), Kind: KindPolicySplit, + Subject: health.Subject{Scope: health.ScopeModule, ID: s.Module}, Severity: health.Warning, Summary: summary, + Headline: clipHeadline(s.Module + "'s upgrade policy is split"), + Needs: "set it with mesh-wants.upgrade, so the assigner and the controller agree", + Explanation: asks.NeedsYou + "set it with mesh-wants.upgrade, so the assigner and the controller agree." + " The controller still " + + "writes a policy beside each composition until it hands its lease away; a choice made at its upgrade verb after the copy " + + "is not the assigner's.", + Resolved: s.Module + "'s upgrade policy agrees again", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindPolicySplit, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +func clipHeadline(s string) string { + if len([]rune(s)) <= 60 { + return s + } + return string([]rune(s)[:60]) +} + +// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4). +const KindProvisionFailed = "provision-failed" + +// ProvisionFailedKey is a failed provision's finding's key, `provider...[.].provision-failed`. +func ProvisionFailedKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed) +} + +// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why. +func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision, + p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module) + return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.", + Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}}, + Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +func also(provider, consumer string) []string { + if provider == "" || provider == consumer { + return nil + } + return []string{provider} +} + +// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound. +const KindProvisionMissing = "provision-missing" + +// ProvisionMissingKey is its key, `provider....provision-missing`. +func ProvisionMissingKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing) +} + +// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `..`). +func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) { + parts := strings.SplitN(key, ".", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("%q is not ..", key) + } + node, module, provision := parts[0], parts[1], parts[2] + summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+ + "no provider of it may be placed", module, node, provision, bound, module) + return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.", + Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6). +const KindUnplaced = "unplaced" + +// UnplacedKey is its finding's key, `machine..unplaced`. +func UnplacedKey(node string) string { + return health.FindingKey(health.ScopeMachine, node, KindUnplaced) +} + +// UnplacedFinding is the finding of a node no placement names since the lease write. +func UnplacedFinding(node string, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+ + "so the assigner writes it no want", node) + return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced, + Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary, + Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.", + Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".", + Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()}) +} diff --git a/modules/mesh-delivery-assigner/internal/assign/seat.go b/modules/mesh-delivery-assigner/internal/assign/seat.go index 0874e81..05a02c6 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat.go @@ -15,10 +15,6 @@ import ( // `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the // operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do. -// Leased says whether the lease write of ADR 0547 has happened, after which the controller's assignments are read no -// more (§8). Not before L3: until then the holder keeps the controller in step and refuses what its verbs cannot take. -var Leased = false - // ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the // revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's // digest. @@ -32,6 +28,10 @@ type ProposeIn struct { Recorded func(node string) bool Seed bool ExportDigest string + // Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk + // buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then + // the holder keeps the controller in step and refuses what its verbs cannot take. + Leased bool } // Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision), @@ -58,7 +58,7 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1)) continue } - if !Leased { + if !in.Leased { if why := c.BeforeLease(); why != "" { findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why)) } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat_test.go b/modules/mesh-delivery-assigner/internal/assign/seat_test.go index 1f9a8fd..257da34 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat_test.go @@ -60,18 +60,18 @@ func TestThePlacementSeatDecides(t *testing.T) { if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 { t.Errorf("config values before the lease write: %v", f) } - Leased = true for _, c := range []delivery.PlacementProposalChange{ {Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}, {Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}, {Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)}, {Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"}, } { - if _, _, f := Propose(in(c)); len(f) != 0 { + leased := in(c) + leased.Leased = true + if _, _, f := Propose(leased); len(f) != 0 { t.Errorf("after the lease write %s is refused: %v", c.Op, f) } } - Leased = false // A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again. if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" || diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 6d54820..237d38a 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the @@ -29,7 +30,12 @@ type Module struct { // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone. GrantedOn func(node string) bool Manifest catalogue.Manifest - Nodes []string + // Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want + // waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it. + Provisions func(node string) map[string]provisions.Provision + // Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none. + Chosen *catalogue.PersonPolicy + Nodes []string // Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its // digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6). Composed func(node, fp string) (delivery.ComposedDigest, bool) @@ -124,9 +130,54 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { if _, ok := m.composition(node, m.FP); !ok { return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) } + if words, waiting := m.provisionWaiting(node); waiting { + return false, delivery.RuleProvisions, words + } return true, 0, "" } +// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has +// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none. +func (m Module) provisionWaiting(node string) (string, bool) { + if m.Provisions == nil { + return "", false + } + if missing := m.MissingProvisions(node); len(missing) > 0 { + return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true + } + keyed := m.Provisions(node) + for _, key := range slices.Sorted(maps.Keys(keyed)) { + if p := keyed[key]; p.State != provisions.Provided { + words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) + if p.Why != "" { + words += ": " + p.Why + } + return words, true + } + } + return "", false +} + +// MissingProvisions are the provisions the module requires (not optionally) that have no key on node yet, sorted; none +// while the seat's state is not read. +func (m Module) MissingProvisions(node string) []string { + if m.Provisions == nil { + return nil + } + have := map[string]bool{} + for _, p := range m.Provisions(node) { + have[p.Provision] = true + } + var out []string + for _, r := range m.Manifest.Requires { + if !have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(out, r) { + out = append(out, r) + } + } + slices.Sort(out) + return out +} + // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned // elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node // offers it) and holds. @@ -639,9 +690,14 @@ func (m Module) busBar(before delivery.Want, had bool, running string) (prev str return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP } -// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the -// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. +// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR +// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the +// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. func (m Module) recorded(d delivery.ComposedDigest) bool { + if m.Chosen != nil { + policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen) + return policy == catalogue.PolicyRecord + } if d.Policy != "" { return d.Policy == delivery.PolicyRecord } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 8d75165..06886cb 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } @@ -69,6 +70,12 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { delivery.RuleComposed: func(m *Module, _ *Nodes) { m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false } }, + // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. + delivery.RuleProvisions: func(m *Module, _ *Nodes) { + m.Provisions = func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} + } + }, } { m, n := ready() breakIt(&m, &n) @@ -82,6 +89,29 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { t.Errorf("rule %d broken: g14's want is %+v", rule, w) } + if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) { + t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) + } + } + // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. + for _, all := range []func(string) map[string]provisions.Provision{ + func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}} + }, nil} { + m, n := ready() + m.Provisions = all + if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew { + t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) + } + } + // 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named. + { + m, n := ready() + m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} } + if out := Wants(m, n, wantedOld(), nil); out.Rules["g14"].Rule != delivery.RuleProvisions || + !strings.Contains(out.Rules["g14"].Words, "provision database has no key yet") || len(m.MissingProvisions("g14")) != 1 { + t.Errorf("a provision with no key: %+v", out.Rules["g14"]) + } } // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not @@ -802,6 +832,12 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) } + // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. + with(delivery.PolicyRecord, "person") + m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} + if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew { + t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) + } } // Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1803f4b..f722280 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -33,9 +33,9 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", - "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"} + want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } @@ -44,7 +44,7 @@ func TestManifest(t *testing.T) { states[s.Name] = s.History } if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20, - "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0}; !reflect.DeepEqual(states, want) { + "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -63,9 +63,14 @@ func TestManifest(t *testing.T) { // ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream // placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its // hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches. - if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) { - t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs", m.Claims) + // ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus. + if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh || + !slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) || + m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) { + t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs and mesh-wants with its two", m.Claims) + } + if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil { + t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat)) } if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil { t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat)) @@ -79,7 +84,8 @@ func TestManifest(t *testing.T) { } if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin", "seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin", - "seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { + "seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat, the controller's only while it has no holder + "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if !slices.Equal(m.Emits, []string{"put"}) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 91b6bca..91ad85a 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -15,21 +15,34 @@ "adopt", "hand-back" ] + }, + { + "name": "mesh-wants", + "scope": "mesh", + "serves": [ + "upgrade", + "bus" + ] } ], "reads": [ "mesh-build-go.steps", + "mesh-conditions.takeover", + "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", - "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", + "mesh-controller.lease", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", + "mesh-grants.grants", + "mesh-provisioner.provisions", "node-engine.have" ], "uses": [ @@ -44,6 +57,9 @@ "seat:mesh-controller.unassign", "seat:mesh-controller.unpin", "seat:mesh-controller.warranted", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.hand-acts", + "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now" ], @@ -64,7 +80,10 @@ { "name": "compositions", "history": 5 }, { "name": "differences", "history": 64 }, { "name": "puts", "history": 20 }, - { "name": "wants", "history": 20 } + { "name": "wants", "history": 20 }, + { "name": "findings", "history": 5 }, + { "name": "policies", "history": 20 }, + { "name": "seat-holders", "history": 5 } ], "streams": [ { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index dd9835d..49f5ddd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -26,7 +26,7 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for // the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer. -func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool { +func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: build.StuckTool, @@ -94,6 +94,20 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer, "next": build.StuckTool + " shows the key and its asks while they wait"}, nil }}, + {Name: delivery.BuildsSinceTool, + Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " + + "fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + + "queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " + + "log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{ + "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}}, + Run: func(args map[string]any) (any, error) { + at, err := time.Parse(time.RFC3339Nano, str(args, "since")) + if err != nil { + return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err) + } + return build.BuildsSince(since, at) + }}, {Name: build.AnnounceAgainTool, Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " + "again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " + @@ -250,7 +264,7 @@ func main() { a, err := build.DecodeReAsk(v) return err == nil && !a.Answered.IsZero() }) - if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil { + if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index b944b16..bae4fd6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -268,6 +269,12 @@ func startBedWith(t *testing.T, closed bool) *bed { mk(delivery.ManifestsBucket, delivery.ManifestsHistory) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) + // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. + logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog) + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, + AllowDirect: true}); err != nil { + t.Fatal(err) + } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.closed.Store(closed) b.fake.serve(t, "mesh-build-go", "n1", "n2") @@ -280,6 +287,29 @@ func startBedWith(t *testing.T, closed bool) *bed { return b } +// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user. +func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry { + t.Helper() + conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"}) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module) + if err != nil { + t.Fatal(err) + } + var out []delivery.BuildLogEntry + for _, le := range entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + // report writes a node's toolchains key of a build module and says so on the module's stream, as the module does. func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() @@ -406,6 +436,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } + // 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a + // reader with the log's read reads them, the end naming the manifest and what was made. + ended := b.logged(t, "netcheck") + if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent || + ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" { + t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended) + } for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f003aae..cef81c7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -120,6 +120,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Manifests: manifestReader{manifests}, Logf: logger.Printf, Stored: storedVia(calls.Call), + Node: cred.Node, } // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision @@ -129,6 +130,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} trunks.start() b.Trunk = trunks.head + // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the + // indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since. + b.Log = func(e delivery.BuildLogEntry) { + if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil { + logger.Printf("%v", err) + } + } b.Said = func(e build.AskedAgain) { if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { logger.Printf("%v", err) @@ -153,11 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The toolchains first, so a build that starts from the first modules value finds the nodes. watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger) - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf) if err != nil { return err } @@ -254,7 +262,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil case err := <-errs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-watch.errs: return err diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 311ba12..f2b4f37 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -138,3 +140,26 @@ func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) } } + +// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key +// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential. +// The id makes a repeated append of one entry one entry. +func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error { + data, err := delivery.EncodeBuildLogEntry(e) + if err != nil { + return err + } + key, err := delivery.BuildLogKey(e.Module) + if err != nil { + return err + } + msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key) + msg.Data = data + id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err) + } + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go new file mode 100644 index 0000000..407c6ba --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the +// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and +// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, +// issue 774). +func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { + set := &buildrun.ToolchainSet{} + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } + // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk + // head is told why and opens it again then. + src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} + src.start() + if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { + t.Errorf("a trunk head with the source refused: %v, want why", err) + } + if src.opened { + t.Error("a refused source counts as opened, so it is never opened again") + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go deleted file mode 100644 index 4a0e7ca..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/process" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } - - // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every - // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without - // is an error naming the bucket and its grant. - var logs bytes.Buffer - logger := log.New(&logs, "", 0) - refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} - if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { - t.Errorf("a refused watch of the assignments stopped the start: %v", err) - } - if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { - t.Errorf("a refused toolchains bucket stopped the start: %v", err) - } - if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { - t.Errorf("the refusals were not said: %q", logs.String()) - } - if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || - !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { - t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) - } - // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk - // head is told why and opens it again then. - src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} - src.start() - if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { - t.Errorf("a trunk head with the source refused: %v, want why", err) - } - if src.opened { - t.Error("a refused source counts as opened, so it is never opened again") - } -} - -// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. -type refusingBus struct { - refuseOpen bool - err error -} - -func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { - if r.refuseOpen { - return nil, r.err - } - return refusedWatch{err: r.err}, nil -} - -type refusedWatch struct { - jetstream.KeyValue - err error -} - -func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} -func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go index 3f2bbef..2b0edb1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go @@ -9,7 +9,9 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "mesh-delivery-builder/internal/build" ) @@ -18,17 +20,27 @@ import ( // consumer of artifacts-gone (mesh-controller internal/broker EventsStream). const eventsStream = "EVENTS" +// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2). +const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held" + // toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call). type toolCaller func(key string, body any) (json.RawMessage, error) // storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue // 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: // the key is then left done, and the controller's take-in still refuses a gone blob. +// +// It asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2), and the controller's while that seat has no holder that +// answers (a named exception of clause 10, until the lease write). func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) { return func(_ context.Context, references []string) ([]string, error) { - raw, err := call(delivery.HeldVerb, map[string]any{"references": references}) + body := map[string]any{"references": references} + raw, err := call(SeatHeld, body) + if handacts.NoResponder(err) { + raw, err = call(delivery.HeldVerb, body) + } if err != nil { - return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err) + return nil, fmt.Errorf("asking held: %w", err) } a, err := delivery.DecodeHeldAnswer(raw, references) if err != nil { diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 4464bf9..e9d789c 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 9dde7f0..a2b8e01 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 935860b..3687c21 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "sort" + "strconv" "strings" "sync" "time" @@ -68,6 +69,11 @@ type Builder struct { Askings int // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. Said func(AskedAgain) + // Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody. + Log func(delivery.BuildLogEntry) + // Now is the clock of the log; nil is time.Now. Node is the node the builder runs on, which asked the build. + Now func() time.Time + Node string once sync.Once sem chan struct{} @@ -429,6 +435,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } b.handClaimed(key, cl.Rev) + b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -478,6 +485,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { b.logf("%s: the build could not be written: %v", cl.Key, err) return } + b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result)) if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } @@ -872,3 +880,60 @@ func needsFetch(tasks []Task) bool { } return false } + +func (b *Builder) now() time.Time { + if b.Now != nil { + return b.Now() + } + return time.Now() +} + +// logBuild tells the log an entry, said in the journal when it cannot be one. +func (b *Builder) logBuild(e delivery.BuildLogEntry) { + if b.Log == nil { + return + } + if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err) + return + } + b.Log(e) +} + +// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's. +func (b *Builder) refOf(module, fp string) string { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "" + } + b.mu.Lock() + defer b.mu.Unlock() + return b.via[key].Ref +} + +// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the +// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder +// that asked it and the time. +func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry { + dir := m.Path + if dir == "" { + dir = delivery.RootPath + } + return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10), + Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()} +} + +// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it +// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from. +func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry { + e := LogStarted(module, m, ref, by, at) + e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest + if result.Manifest != "" { + e.Manifest = result.Manifest + } + if m.Inputs2 != nil { + e.Against = append([]string(nil), m.Inputs2.Bases...) + } + e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...) + return e +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 6cbb41c..a348fdc 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strconv" "strings" "sync" @@ -13,9 +12,11 @@ import ( "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the @@ -221,7 +222,42 @@ var ( func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) + var logged []delivery.BuildLogEntry + var lmu sync.Mutex + r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() } v := r.consider(t, "modules/m") + // 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read + // from the two. + if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent || + logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" || + logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" || + logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" || + logged[0].FP != fp1 || logged[0].Module != "m" { + t.Fatalf("logged = %+v", logged) + } + for _, e := range logged { + if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + t.Errorf("an entry the log refuses: %v", err) + } + } + if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok { + t.Error("no duration from the two entries") + } + // builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more. + var page stdio.LogPage + for i, e := range logged { + raw, _ := delivery.EncodeBuildLogEntry(e) + page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At}) + } + page.Next = 3 + since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At) + if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { + t.Errorf("builds since = %+v, %v, want the one end and more", since, err) + } + page.Entries[0].Value = []byte(`{"v":0}`) + if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil { + t.Error("an entry that does not read was skipped, want it refused") + } if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } diff --git a/modules/mesh-delivery-builder/internal/build/since.go b/modules/mesh-delivery-builder/internal/build/since.go new file mode 100644 index 0000000..968e597 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/since.go @@ -0,0 +1,43 @@ +package build + +import ( + "fmt" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more. +const BuildsSinceLimit = 500 + +// LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since. +type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) + +// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, +// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again +// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that +// skipped it would register too little. +func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) { + out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}} + page, err := read(since, "", BuildsSinceLimit) + if err != nil { + return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err) + } + if page == nil { + return out, nil + } + var entries []delivery.BuildLogEntry + for _, le := range page.Entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err) + } + entries = append(entries, e) + } + if ended := delivery.EndedSince(entries, since); ended != nil { + out.Builds = ended + } + out.More = page.Next != 0 + return out, nil +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 44c3002..ce7f764 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -10,6 +10,8 @@ import ( "slices" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, @@ -40,10 +42,14 @@ type manifest struct { Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` - Reads []string `json:"reads"` - Uses []string `json:"uses"` - Invokes []string `json:"invokes"` - Config map[string]struct { + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` + Reads []string `json:"reads"` + Uses []string `json:"uses"` + Invokes []string `json:"invokes"` + Config map[string]struct { Type string `json:"type"` Description string `json:"description"` Pattern string `json:"pattern"` @@ -92,8 +98,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). - if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments", - "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -101,12 +107,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. - if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 { + // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's only while the seat has no holder (clause 10). + if want := []string{"seat:mesh-artifact-sweep.held", "seat:mesh-controller.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it // starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4). - if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // 1955/L7: and the seat mesh-artifact-sweep's, the controller's until the lease write (its take-in sweeps until then). + if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -116,13 +124,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } - for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + // 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream. + if len(m.Logs) != 1 || m.Logs[0].Name != delivery.BuilderLog || m.Logs[0].MaxAgeDays != delivery.BuilderLogDays { + t.Errorf("logs = %+v, want %s kept %d days", m.Logs, delivery.BuilderLog, delivery.BuilderLogDays) + } + for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 2929850..1631df0 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -6,6 +6,7 @@ "builder_announce_again", "builder_ask_again", "builder_build_again", + "builder_builds_since", "builder_seat_queue", "builder_stuck" ], @@ -13,11 +14,13 @@ "build.asked-again" ], "consumes": [ + "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone" ], "invokes": [ + "seat:mesh-artifact-sweep.held", "seat:mesh-controller.held" ], "state": [ @@ -27,13 +30,16 @@ { "name": "queues", "history": 1 }, { "name": "reasks", "history": 1 } ], + "logs": [ + { "name": "build-log", "max-age-days": 30 } + ], "reads": [ "mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.assignments", + "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", @@ -44,7 +50,6 @@ "node-build-image", "node-build-node" ], - "own-secrets": { "broker": "${dir:state}/broker" }, diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 660b6d5..33045f3 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -170,6 +170,21 @@ func TryTools(tr Trying) []stdio.Tool { } } +// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named .. +const Seat = "mesh-delivery-checker" + +// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try, +// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them. +func SeatTryTools(tr Trying) []stdio.Tool { + verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"} + var out []stdio.Tool + for _, t := range TryTools(tr) { + t.Name = Seat + "." + verbs[t.Name] + out = append(out, t) + } + return out +} + // Tools are the read tools over the checker's states. func Tools(r check.Reader) []stdio.Tool { return []stdio.Tool{ @@ -308,7 +323,7 @@ func main() { _, err := verdicts.Update(key, json.RawMessage(value), revision) return err }} - tools := append(append(Tools(r), TryTools(trying)...), RecoveryTools(r, written)...) + tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 282db23..779922b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -88,6 +88,24 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) { t.Errorf("read = %s, %v", raw, err) } + // The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike. + verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) }, + Trunks: func() (map[string]string, error) { + return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil + }}) + if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" { + t.Fatalf("verbs = %+v", verbs) + } + again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"}) + if err != nil || again.(map[string]any)["id"] != id { + t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id) + } + if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) { + t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw) + } + if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil { + t.Error("the verb tried a repository nothing checks") + } // Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew. ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"}) if err != nil || ended.(map[string]any)["state"] != "error" { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 5f3018c..732bdfb 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -208,11 +208,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The toolchains first, so a check that starts from the first candidate finds the nodes. watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf) if err != nil { return err } @@ -362,7 +362,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil case err := <-errs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-watch.errs: return err diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go new file mode 100644 index 0000000..3c20242 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the +// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and +// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, +// issue 774). +func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { + set := &buildrun.ToolchainSet{} + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go deleted file mode 100644 index 0715aea..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/process" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } - - // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every - // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without - // is an error naming the bucket and its grant. - var logs bytes.Buffer - logger := log.New(&logs, "", 0) - refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} - if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { - t.Errorf("a refused watch of the assignments stopped the start: %v", err) - } - if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { - t.Errorf("a refused toolchains bucket stopped the start: %v", err) - } - if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { - t.Errorf("the refusals were not said: %q", logs.String()) - } - if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || - !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { - t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) - } -} - -// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. -type refusingBus struct { - refuseOpen bool - err error -} - -func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { - if r.refuseOpen { - return nil, r.err - } - return refusedWatch{err: r.err}, nil -} - -type refusedWatch struct { - jetstream.KeyValue - err error -} - -func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} -func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { - return nil, r.err -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 790655b..2cb4fa6 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 9dde7f0..a2b8e01 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 5d708df..d80dd7e 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -425,9 +425,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1 } } -// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run -// after it (so one check names every failure), and no later level runs. -func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) { +// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends +// there: no other package of the level and no later level runs, so the rest of the suite is not paid for. +func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) { b := newBench(t) b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"} b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} @@ -443,8 +443,8 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing. t.Fatalf("verdict = %+v", out.Verdict) } tests := b.steps.tool("test") - if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" { - t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests) + if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" { + t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests) } if n := len(b.steps.tool("digest")); n != 1 { t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n) diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 404988d..218d358 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -38,6 +38,11 @@ type stream struct { } type manifest struct { + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` Streams []stream `json:"streams"` Tools []string `json:"tools"` Module string `json:"module"` @@ -96,8 +101,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments", - "mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", + if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-delivery-assigner.compositions", "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } @@ -129,7 +134,12 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code. + if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-delivery-checker" || m.Claims[0].Scope != "mesh" || + !reflect.DeepEqual(m.Claims[0].Serves, []string{"try", "try-read", "try-end"}) { + t.Errorf("claims = %+v, want the mesh seat mesh-delivery-checker serving try, try-read and try-end", m.Claims) + } + for _, k := range []string{"logs", "emits", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the checker has none", k) } diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 6d12e3b..f020663 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -2,6 +2,17 @@ "module": "mesh-delivery-checker", "version": "1", "slug": "checker", + "claims": [ + { + "name": "mesh-delivery-checker", + "scope": "mesh", + "serves": [ + "try", + "try-read", + "try-end" + ] + } + ], "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported" @@ -23,8 +34,8 @@ "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.assignments", "mesh-delivery-assigner.compositions", + "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index f7661dc..3889df4 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 6245202..a2b8e01 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index 087279a..9538477 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -46,6 +46,13 @@ var ErrNotFound = errors.New("the forge has no such file") // says is not there is ErrNotFound, any other failure any other error (issue 1448). type Files func(owner, repo, path, ref string) ([]byte, error) +// CatalogRepository is the catalogue, whose merged message carries its predefined list, and PredefinedFile that list's +// file at its root (novox/hq ADR 0565 §2, ADR 0348 §7). +const ( + CatalogRepository = "novox/mesh-catalog" + PredefinedFile = "predefined.json" +) + // LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465 // clause 1: go.sum for a Go module, package-lock.json for a Node one). var LockFiles = []string{"go.sum", "package-lock.json"} @@ -315,6 +322,18 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag } f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks} + if repository == CatalogRepository { + // The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. + raw, err := files(p.Owner, p.Repo, PredefinedFile, p.MergeSHA) + switch { + case errors.Is(err, ErrNotFound): + f.PredefinedRead = true + case err != nil: + return Message{}, fmt.Errorf("%w: %s: %s of %s at the merge commit %s: %v", ErrRetry, subject, PredefinedFile, repository, p.MergeSHA, err) + default: + f.Predefined = json.RawMessage(raw) + } + } body, err := delivery.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index 3e5e681..6c558c8 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -97,6 +97,27 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 t.Errorf("the forge was asked for %s", c.path) } } + // 1953/D1 (ADR 0565 §2): the catalogue's merged message carries its predefined.json at the merge, or says it holds + // none; another repository's asks the forge for none. + if !f.PredefinedRead || len(f.Predefined) != 0 { + t.Errorf("a merge of the catalogue with no predefined.json does not say so: %s", msg.Data) + } + forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`) + msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead { + t.Errorf("predefined = %s (%v)", f.Predefined, err) + } + forge.calls = nil + msg, _ = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + for _, c := range forge.calls { + if c.path == PredefinedFile { + t.Errorf("another repository's merge asked for %s", c.path) + } + } + forge.errs = map[string]error{PredefinedFile: errors.New("forge down")} + if _, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); !errors.Is(err, ErrRetry) { + t.Errorf("a forge that cannot say is %v, want a retry", err) + } // Issue 1126: a merge reads the forge only for the module directories whose tree changed. t.Run("a merge reads only the directories whose tree changed", mergeReadsOnlyChangedDirs) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index edecd2f..3262321 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -19,6 +19,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -44,21 +45,10 @@ accounts { // manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and // no mesh-delivery-gate_source bucket. func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - // Build records in the controller's layout (mesh-sdk go/delivery/testdata/build-record-build-agent.json, captured from - // the controller's encoder), under delivery.BuildKey, with the newest pointer beside them. - for _, b := range []struct{ repo, module, path, id string }{ - {"novox/mesh-delivery", "a", "modules/a", "build-1"}, {"novox/mesh-catalog", "c", "modules/c", "build-2"}, - } { - rec, _ := json.Marshal(map[string]any{"id": b.id, "repository": b.repo, "ref": "main", "module": b.module, - "commit": "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", "path": b.path, "at": "2026-10-19T09:38:00Z"}) - if _, err := builds.Put(ctx, delivery.BuildKey(b.module, b.id), rec); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey(b.module), []byte(b.id)); err != nil { - t.Fatal(err) - } - } + js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { + // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. + builds.end(t, "novox/mesh-delivery", "a", "modules/a") + builds.end(t, "novox/mesh-catalog", "c", "modules/c") }) h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef" for _, f := range []delivery.Forge{ @@ -99,7 +89,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 // startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill // mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created. -func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -162,11 +152,12 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil { t.Fatal(err) } - builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5}) - if err != nil { + // The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog), + Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - put(ctx, builds) + put(ctx, buildLog{ctx: ctx, js: js}) for _, b := range before { b(ctx, js) } @@ -185,15 +176,32 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b return js, ctx, candidates, modules } +// buildLog appends entries to the builder's log as the builder does. +type buildLog struct { + ctx context.Context + js jetstream.JetStream +} + +// end appends the end of a done trunk build of module from repo's directory path. +func (l buildLog) end(t *testing.T, repo, module, path string) { + t.Helper() + raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + module, + Repository: repo, Directory: path, Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Node: "n1", + At: time.Date(2026, 10, 19, 9, 38, 0, 0, time.UTC), Outcome: delivery.BuildDone}) + if err != nil { + t.Fatal(err) + } + if _, err := l.js.Publish(l.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil { + t.Fatal(err) + } +} + // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) // The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1). - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}` - if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, @@ -229,15 +237,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -287,15 +288,8 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -362,16 +356,9 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` const trunk, head = "5555555555555555555555555555555555555555", "0123456789abcdef0123456789abcdef01234567" - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { - t.Fatal(err) - } + js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }, func(ctx context.Context, js jetstream.JetStream) { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: PreviewsStream, Subjects: []string{previewWrittenSubject}}); err != nil { t.Fatal(err) @@ -451,7 +438,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) { } objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} - js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) + js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}) publish := func(merge, root string, objects map[string]string) { data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70, Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root}, @@ -518,7 +505,7 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" digest := delivery.ManifestDigest(manifest) - _, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) { + _, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { mkey, err := delivery.ManifestsKey(digest) if err != nil { t.Fatal(err) @@ -578,23 +565,39 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC } } -// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where -// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the -// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain -// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard -// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch. -func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { +// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of +// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside +// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's +// toolchain alone; when the placement puts mesh-build-go on n2 again, its report written since counts and the builders +// disagree (the guard stays); when it is taken off again, the fingerprint comes back. No restart, no timer: the +// placement's watch (ADR 0549 §3), not the controller's assignments. +func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - var assignments jetstream.KeyValue - js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) { + var placement, toolchains jetstream.KeyValue + place := func(ctx context.Context, t *testing.T, node string, modules ...string) { + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + key, _ := delivery.PlacementKey(node) + value, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + if _, err := placement.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + } + js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { t.Fatal(err) } + toolchains = kv for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} { key, _ := delivery.ToolchainKey(node, "go") value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()}) @@ -605,18 +608,12 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } } - // The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment). - if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil { + // The seat mesh-placement's placement, as the assigner writes it (ADR 0567): n1 places mesh-build-go, n2 nothing. + if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil { t.Fatal(err) } - for key, value := range map[string]string{ - "n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`, - "n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`, - } { - if _, err := assignments.Put(ctx, key, []byte(value)); err != nil { - t.Fatal(err) - } - } + place(ctx, t, "n1", "mesh-build-go") + place(ctx, t, "n2") }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -647,16 +644,22 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { if !agreed(held) { t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains) } - if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil { + place(ctx, t, "n2", "mesh-build-go") + key, _ := delivery.ToolchainKey("n2", "go") + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: time.Now().Add(time.Second)}) + if err != nil { t.Fatal(err) } - // n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept + if _, err := toolchains.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + // n2's report, written since its placement, counts: the builders disagree. A fingerprint already computed is kept // (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree. time.Sleep(500 * time.Millisecond) if e, err := modules.Get(ctx, "avahi"); err != nil { t.Fatal(err) } else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 { - t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2) + t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2) } moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1) if moved == string(data) { @@ -669,12 +672,10 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) { return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2") } if m := read(disagree); !disagree(m) { - t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2) - } - if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil { - t.Fatal(err) + t.Fatalf("within 5 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2) } + place(ctx, t, "n2") if m := read(agreed); !agreed(m) { - t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2) + t.Fatalf("within 5 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.FP2, m.None2) } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index aa732ef..e02d915 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -10,20 +10,22 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "io" "log" "os" "os/signal" + "sync/atomic" "syscall" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" @@ -273,28 +275,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then // a merge's taken key is said and not written, and the rest of the indexer runs. taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf) - // The controller's build records only name the directory of a module whose key names none (fillPaths): without them - // such a module waits for its path, every other is indexed (issue 1945). - builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) + // The control records (ADR 0565 §2), new with this delivery and bound when they appear: until then nothing is written to them. + predefined := process.AwaitBucket(ctx, js, conn, index.PredefinedBucket, "no predefined list is written", logger.Printf) + configCandidates := process.AwaitBucket(ctx, js, conn, index.ConfigCandidatesBucket, "no pull request's config declarations are written", logger.Printf) + // The builder's log (ADR 0571 clause 1), read on the indexer's own credential (the calls connection) through its declared read: every entry it + // keeps at the start, then each new one as the builder appends it (no timer). It names the directory of a module whose key + // names none (fillPaths) and the places a registered module was built from; no build moves a fingerprint (ADR 0530 §1). + // Where the log cannot be read such a module waits for its path, every other is indexed (issue 1945), and it is said. known := index.NewBuilds() - // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any - // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). - each := func(e jetstream.KeyValueEntry) { - if opOf(e) { - known.Put(e.Key(), e.Value()) - } else { - known.Delete(e.Key()) - } - } errs := make(chan error, 1) - var buildsWatch jetstream.KeyWatcher - if builds != nil { - if w, err := settle(ctx, builds, each); err != nil { - process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again") - } else { - buildsWatch = w - } - } // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) @@ -302,9 +291,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer calls.Close() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf, + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call)} + // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each + // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). + var started atomic.Bool + if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error { + if err := known.Put(le.Value); err != nil { + logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err) + return nil + } + if started.Load() { + if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil { + logger.Printf("the registered module %s is not indexed: %v", le.Key, err) + } + } + return nil + }); err != nil { + logger.Printf("%s is not read (%v): a module whose key names no directory is not given one, and no registered module is "+ + "indexed from it, until the indexer starts again", delivery.BuildLogRead, err) + } + started.Store(true) // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. @@ -321,14 +330,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { sourceErrs <- follow(ctx, sw, each) }() } } - // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's - // assignments are settled first, so the toolchains settle through the gate already knowing them. - gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf) - assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) { + // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the + // placement is settled first, so the toolchains settle through the gate already knowing it. + gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf) + placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, func(ctx context.Context) { if err := ix.Recompute(ctx); err != nil { - logger.Printf("writing the modules again after an assignment of a build module changed: %v", err) + logger.Printf("writing the modules again after the placement of a build module changed: %v", err) } - }, logger) + }, logger.Printf) if err != nil { return err } @@ -355,28 +364,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.RecomputeAtStart(ctx); err != nil { logger.Printf("computing the new-form fingerprints at start: %v", err) } - // A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records - // name it and the bucket holds none (registered.go); the records that follow are heard below. + // A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names + // it and the bucket holds none (registered.go); the entries that follow are heard above. if err := ix.IndexRegisteredAll(ctx); err != nil { logger.Printf("indexing the registered modules at start: %v", err) } - if buildsWatch != nil { - go func() { - errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) { - each(e) - if !opOf(e) { - return - } - var r delivery.BuildRecord - if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" { - return - } - if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil { - logger.Printf("the registered module %s is not indexed: %v", r.Module, err) - } - }) - }() - } // A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched, // as every other input. A module the forge could not read is also read again by itself on the next source revision and // merged message (index.RetryUnread). @@ -442,7 +434,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-sourceErrs: return err - case err := <-assignErrs: + case err := <-placementErrs: return err case err := <-tools.errs: return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go deleted file mode 100644 index 9a06cec..0000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves -// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very -// toolchain agreement this file repairs. - -import ( - "context" - "encoding/json" - "fmt" - "log" - "maps" - "slices" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of -// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace -// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments, -// key .) hold an open assignment of its build module on its node, and only once it was written since that -// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report -// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node -// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran -// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot -// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that -// refusal is the guard of novox/hq ADR 0465, and it stays. -type toolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - known bool // the assignments are settled - standing map[string]time.Time // . with an open assignment -> since when it stands - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once -} - -func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate { - g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{}, - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers -// whether what it passed on changed. -func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool { - node, module, ok := strings.Cut(key, ".") - if !ok || !g.modules[module] { - return false - } - assigned, open := openAssignment(value) - open = open && !deleted - g.mu.Lock() - defer g.mu.Unlock() - was, stood := g.standing[key] - since := assigned - if since.IsZero() { - // A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it. - since = at - if stood { - since = was - } - } - if open == stood && (!open || since.Equal(was)) { - return false - } - if open { - g.standing[key] = since - } else { - delete(g.standing, key) - } - if !g.known { - return false - } - changed := false - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it -// passed on changed. -func (g *toolchainGate) Settled() bool { - g.mu.Lock() - defer g.mu.Unlock() - g.known = true - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not -// known), deleted otherwise. The caller holds mu. -func (g *toolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := delivery.ParseToolchainKey(key) - why := "" - if err == nil && g.known { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module, -// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *toolchainGate) leftOut(module, node string, value []byte) string { - since, open := g.standing[node+"."+module] - if !open { - return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node) - } - tc, err := delivery.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" -} - -// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not -// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when -// it was assigned, zero where the value does not say. -func openAssignment(value []byte) (time.Time, bool) { - var a struct { - Assigned time.Time `json:"assigned"` - Ended time.Time `json:"ended"` - State string `json:"state"` - } - if json.Unmarshal(value, &a) != nil { - return time.Time{}, false - } - return a.Assigned, a.Ended.IsZero() && a.State == "" -} - -// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is -// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read -// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel. -func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) { - errs := make(chan error, 1) - kv, err := js.KeyValue(ctx, provider.AssignmentsBucket) - if err != nil { - // Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery). - logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } - w, err := settle(ctx, kv, each) - if err != nil { - // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. - logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) - return errs, nil - } - if gate.Settled() { - changed(ctx) - } - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) { - changed(ctx) - } - }) - }() - return errs, nil -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go deleted file mode 100644 index 28fccc4..0000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). - -import ( - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func gateReport(t *testing.T, node, version string) (string, []byte) { - t.Helper() - key, err := delivery.ToolchainKey(node, "typescript") - if err != nil { - t.Fatal(err) - } - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()}) - if err != nil { - t.Fatal(err) - } - return key, value -} - -func agreedOf(set *buildrun.ToolchainSet) (string, string) { - tc, none := delivery.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the -// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them -// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again. -func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) }) - for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} { - key, value := gateReport(t, node, version) - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") { - t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none) - } - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{}) - } - gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{}) - gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter - if !gate.Settled() { - t.Fatal("settling the assignments left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(said) != 1 { - t.Fatalf("ace's report is said left out once; said %v", said) - } - // ace reports again while unassigned (an old value seen again): still left out. - key, value := gateReport(t, "ace", "v26.12.0") - if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("an unassigned node's report changed what was passed on") - } - // Assigned to ace again: its report counts, and the builders really disagree: the guard stays. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) { - t.Fatal("assigning ace again passed nothing on") - } - if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("builders that run different versions still disagree; got %q", none) - } - // The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again. - if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) { - t.Fatal("a pending assignment is no assignment on the machine") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - gate.Assignment("ace.mesh-build-node", nil, true, time.Time{}) - if v, _ := agreedOf(set); v != "v26.12.1" { - t.Fatalf("got %q", v) - } - // A report deleted from the bucket goes whether its node is assigned or not. - key, _ = gateReport(t, "g14", "v26.12.1") - if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) { - t.Fatal("deleting g14's report passed nothing on") - } - if n := len(set.Entries("typescript")); n != 2 { - t.Fatalf("novox and shanks are left; got %d entries", n) - } -} - -// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not -// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript -// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node -// writes it again. An assignment whose value names no time stands since the revision that opened it. -func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) { - left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again - report := func(t *testing.T, node, version string, at time.Time) (string, []byte) { - key, _ := gateReport(t, node, version) - value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value - } - for _, c := range []struct { - name string - assignment string // ace's assignment value - revision time.Time // when the bus wrote it - at time.Time // when ace wrote its report - counts bool - }{ - {"no report yet, then one", `{}`, time.Time{}, time.Now(), true}, - {"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false}, - {"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false}, - {"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - set := &buildrun.ToolchainSet{} - var said []string - gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - for _, node := range []string{"g14", "novox", "shanks"} { - gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{}) - key, value := gateReport(t, node, "v26.12.1") - gate.Toolchain(buildrun.NodeBuildModule, key, value, false) - } - gate.Settled() - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) { - t.Fatal("an assignment with no report passed something on") - } - key, value := report(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // A rewrite of the assignment that does not move it changes nothing, and nothing is said again. - if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) { - t.Fatal("the same assignment written again changed what was passed on") - } - v, none := agreedOf(set) - if c.counts { - if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") { - t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none) - } - return - } - if v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none) - } - if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", said) - } - // ace writes its report again since it was assigned: it counts. - key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute)) - if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) { - t.Fatal("ace's new report was not passed on") - } - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("four builders report; got %d", n) - } - }) - } -} diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 9460726..bf49249 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 9dde7f0..a2b8e01 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 7c3eacd..02a7a63 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -1,72 +1,63 @@ package index import ( - "encoding/json" + "fmt" "maps" "slices" "strings" "sync" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Builds holds the controller's build records (mesh-controller_builds) for the two things the indexer still reads from -// them until release R4g, when build-agent goes (the lead's decision of R4e, 2026-10-22): -// - the Builder input of the builder's own module, which build-agent builds (TrunkOf; novox/hq ADR 0453 clause 5, -// issue 1727); -// - the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no fingerprint. +// Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads +// of past builds: the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no +// fingerprint, and the places a module registered from its own repository was built from (RegistrationsOf). It is filled +// from the log's entries at the indexer's start (every one the log keeps, 30 days) and then each new one as the builder +// appends it; no entry is ever taken out, since only age takes one from the log. // // It lists no module and maps no directory to a module: the indexer's modules are those its own messages and bucket name // (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once. type Builds struct { mu sync.Mutex - records map[string]delivery.BuildRecord // by bucket key + records map[string]delivery.BuildRecord // by the entry's event, module, fingerprint and time } // NewBuilds is an empty Builds. func NewBuilds() *Builds { return &Builds{records: map[string]delivery.BuildRecord{}} } -// Put takes a key of the builds bucket and its value. Only a build record of a module (key `.build.`, -// delivery.BuildKey) counts; the newest-pointer keys and requests that knew no module are not records of one. -func (b *Builds) Put(key string, value []byte) { - if !strings.Contains(key, ".build.") { - return +// Put takes one entry of the builder's log. A build's end is an outcome: done registers, failed or queued does not; its +// start is a request, which registers nothing either. An entry that does not read, or names no module or repository, is +// refused and said. +func (b *Builds) Put(value []byte) error { + e, err := delivery.DecodeBuildLogEntry(value) + if err != nil { + return err } - var r delivery.BuildRecord - if err := json.Unmarshal(value, &r); err != nil || r.Module == "" || r.Repository == "" { - return + if e.Module == "" || e.Repository == "" { + return fmt.Errorf("a build log entry naming no module or repository: %s", value) + } + r := delivery.BuildRecord{ID: e.At.UTC().Format(time.RFC3339Nano) + "." + e.FP, Repository: e.Repository, Ref: e.Ref, Module: e.Module, + Path: e.Directory, Commit: e.Commit, At: e.At} + switch { + case e.Event != delivery.BuildEndedEvent: + r.State = e.Event + case e.Outcome == delivery.BuildFailed: + r.Failed = "the build failed" + case e.Outcome != delivery.BuildDone: + r.State = e.Outcome } b.mu.Lock() defer b.mu.Unlock() - b.records[key] = r -} - -// Delete takes a key out. -func (b *Builds) Delete(key string) { - b.mu.Lock() - defer b.mu.Unlock() - delete(b.records, key) + b.records[e.Event+"."+e.Module+"."+r.ID] = r + return nil } func newer(a, b delivery.BuildRecord) bool { return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID) } -// TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when -// there is none (issue 1727). -func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { - var best *delivery.BuildRecord - b.mu.Lock() - defer b.mu.Unlock() - for _, r := range b.records { - if r.Module == module && r.IsTrunkBuild() && (best == nil || newer(r, *best)) { - c := r - best = &c - } - } - return best -} - // PathOf is the directory of a module of the repository as its newest build record of that repository names it, its // newest successful trunk build's first ("." for the repository's root), and false when no record of the repository // names the module (issue 1614). @@ -79,7 +70,7 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { if r.Module != module || !delivery.SameRepository(r.Repository, repository) { continue } - t := r.IsTrunkBuild() + t := registered(r) && trunkLike[r.Ref] if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) { c := r best, trunk = &c, t @@ -91,15 +82,14 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { return pathOf(strings.Trim(best.Path, "/")), true } -// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record +// Registration is a module registered in the catalogue from its own repository, as the builder log.s newest end of a build // of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was // built at. type Registration struct { Module, Repository, Path, Commit, Ref string } -// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it -// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a +// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a // repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were // built from nox-mesh, and every record of them names that ref). func registered(r delivery.BuildRecord) bool { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index ebb36fe..5f85619 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -48,6 +48,8 @@ type Indexer struct { Modules Store // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store + // Predefined and ConfigCandidates are the control records' states (control_records.go, ADR 0565 §2); nil writes none. + Predefined, ConfigCandidates Store // Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none. Taken Store // Now is the clock of a taken value; nil is time.Now. @@ -121,6 +123,14 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil { return applied, err } + // The control records (ADR 0565 §2): the catalogue's predefined list, and the open pull requests' config + // declarations judged again against the trunk this merge moved. + if err := ix.writePredefined(ctx, f); err != nil { + return applied, err + } + if err := ix.configCandidatesAfterMerge(ctx, f, writtenModules(w.written)); err != nil { + return applied, err + } } ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) } @@ -134,6 +144,12 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, if held, err = ix.heldModules(ctx); err != nil { return applied, err } + if err := ix.configCandidateOfPull(ctx, f, held); err != nil { + if !errors.Is(err, ErrUnreadable) { + return applied, err + } + ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err) + } } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go new file mode 100644 index 0000000..0feb4ec --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -0,0 +1,292 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The indexer's two control records (novox/hq ADR 0565 §2, issue 1953 D2, D3): the catalogue's predefined list, read from the +// forge bridge's merged message, and each open pull request's config declarations, read from its pull message's manifests +// against the trunk's manifests the indexer holds. mesh-config reads both in place of the controller's mirror and its +// `pull-declares`; a key's write is its event, nothing polls. + +// The two states' buckets. +var ( + PredefinedBucket = delivery.IndexerModule + "_" + delivery.IndexerPredefinedState + ConfigCandidatesBucket = delivery.IndexerModule + "_" + delivery.IndexerConfigCandidatesState +) + +// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older +// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1). +func (ix *Indexer) writePredefined(ctx context.Context, f delivery.Forge) error { + if ix.Predefined == nil || len(f.Predefined) == 0 { + return nil + } + return ix.notYet(PredefinedBucket, write(ctx, ix.Predefined, delivery.PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) { + if found { + var cur delivery.PredefinedKept + if err := json.Unmarshal(value, &cur); err == nil { + if cur.Commit == f.Merge && string(cur.List) == string(f.Predefined) { + return nil, false, nil + } + older, err := ix.older(ctx, f.Repository, f.Merge, cur.Commit) + if err != nil { + return nil, false, err + } + if older { + ix.logf("%s: the predefined list of %.12s is older than the one kept (%.12s): left", f.Repository, f.Merge, cur.Commit) + return nil, false, nil + } + } + } + next, err := json.Marshal(delivery.PredefinedKept{V: delivery.Version, Commit: f.Merge, At: ix.now(), List: f.Predefined}) + return next, false, err + })) +} + +// notYet says a state not on the bus yet (it comes with this delivery, bound when it appears) and lets the message go: +// what it would have written is written at the next message that carries it. +func (ix *Indexer) notYet(bucket string, err error) error { + if errors.Is(err, process.ErrNotYet) { + ix.logf("the state %s is not on the bus yet: nothing is written to it for this message", bucket) + return nil + } + return err +} + +func (ix *Indexer) now() time.Time { + if ix.Now != nil { + return ix.Now() + } + return time.Now() +} + +// PullName is a pull request as mesh-config names it, `owner/repository#number`. +func PullName(repository string, number int) string { return repository + "#" + strconv.Itoa(number) } + +// DeclarationsOf is a manifest's config declarations by key, in the layout of the controller's declared-config mirror: the +// config form's `config`, else the settings form's `settings`, each default and enum encoded the one way. +func DeclarationsOf(raw string) (map[string]delivery.ConfigDeclaration, error) { + var m struct { + Config map[string]delivery.ConfigDeclaration `json:"config"` + Settings map[string]delivery.ConfigDeclaration `json:"settings"` + } + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return nil, err + } + declared := m.Config + if declared == nil { + declared = m.Settings + } + out := map[string]delivery.ConfigDeclaration{} + for key, d := range declared { + for _, field := range []*json.RawMessage{&d.Default, &d.Enum} { + if len(*field) == 0 { + continue + } + var v any + if err := json.Unmarshal(*field, &v); err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + canonical, err := json.Marshal(v) + if err != nil { + return nil, err + } + *field = canonical + } + out[key] = d + } + return out, nil +} + +// changedDeclarations are the declarations head adds or changes against trunk. +func changedDeclarations(head, trunk map[string]delivery.ConfigDeclaration) map[string]delivery.ConfigDeclaration { + out := map[string]delivery.ConfigDeclaration{} + for key, d := range head { + if before, there := trunk[key]; !there || !reflect.DeepEqual(before, d) { + out[key] = d + } + } + return out +} + +// trunkManifest is the manifest the indexer holds for a module on the trunk, "" when it holds none. +func (ix *Indexer) trunkManifest(ctx context.Context, held map[string]delivery.Module, module string) (string, error) { + m, ok := held[module] + if !ok || m.Manifest == "" || ix.Manifests == nil { + return "", nil + } + raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest) + if err != nil || !found { + return "", err + } + kept, err := delivery.DecodeManifest(raw) + if err != nil { + return "", err + } + return kept.JSON, nil +} + +// candidateDeclares are, per module of a pull request's head, the declarations its manifest adds or changes against the +// trunk's (an empty set for a module that adds none): heads are the head's manifests by module. +func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]delivery.Module, heads map[string]string) (map[string]map[string]delivery.ConfigDeclaration, error) { + out := map[string]map[string]delivery.ConfigDeclaration{} + for module, raw := range heads { + head, err := DeclarationsOf(raw) + if err != nil { + return nil, fmt.Errorf("%w: the manifest of %s at the head: %v", ErrUnreadable, module, err) + } + trunkRaw, err := ix.trunkManifest(ctx, held, module) + if err != nil { + return nil, err + } + trunk := map[string]delivery.ConfigDeclaration{} + if trunkRaw != "" { + if trunk, err = DeclarationsOf(trunkRaw); err != nil { + return nil, fmt.Errorf("the trunk's manifest of %s: %w", module, err) + } + } + if changed := changedDeclarations(head, trunk); len(changed) > 0 { + out[module] = changed + } + } + return out, nil +} + +// writeConfigCandidate writes a pull request's key of config-candidates from its head's manifests by module: written when +// what it says differs (its time apart), never for the same head and declarations twice. +func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string, number int, head string, held map[string]delivery.Module, heads map[string]string) error { + if ix.ConfigCandidates == nil { + return nil + } + pull := PullName(repository, number) + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + return fmt.Errorf("%w: %v", ErrUnreadable, err) + } + declares, err := ix.candidateDeclares(ctx, held, heads) + if err != nil { + return err + } + return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + var cur delivery.ConfigCandidate + if json.Unmarshal(value, &cur) == nil && cur.Head == head && reflect.DeepEqual(cur.Declares, declares) { + return nil, false, nil + } + } + next, err := json.Marshal(delivery.ConfigCandidate{V: delivery.Version, Pull: pull, Head: head, Declares: declares, At: ix.now()}) + return next, false, err + })) +} + +// dropConfigCandidate deletes a pull request's key: it merged or closed. +func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, number int) error { + if ix.ConfigCandidates == nil { + return nil + } + key, err := delivery.ConfigCandidateKey(PullName(repository, number)) + if err != nil { + return nil + } + return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil })) +} + +// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written, +// a closed one deleted. A head held again without its directories leaves the key as it is. +func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, held map[string]delivery.Module) error { + if f.Action == delivery.PullClosed { + return ix.dropConfigCandidate(ctx, f.Repository, f.Number) + } + if !f.DirsKnown { + return nil + } + heads := map[string]string{} + for dir, raw := range f.Manifests { + heads[moduleOfDir(held, f, dir)] = raw + } + return ix.writeConfigCandidate(ctx, f.Repository, f.Number, f.Head, held, heads) +} + +// configCandidatesAfterMerge writes again every open pull request's key that names a module the merge changed, since what +// it adds is judged against the trunk (ADR 0565 §2), from the head's manifests its candidate keeps; and deletes the merged +// pull request's. +func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Forge, changed []string) error { + if ix.ConfigCandidates == nil { + return nil + } + if err := ix.dropConfigCandidate(ctx, f.Repository, f.Number); err != nil { + return err + } + if len(changed) == 0 || ix.Store == nil || ix.Manifests == nil { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + if _, err := ix.ConfigCandidates.Keys(ctx); errors.Is(err, process.ErrNotYet) { + return nil + } + groups, err := ix.Store.Keys(ctx) + if err != nil { + return err + } + var errs []error + for _, g := range groups { + raw, _, found, err := ix.Store.Get(ctx, g) + if err != nil || !found { + continue + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + continue + } + for _, p := range c.Pulls { + if !touchesAny(p, changed) { + continue + } + heads := map[string]string{} + for module, fp := range p.Fps { + mraw, _, found, err := ix.Manifests.Get(ctx, fp.Manifest) + if err != nil || !found { + continue + } + if kept, err := delivery.DecodeManifest(mraw); err == nil { + heads[module] = kept.JSON + } + } + if err := ix.writeConfigCandidate(ctx, p.Repository, p.Number, p.Head.Head, held, heads); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func touchesAny(p delivery.CandidatePull, modules []string) bool { + for _, m := range modules { + if _, ok := p.Fps[m]; ok { + return true + } + } + return false +} + +// writtenModules are the modules a merged message wrote, from its written keys (`modules.`). +func writtenModules(written []string) []string { + var out []string + for _, k := range written { + out = append(out, strings.TrimPrefix(k, "modules.")) + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go new file mode 100644 index 0000000..119a7fe --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -0,0 +1,88 @@ +package index + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// 1953/D2, D3 (novox/hq ADR 0565 §2): the indexer keeps the catalogue's predefined list from the merged message, a merge older +// than the one kept never replacing it; and each open pull request's config declarations its head adds or changes against the +// trunk's manifest, an empty set when none, written only when they change and deleted when it merges or closes. +func TestTheControlRecords(t *testing.T) { + ctx := context.Background() + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + t.Run("the predefined list, never replaced by an older merge", func(t *testing.T) { + ix, _, _ := manifestsIndexer(t) + pre := newMem() + ix.Predefined, ix.Now = pre, func() time.Time { return at } + f := mergedMsg(catalogueRepo, map[string]string{"modules/a": "t1"}) + f.Predefined = json.RawMessage(`{"font-size":{"type":"integer","value":13}}`) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(pre.vals[delivery.PredefinedListKey], &kept); err != nil || kept.Commit != merge || string(kept.List) != string(f.Predefined) { + t.Fatalf("kept = %+v (%v)", kept, err) + } + writes := pre.writes + if err := ix.Apply(ctx, f); err != nil || pre.writes != writes { + t.Errorf("the same list at the same merge wrote again (%v)", err) + } + older := f + older.Merge, older.Predefined = strings.Repeat("e", 40), json.RawMessage(`{}`) + ix.Contains = func(context.Context, string, string, string) (bool, error) { return true, nil } + if err := ix.writePredefined(ctx, older); err != nil || pre.writes != writes { + t.Errorf("an older merge replaced the list (%v)", err) + } + ix.Predefined = &memStore{unbound: process.ErrNotYet} + if err := ix.writePredefined(ctx, f); err != nil { + t.Errorf("a state not on the bus yet stopped the message: %v", err) + } + }) + t.Run("a pull request's config declarations against the trunk", func(t *testing.T) { + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + cands := newMem() + ix.ConfigCandidates, ix.Now = cands, func() time.Time { return at } + trunk := `{"module":"avahi","version":"1","config":{"port":{"type":"integer","default":5353,"why":"mDNS"}}}` + if err := ix.Apply(ctx, withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t1"}), map[string]string{"modules/avahi": trunk}, nil)); err != nil { + t.Fatal(err) + } + key := "novox.mesh-catalog.7" + head := strings.Replace(trunk, `"why":"mDNS"}`, `"why":"mDNS"},"name":{"type":"string","default":"x","why":"shown"}`, 1) + p := withManifests(pull(catalogueRepo, 7, "feat/y", h1, "modules/avahi"), map[string]string{"modules/avahi": head}, nil) + if err := ix.Apply(ctx, p); err != nil { + t.Fatal(err) + } + var c delivery.ConfigCandidate + if err := json.Unmarshal(cands.vals[key], &c); err != nil || c.Pull != "novox/mesh-catalog#7" || c.Head != h1 || len(c.Declares["avahi"]) != 1 || + string(c.Declares["avahi"]["name"].Default) != `"x"` { + t.Fatalf("candidate = %+v (%v)", c, err) + } + writes := cands.writes + if err := ix.Apply(ctx, p); err != nil || cands.writes != writes { + t.Errorf("the same head wrote again (%v)", err) + } + same := withManifests(pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40), "modules/avahi"), map[string]string{"modules/avahi": trunk}, nil) + if err := ix.Apply(ctx, same); err != nil { + t.Fatal(err) + } + c = delivery.ConfigCandidate{} + if err := json.Unmarshal(cands.vals[key], &c); err != nil || len(c.Declares) != 0 || c.Declares == nil { + t.Errorf("a head declaring nothing new = %+v, want an empty set", c) + } + closed := pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40)) + closed.Action = delivery.PullClosed + if err := ix.Apply(ctx, closed); err != nil { + t.Fatal(err) + } + if _, ok := cands.vals[key]; ok { + t.Error("a closed pull request's declarations stayed") + } + }) +} diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 3dd794d..e6f9da4 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -2,7 +2,6 @@ package index import ( "context" - "encoding/json" "strings" "testing" @@ -85,14 +84,7 @@ func (m *memStore) Delete(_ context.Context, key string, rev uint64) error { // the controller's encoder; copied here, the vendored SDK does not carry testdata) has it, with the repository, module and path changed. func buildRecord(t *testing.T, repo, module, path, at string) []byte { t.Helper() - raw := []byte(`{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`) - var m map[string]any - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - m["repository"], m["module"], m["path"], m["at"] = repo, module, path, at - b, _ := json.Marshal(m) - return b + return logged(t, "build-"+module, module, repo, path, "", "", at) } func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge { @@ -104,9 +96,9 @@ func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) { t.Helper() st, logs := newMem(), &strings.Builder{} b := NewBuilds() - b.Put(delivery.BuildKey("a", "build-1"), buildRecord(t, "novox/mesh-delivery", "a", "modules/a", "2026-10-19T09:00:00Z")) - b.Put(delivery.BuildKey("c", "build-2"), buildRecord(t, "novox/mesh-catalog", "c", "modules/c", "2026-10-19T09:00:00Z")) - b.Put(delivery.BuildKey("odd", "build-3"), buildRecord(t, "novox/mesh-delivery", "odd-name", "modules/odd", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-delivery", "a", "modules/a", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-catalog", "c", "modules/c", "2026-10-19T09:00:00Z")) + _ = b.Put(buildRecord(t, "novox/mesh-delivery", "odd-name", "modules/odd", "2026-10-19T09:00:00Z")) return &Indexer{Store: st, Builds: b, Logf: func(f string, a ...any) { logs.WriteString(strings.TrimSpace(strings.ReplaceAll(f, "\n", " ")) + "\n") _ = a diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index 7aacda2..fe8c43a 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -2,6 +2,7 @@ package index import ( "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -23,13 +24,10 @@ const ( ) func liveRecord(id, module, repository, ref, commit, path, at, source string) [2]any { - raw := `{"id":"` + id + `","repository":"` + repository + `",` - if ref != "" { - raw += `"ref":"` + ref + `",` - } - raw += `"module":"` + module + `","commit":"` + commit + `","path":"` + path + `","at":"` + at + `","source-fingerprint":"` + source + - `","source-inputs":{"Module":"` + module + `",` + liveInputsTail + `}` - return [2]any{delivery.BuildKey(module, id), []byte(raw)} + when, _ := time.Parse(time.RFC3339, at) + raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: source, Repository: repository, + Directory: path, Commit: commit, Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone}) + return [2]any{module, raw} } func liveBuilds() [][2]any { diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 08455bb..064dc48 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -42,7 +42,12 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri return f } -func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 +func TestAMergedManifestIsKeptOnceByItsDigestAndNamedByTheModule(t *testing.T) { + t.Run("kept by its digest and named by the module", mergedManifestIsKeptByItsDigestAndNamedByTheModule) + t.Run("the same manifest applied again is written once", sameManifestAppliedAgainIsWrittenOnce) +} + +func mergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 avahi := catalogueManifest(t, "avahi") ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) @@ -63,7 +68,7 @@ func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // } } -func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 +func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 avahi := catalogueManifest(t, "avahi") ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index e65bc31..d090f08 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -2,12 +2,12 @@ package index import ( "context" - "encoding/json" "errors" "fmt" "path" "strings" "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -15,29 +15,38 @@ import ( const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -// record is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is -// the captured shape): SourceInputs under its Go field names, no json keys. inputs is that object's JSON. -func record(t *testing.T, id, module, repo, path, ref, state, at, inputs string) (string, []byte) { +// logged is one entry of the builder's log (mesh-sdk go/delivery BuildLogEntry, as the builder appends it): a build of +// module from repo's directory path at ref ("" for the trunk), its outcome ("" done, "failed", or "asked" for a build only +// started), at the time at; id makes its fingerprint. +func logged(t *testing.T, id, module, repo, path, ref, outcome, at string) []byte { t.Helper() - // an outcome has no state key (mesh-controller internal/chain/records.go:57-59); state asked is a request - raw := `{"id":"` + id + `",` - switch state { - case "": - case "failed": // a failure is an outcome with failed set and no state (mesh-controller internal/chain/records.go:109) - raw += `"failed":"the build failed",` - default: - raw += `"state":"` + state + `",` - } - raw += `"repository":"` + repo + `","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"` + path + `","at":"` + at + `"` - if inputs != "" { - raw += `,"source-fingerprint":"src1:ignored","source-inputs":` + inputs - } - raw += `}` - var check delivery.BuildRecord - if err := json.Unmarshal([]byte(raw), &check); err != nil { + when, err := time.Parse(time.RFC3339, at) + if err != nil { t.Fatal(err) } - return delivery.BuildKey(module, id), []byte(raw) + if path == "" { + path = delivery.RootPath + } + e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + id, Repository: repo, Directory: path, + Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone} + switch outcome { + case "": + case "failed": + e.Outcome = delivery.BuildFailed + default: + e.Event, e.Outcome = delivery.BuildStartedEvent, "" + } + raw, err := delivery.EncodeBuildLogEntry(e) + if err != nil { + t.Fatal(err) + } + return raw +} + +// record is logged under the module's log key, as the tests' tables carry it. +func record(t *testing.T, id, module, repo, path, ref, state, at, _ string) (string, []byte) { + t.Helper() + return module, logged(t, id, module, repo, path, ref, state, at) } func modulesIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore) { @@ -45,7 +54,9 @@ func modulesIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore) { mods := newMem() ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()} for _, r := range recs { - ix.Builds.Put(r[0].(string), r[1].([]byte)) + if err := ix.Builds.Put(r[1].([]byte)); err != nil { + t.Fatal(err) + } } return ix, mods } diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index 78101a4..d5751cd 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -302,14 +302,14 @@ func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""), rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""), } { - b.Put(r[0].(string), r[1].([]byte)) + _ = b.Put(r[1].([]byte)) } got := b.RegistrationsOf("photos") if len(got) != 1 || got[0].Ref != "nox-mesh" { t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got) } k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "") - b.Put(k, v) // a trunk's ref beats a branch of its own + _, _ = k, b.Put(v) // a trunk's ref beats a branch of its own if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" { t.Fatalf("registrations = %+v", got) } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index b9d01be..39b890c 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -46,13 +46,10 @@ const ( ) func agentBuild(id, src, at string) [2]any { - commit := "cada7e3c" - if id == "build-1792587792086548572" { - commit = "f10fbda1" - } - raw := `{"id":"` + id + `","repository":"` + agentCloneURL + `","ref":"main","module":"build-agent","commit":"` + commit + `",` + - `"path":"modules/build-agent","at":"` + at + `","source-fingerprint":"` + src + `","source-inputs":` + aInputs + `}` - return [2]any{delivery.BuildKey("build-agent", id), []byte(raw)} + when, _ := time.Parse(time.RFC3339, at) + raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "build-agent", FP: src, Repository: agentCloneURL, + Directory: "modules/build-agent", Commit: "cada7e3c", Node: "n1", At: when, Outcome: delivery.BuildDone}) + return [2]any{"build-agent", raw} } type mod struct { @@ -184,7 +181,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2) } a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z") - ix.Builds.Put(a[0].(string), a[1].([]byte)) + _ = ix.Builds.Put(a[1].([]byte)) if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index e05aab4..161bf20 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -86,10 +86,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want none", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 544e452..6b26c33 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -16,14 +16,16 @@ { "name": "heard", "history": 64 }, { "name": "manifests", "history": 1 }, { "name": "modules", "history": 20 }, + { "name": "predefined", "history": 5 }, + { "name": "config-candidates", "history": 5 }, { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 } ], "reads": [ "mesh-build-go.toolchains", "mesh-build-node.toolchains", - "mesh-controller.assignments", - "mesh-controller.builds", + "mesh-delivery-assigner.placement", + "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index 4135620..0aef26d 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index 6245202..a2b8e01 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go index 1e0c713..a9ae081 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go @@ -19,7 +19,11 @@ import ( func serve(t *testing.T) string { t.Helper() m := view.New() - m.Apply(view.Entry{Bucket: view.AssignmentsBucket, Key: "a.x", Value: []byte(`{"node":"a","module":"x","build":"b1"}`)}) + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "x"}}}) + if err != nil { + t.Fatal(err) + } + m.Apply(view.Entry{Bucket: delivery.PlacementBucket, Key: "placement.a", Value: placed}) dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c") if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index d76f73b..2af9092 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strings" + "time" "mesh-delivery-view/internal/view" @@ -19,6 +20,14 @@ import ( // Tools are the read tools over a modules state, and build_usage over the build modules' states. func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { return []stdio.Tool{ + {Name: view.StatusTool, + Description: "The mesh's status, derived by the view from the buckets it reads (novox/hq ADR 0564 §10), nothing written as a " + + "summary: every open condition of the condition keeper (mesh-conditions), urgent first then oldest; what is quiet (each " + + "node the watchman says is offline, each key it says stopped moving); each module whose want and have differ on a node " + + "(moving, or stuck with the have's or the watchman's words), as mesh-sdk's delivery motion derives it; and what " + + "mesh-healers did in the last seven days, by outcome. A part that cannot be read says so, never as none; well is true " + + "only when nothing is open, quiet or stuck and every part was read. Read-only.", + Run: func(map[string]any) (any, error) { return r.Status(time.Now()), nil }}, {Name: view.ReachReadTool, Description: "Whether a merge is on the machines: per repository, the reached head (the newest merge every node runs a build " + "containing, as the view derives it from the indexer's modules, the wants and the haves; a merge at or before it is on the " + @@ -95,6 +104,23 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { group, _ := args["group"].(string) return r.Candidate(group) }}, + {Name: view.PredefinedReadTool, + Description: "The catalogue's predefined list as the indexer keeps it (novox/hq ADR 0565 §2): its key list, the list whole as " + + "predefined.json holds it with the merge it was read at; a merge older than that never replaces it. mesh-config " + + "publishes the predefined keys from it after its take-over. Read-only.", + Run: func(map[string]any) (any, error) { + return view.ControlRecord(r.PredefinedState, delivery.IndexerPredefinedState, delivery.PredefinedListKey) + }}, + {Name: view.ConfigCandidatesReadTool, + Description: "Each open pull request's config declarations as the indexer keeps them (novox/hq ADR 0565 §2): per pull " + + "request (owner.repository.number), its head and per module the config keys its manifests add or change against the " + + "trunk's, an empty set when none; deleted when it merges or closes. Give key for one. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "one pull request's key, owner.repository.number"}}}, + Run: func(args map[string]any) (any, error) { + key, _ := args["key"].(string) + return view.ControlRecord(r.ConfigCandidatesState, delivery.IndexerConfigCandidatesState, key) + }}, {Name: view.CandidatesReadTool, Description: "Every candidate the indexer holds, by group, each as stored under candidates.; with module, only those with a " + "pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.", @@ -119,6 +145,27 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { limit, _ := args["limit"].(float64) return r.BuilderBuilds(module, state, int(limit)) }}, + {Name: view.BuildHistoryTool, + Description: "Every build that ended since a time (default the last 24 hours), oldest first, from the builder's log build-log (kept 30 " + + "days): its module, fingerprint, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + + "queued), the manifest, the bases it was built against, each artifact made, and seconds, how long it took from its start (absent " + + "when the start is older than since). With module, that module's builds only; more says the log held more: ask again from the " + + "last build's time. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's builds"}, + "since": map[string]any{"type": "string", "description": "RFC 3339 time; default 24 hours ago"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + since := time.Now().Add(-view.DefaultHistorySince) + if raw, _ := args["since"].(string); strings.TrimSpace(raw) != "" { + at, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)) + if err != nil { + return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", raw, err) + } + since = at + } + return r.BuildHistory(module, since) + }}, {Name: view.BuilderBuildReadTool, Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " + "at the fingerprint the indexer holds for it now, joined through modules.; with commit, only if the indexer holds the module at that commit (a module's value names the commit its fingerprint first appeared at, and stays there through merges that leave the module alone). Read-only.", @@ -166,6 +213,15 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { node, module := nodeModuleOf(args) return r.Haves(node, module) }}, + {Name: view.TrustsReadTool, + Description: "What every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2): each node's record's trusts " + + "(from node-engine_have .node), whether it names the vault's signing key as mesh-vault publishes it in its " + + "state signers and whether it still names the controller's, the nodes that do not name the vault's yet (not_yet), " + + "and every: true once all do, the gate before the controller's lease is handed away. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return r.Trusts() + }}, } } @@ -223,8 +279,14 @@ func read(state stdio.KeptState) view.StateRead { func main() { r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")), - ReachState: read(stdio.State("reach")), - HeardState: read(stdio.State("mesh-delivery-indexer.heard"))} + ReachState: read(stdio.State("reach")), + HeardState: read(stdio.State("mesh-delivery-indexer.heard")), + SignersState: read(stdio.State("mesh-vault.signers")), + ConditionsState: read(stdio.State(view.StatusConditionsState)), AttemptsState: read(stdio.State(view.StatusAttemptsState)), + PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState)), + PredefinedState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerPredefinedState)), + ConfigCandidatesState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState)), + BuildLog: stdio.Log(delivery.BuildLogRead).Since} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 3cb27d2..0e230ce 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -1,13 +1,18 @@ package main import ( + "crypto/ed25519" "encoding/json" "fmt" + "reflect" "strings" "sync/atomic" "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-view/internal/view" ) @@ -173,6 +178,28 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) { t.Errorf("%s filter = %+v, want %s", args, got, want) } } + // 1955/L7 (ADR 0571 clause 1): builder_history answers each end from the builder's log with its duration from its start, + // the start of a later build of the same fingerprint not counted, and narrows by module through the log's key. + t0 := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + entry := func(event string, at time.Duration) []byte { + e := delivery.BuildLogEntry{Event: event, Module: "netcheck", FP: fp2A, Repository: "novox/mesh-catalog", Directory: "modules/netcheck", + Commit: strings.Repeat("7", 40), Node: "ace", At: t0.Add(at)} + if event == delivery.BuildEndedEvent { + e.Outcome = delivery.BuildDone + } + raw, _ := delivery.EncodeBuildLogEntry(e) + return raw + } + var asked string + r.BuildLog = func(at time.Time, keys string, limit int) (*stdio.LogPage, error) { + asked = keys + return &stdio.LogPage{Entries: []stdio.LogEntry{{Sequence: 1, Value: entry(delivery.BuildStartedEvent, 0)}, + {Sequence: 2, Value: entry(delivery.BuildEndedEvent, 90*time.Second)}, {Sequence: 3, Value: entry(delivery.BuildStartedEvent, time.Hour)}}}, nil + } + out, err = run(t, r, view.BuildHistoryTool, map[string]any{"module": "netcheck", "since": t0.Format(time.RFC3339)}) + if h, _ := out.(view.HistoryAnswer); err != nil || len(h.Builds) != 1 || h.Builds[0].Seconds != 90 || h.Builds[0].Outcome != delivery.BuildDone || asked != "netcheck" || h.More { + t.Errorf("history = %+v, %v (keys %q), want the one end at 90 s", out, err, asked) + } } // 1723/AC2: builder_build answers one key by module and fp2, and by module alone through the indexer's fp2 (joined at its commit). @@ -438,6 +465,41 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) { t.Errorf("haves of ace (the node's own key too) = %v", got) } + // L3 s6b (ADR 0550 §2): node_trusts answers for every node in one call whether its record names the vault's key the + // vault publishes, and every is true only once all do. + vaultPub, _, _ := ed25519.GenerateKey(nil) + vaultKey, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now())) + vaultFP := delivery.SignerFingerprint(vaultPub) + record := func(node string, trusts ...string) string { + raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{V: delivery.Version, Record: &delivery.NodeRecord{Node: node, Trusts: trusts}}) + return string(raw) + } + for _, row := range []struct { + name string + haves fake + every bool + notYet []string + }{ + {"one node trusts the controller's alone", fake{"ace.node": record("ace", "controller key sha256:aa", "vault key "+vaultFP), + "g14.node": record("g14", "controller key sha256:aa"), "ace.zsh": `{"fp":"x"}`}, false, []string{"g14"}}, + {"every node trusts the vault's key", fake{"ace.node": record("ace", "vault key "+vaultFP), + "g14.node": record("g14", "controller key sha256:aa", "vault key "+vaultFP)}, true, []string{}}, + } { + tr := view.Reader{HavesState: row.haves.reader().State, SignersState: fake{"key": string(vaultKey)}.reader().State} + out, err := run(t, tr, view.TrustsReadTool, map[string]any{}) + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + a := out.(view.TrustsAnswer) + if a.Every != row.every || !reflect.DeepEqual(a.NotYet, row.notYet) || a.VaultKey != vaultFP || len(a.Nodes) != 2 { + t.Errorf("%s: %+v", row.name, a) + } + } + if out, err := run(t, view.Reader{HavesState: fake{"ace.node": record("ace", "vault key "+vaultFP)}.reader().State, + SignersState: fake{}.reader().State}, view.TrustsReadTool, map[string]any{}); err != nil || out.(view.TrustsAnswer).Every { + t.Errorf("with no key published: %+v, %v", out, err) + } + // More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list // nothing; the whole answer reads MaxStateReads values in parallel and says it was cut. slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) }) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 55eedf8..3cbf53e 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -48,7 +48,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -68,11 +68,20 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 } return b } - put(view.AssignmentsBucket, "a.x", []byte(`{"node":"a","module":"x","build":"b1"}`)) - put(view.AssignmentsBucket, "a.y", []byte(`{"node":"a","module":"y","build":"b2"}`)) - put(view.AssignmentsBucket, "b.x", []byte(`{"node":"b","module":"x","build":"b1","ended":"2026-10-18T11:30:00Z"}`)) - put(view.BuildsBucket, "x.build.b1", []byte(`{"id":"b1","module":"x"}`)) - put(view.BuildsBucket, "x.newest", []byte("b1")) + // The seat's placement (issue 774, ADR 0549 §3): x and y on a, nothing on b. + placeValue := func(node string, modules ...string) []byte { + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + v, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + return v + } + put(delivery.PlacementBucket, "placement.a", placeValue("a", "x", "y")) + put(delivery.PlacementBucket, "placement.b", placeValue("b")) put(delivery.HaveBucket, "a.x", haveValue("b1", delivery.PhaseChecking, delivery.HealthWaits)) put(delivery.HaveBucket, "b.x", haveValue("b1", delivery.PhaseRunning, delivery.HealthOK)) @@ -150,7 +159,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}) if err != nil { t.Fatal(err) @@ -236,7 +245,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} { + for _, b := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index ba9c342..a9953e7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -1,4 +1,4 @@ -// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, assignments +// Command mesh-delivery-view is the operator's view of delivery (novox/hq ADR 0429 §3): it watches the have, the seat's placement // and builds buckets on its own bus credential, derives one row per module and node in memory, and serves them on a // local socket (for mesh-delivery-cli) and on a local web page. It starts, stops or retries nothing; the one thing it writes is the operator's answer to an ask that needs no proof of who // answered, under `.view` in its bucket answers (ADR 0528 clause 6). @@ -106,7 +106,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { + for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { go follow(ctx, js, bucket, false, model, logger) } // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 2d50f31..3e7e9f0 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 6245202..a2b8e01 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/fingerprints_test.go b/modules/mesh-delivery-view/internal/view/fingerprints_test.go index d4faa9e..3004d0d 100644 --- a/modules/mesh-delivery-view/internal/view/fingerprints_test.go +++ b/modules/mesh-delivery-view/internal/view/fingerprints_test.go @@ -50,30 +50,6 @@ func modEntry(t *testing.T, name string, m delivery.Module) Entry { return Entry{Bucket: delivery.ModulesBucket, Key: name, Value: b, At: t0} } -// trunkBuild is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is -// the captured shape: id, state, repository, ref, module, commit, path, source-fingerprint, source-inputs, at). -func trunkBuild(module, id, ref, state, fp, at string) []Entry { - // an outcome has no state key; a failed one has "failed" (mesh-controller internal/chain/records.go:57-59, :109) - rec := `{"id":"` + id + `",` - switch state { - case "": - case "failed": - rec += `"failed":"the build failed",` - default: - rec += `"state":"` + state + `",` - } - rec += `"repository":"novox/mesh-delivery","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/` + module + `","at":"` + at + `"` - if fp != "" { - // the controller writes the inputs whenever it writes the fingerprint (mesh-sdk testdata/build-record-build-agent.json) - rec += `,"source-fingerprint":"` + fp + `","source-inputs":{"Module":"` + module + `","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":""}` - } - rec += `}` - return []Entry{ - {Bucket: BuildsBucket, Key: module + ".build." + id, Value: []byte(rec), At: t0}, - {Bucket: BuildsBucket, Key: module + ".newest", Value: []byte(id), At: t0}, - } -} - func fpRow(t *testing.T, s Snapshot, module string) FingerprintRow { t.Helper() for _, f := range s.Fingerprints { @@ -111,62 +87,27 @@ func TestSnapshotListsTheCandidateGroups(t *testing.T) { // 1241/AC1 } } -func TestFingerprintAgainstTheNewestTrunkBuild(t *testing.T) { // 1241/AC2 +// 1955/L7 (ADR 0571 §1): the view reads no build record of the controller's: a module with no fingerprint of the new +// form is said so, never compared with the controller's builds. +func TestAModuleOfTheOldFormIsComparedWithNoBuild(t *testing.T) { m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...) - fill(m, trunkBuild("d", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...) fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "d", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "b", delivery.Module{None: "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4", Repository: "novox/mesh-delivery", Commit: hA}), - modEntry(t, "e", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA})) + modEntry(t, "b", delivery.Module{None: "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4", Repository: "novox/mesh-delivery", Commit: hA})) s := m.Snapshot() - if r := fpRow(t, s, "a"); r.State != FPSame || r.Indexer != fF || r.Build != fF { - t.Errorf("a = %+v, want same", r) - } - if r := fpRow(t, s, "d"); r.State != FPDiffers || r.Indexer != fG || r.Build != fF { - t.Errorf("d = %+v, want differs with both", r) - } - if r := fpRow(t, s, "b"); r.State != FPNone || !strings.Contains(r.Reason, "release 4") || r.Indexer != "" { - t.Errorf("b = %+v, want none with the reason", r) - } - if r := fpRow(t, s, "e"); r.State != FPNoBuild || r.Indexer != fG || r.Build != "" { - t.Errorf("e = %+v, want no build with the indexer's alone", r) - } -} - -func TestOnlyTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1241/AC2 (the newest key is only a hint) - m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fG, "2026-10-19T09:00:00Z")...) - fill(m, trunkBuild("a", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...) - fill(m, trunkBuild("a", "build-3", "main", "failed", fG, "2026-10-19T11:00:00Z")...) // newer, not built - fill(m, trunkBuild("a", "build-4", "feat/x", "", fG, "2026-10-19T12:00:00Z")...) // newer, not trunk; also the newest key - fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "a"); r.State != FPSame || r.Build != fF { - t.Errorf("a = %+v, want same with the 10:00 trunk build's fingerprint", r) - } -} - -func TestBuildFingerprintIsReadWithTheSDKRecord(t *testing.T) { // 1241/AC3 - // The record as the controller's encoder wrote it (mesh-sdk go/delivery/testdata/build-record-build-agent.json), whole, - // with its source-inputs under their Go field names: the view reads source-fingerprint from it. - const rec = `{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","manifest-digest":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","source-inputs":{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""},"made":[{"name":"server","kind":"image","reference":"registry.example/build-agent@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"}],"at":"2026-10-19T09:38:00Z"}` - const fp = "src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7" - m := New() - fill(m, Entry{Bucket: BuildsBucket, Key: delivery.BuildKey("build-agent", "build-1792395447088936520"), Value: []byte(rec), At: t0}, - Entry{Bucket: BuildsBucket, Key: delivery.NewestKey("build-agent"), Value: []byte("build-1792395447088936520"), At: t0}, - modEntry(t, "build-agent", delivery.Module{FP: fp, Repository: "novox/mesh-controller", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "build-agent"); r.State != FPSame || r.Build != fp { - t.Errorf("build-agent = %+v, want same with %s", r, fp) + for _, name := range []string{"a", "b"} { + if r := fpRow(t, s, name); r.State != FPNone || !(strings.Contains(r.Reason, "new form") || strings.Contains(r.Reason, "release 4")) || r.Build != "" { + t.Errorf("%s = %+v, want none: no build of the builder's is compared", name, r) + } } } func shown(t *testing.T) *Model { m := New() - fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...) fill(m, candidate(t, "feat/x", pullOf("novox/mesh-delivery", 12, hA, "a"), pullOf("novox/mesh-catalog", 300, hB, "c")), - modEntry(t, "a", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}), + modEntry(t, "a", delivery.Module{FP2: fp2A, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "a", fp2A, delivery.BuilderBuild{State: delivery.BuildFailed, Error: "the compile failed"}), modEntry(t, "b", delivery.Module{None: "no tree", Repository: "novox/mesh-delivery", Commit: hA})) return m } @@ -175,14 +116,14 @@ func TestTableAndPagePrintTheGroupsAndTheComparison(t *testing.T) { // 1241/AC6 m := shown(t) var out bytes.Buffer Table(&out, m.Snapshot()) - for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "novox/mesh-catalog#300", "FINGERPRINT", "differs", "indexer " + fG, "build " + fF, "no tree"} { + for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "novox/mesh-catalog#300", "FINGERPRINT", "the compile failed", fp2A, "no tree"} { if !strings.Contains(out.String(), want) { t.Errorf("the table lacks %q:\n%s", want, out.String()) } } rec := httptest.NewRecorder() Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "FINGERPRINTS", "differs", fG, fF, "no tree"} { + for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "FINGERPRINTS", "the compile failed", fp2A, "no tree"} { if !strings.Contains(rec.Body.String(), want) { t.Errorf("the page lacks %q:\n%s", want, rec.Body.String()) } @@ -198,23 +139,3 @@ func TestUnreadableIndexerValuesAreNamed(t *testing.T) { // never dropped silent t.Errorf("snapshot = %+v", s) } } - -func TestAPlanAskedBuildWithNoRefIsATrunkBuild(t *testing.T) { // 1281/AC5 - // The build record of module mesh-delivery from mesh-catalog, build-1792435539606203626, as `mesh-controller.command - // builds` showed it on 2026-10-19 at about 21:20 local: built on ace at 20:45 local, the repository the clone URL, the - // source fingerprint below, and no ref (a delivery plan asked it after a merge; the listing shows no "at main"). - // No reading command shows the whole record: its path is the module's directory in the catalogue, and its - // source-inputs are mailu's live ones (mesh-sdk go/delivery/fingerprint_test.go) with the module's name put in. - const live = "src1:f79e2eedc2689cf6b3591ebfdb81ec65899131122fb2fe3dd9e384091d8a2a6d" - const rec = `{"id":"build-1792435539606203626","repository":"http://novox.internal:20000/novox/mesh-catalog.git",` + - `"module":"mesh-delivery","commit":"9efdd87a","path":"modules/mesh-delivery","at":"2026-10-19T18:45:00Z","source-fingerprint":"` + live + `",` + - `"source-inputs":{"Module":"mesh-delivery","Tree":"cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,"Toolchains":null,"Unpinned":""}}` - m := New() - fill(m, - Entry{Bucket: BuildsBucket, Key: "mesh-delivery.build.build-1792435539606203626", Value: []byte(rec), At: t0}, - Entry{Bucket: BuildsBucket, Key: "mesh-delivery.newest", Value: []byte("build-1792435539606203626"), At: t0}, - modEntry(t, "mesh-delivery", delivery.Module{FP: live, Repository: "novox/mesh-catalog", Commit: hA})) - if r := fpRow(t, m.Snapshot(), "mesh-delivery"); r.State != FPSame || r.Build != live { - t.Errorf("mesh-delivery = %+v, want same as the no-ref build's fingerprint", r) - } -} diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 05ee803..fc9ac7c 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -12,7 +12,9 @@ import ( "strings" "sync" "sync/atomic" + "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -44,6 +46,57 @@ type Reader struct { ReachState StateRead // the view's own reach (ADR 0556) // HeardState is the FORGE messages the indexer handled (issue 1126 note 76); unset reads none. HeardState StateRead + // SignersState is the vault's public signing halves (ADR 0550 §2); unset reads none. + SignersState StateRead + // ConditionsState, AttemptsState, PresenceState and StuckState are what status reads beside the wants and the haves + // (ADR 0564 §10): the condition keeper's open conditions, mesh-healers' attempts, the watchman's presence and stuck keys. + ConditionsState, AttemptsState, PresenceState, StuckState StateRead + // PredefinedState and ConfigCandidatesState are the indexer's control records (ADR 0565 §2, §15); unset reads none. + PredefinedState, ConfigCandidatesState StateRead + // BuildLog reads the builder's log from a time (stdio.Log(delivery.BuildLogRead).Since); unset reads none. + BuildLog LogSince +} + +// LogSince is one page of a log from a time on, at most limit entries of keys (stdio.KeptLog.Since). +type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) + +// The read tools of the indexer's control records (ADR 0565 §15): the indexer serves no tool of its own (ADR 0437 §4). +const ( + PredefinedReadTool = "indexer_predefined" + ConfigCandidatesReadTool = "indexer_config_candidates" +) + +// ControlRecord is every key of one of the indexer's control records as stored, or one key, with how many it holds. +func ControlRecord(s StateRead, name, key string) (any, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("the indexer's %s cannot be read here", name) + } + if key = strings.TrimSpace(key); key != "" { + raw, err := s.Get(key) + if err != nil { + return nil, err + } + if raw == nil { + return nil, fmt.Errorf("the indexer's %s holds no key %s", name, key) + } + return raw, nil + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + values, err := s.values(keys) + if err != nil { + return nil, err + } + return map[string]any{"state": name, "holds": len(keys), "keys": values}, nil +} + +// Status is the view's derivation of status at now (status.go). +func (r Reader) Status(now time.Time) Status { + return DeriveStatus(StatusRead{Conditions: r.ConditionsState, Attempts: r.AttemptsState, Presence: r.PresenceState, + Stuck: r.StuckState, Wants: r.WantsState, Haves: r.HavesState}, now) } // ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556). @@ -649,3 +702,168 @@ func (r Reader) Differences(node, module string) (KeptAnswer, error) { func (r Reader) Haves(node, module string) (KeptAnswer, error) { return r.kept(r.HavesState, "the haves", node, module) } + +// TrustsReadTool reads what every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2; design 47's L3 +// s6b): the gate before the lease write is that every node's record names the vault's key. +const TrustsReadTool = "node_trusts" + +// NodeTrusts is what one node's record says it trusts: its lines (delivery.Trusts) and whether they name the vault's key +// the vault publishes and the controller's key. +type NodeTrusts struct { + Trusts []string `json:"trusts"` + TrustsVault bool `json:"trusts_vault"` + TrustsController bool `json:"trusts_controller"` +} + +// TrustsAnswer is node_trusts' answer: the vault's key as it publishes it, each node's trusts, the nodes whose record does +// not name the vault's key yet, and whether every node does. +type TrustsAnswer struct { + VaultKey string `json:"vault_key,omitempty"` + Nodes map[string]NodeTrusts `json:"nodes"` + NotYet []string `json:"not_yet"` + Every bool `json:"every"` + Unreadable []string `json:"unreadable,omitempty"` + Note string `json:"note,omitempty"` +} + +// Trusts reads every node's record (`.node` of the haves) and the vault's `signers` key, and answers which nodes +// trust the vault's key. Read-only. +func (r Reader) Trusts() (TrustsAnswer, error) { + if r.HavesState.Get == nil || r.HavesState.Keys == nil { + return TrustsAnswer{}, fmt.Errorf("the haves cannot be read here") + } + out := TrustsAnswer{Nodes: map[string]NodeTrusts{}, NotYet: []string{}} + if r.SignersState.Get != nil { + raw, err := r.SignersState.Get(delivery.SignersKeyKey) + if err != nil { + return TrustsAnswer{}, err + } + if len(raw) > 0 { + k, err := delivery.DecodeSignerKey(raw) + if err != nil { + return TrustsAnswer{}, err + } + out.VaultKey = k.Fingerprint + } + } + keys, err := r.HavesState.Keys() + if err != nil { + return TrustsAnswer{}, err + } + var records []string + for _, k := range keys { + if _, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode { + records = append(records, k) + } + } + values, _, err := r.HavesState.read(records) + if err != nil { + return TrustsAnswer{}, err + } + for key, raw := range values { + node := strings.TrimSuffix(key, "."+delivery.HaveNodeWord) + h, err := delivery.DecodeHaveNode(raw) + if err != nil { + out.Unreadable = append(out.Unreadable, key) + continue + } + var t NodeTrusts + if h.Record != nil { + t.Trusts = h.Record.Trusts + } + for _, line := range t.Trusts { + t.TrustsController = t.TrustsController || strings.HasPrefix(line, delivery.SignerController+" key ") + } + t.TrustsVault = out.VaultKey != "" && delivery.TrustsKey(t.Trusts, delivery.SignerVault, out.VaultKey) + out.Nodes[node] = t + if !t.TrustsVault { + out.NotYet = append(out.NotYet, node) + } + } + sort.Strings(out.NotYet) + sort.Strings(out.Unreadable) + out.Every = out.VaultKey != "" && len(out.Nodes) > 0 && len(out.NotYet) == 0 && len(out.Unreadable) == 0 + if out.VaultKey == "" { + out.Note = "the vault publishes no signing key yet (" + delivery.SignersRead + "), so no node can trust it" + } + return out, nil +} + +// BuildHistoryTool is the view's tool of the builds that ended, read from the builder's log (novox/hq ADR 0571 clause 1). +const BuildHistoryTool = "builder_history" + +// The bounds of builder_history: how far back it reads when no since is given, and the entries it reads of the log. +const ( + DefaultHistorySince = 24 * time.Hour + MaxHistoryEntries = 1000 +) + +// HistoryRow is one ended build of the builder's log, with how long it took from its start (none when its start is not in +// what was read). +type HistoryRow struct { + delivery.BuildLogEntry + Seconds float64 `json:"seconds,omitempty"` +} + +// HistoryAnswer is builder_history's answer: every build that ended since, oldest first, and More when the log held more +// entries than one read takes (ask again from the last build's time). +type HistoryAnswer struct { + Since time.Time `json:"since"` + Builds []HistoryRow `json:"builds"` + More bool `json:"more,omitempty"` +} + +// BuildHistory is every build of module (every module when empty) that ended at or after since, oldest first, each with its +// duration from the start the log holds of the same module and fingerprint before it (delivery.BuildDuration). An entry that +// does not read is refused, naming its sequence. +func (r Reader) BuildHistory(module string, since time.Time) (HistoryAnswer, error) { + out := HistoryAnswer{Since: since.UTC(), Builds: []HistoryRow{}} + if r.BuildLog == nil { + return out, fmt.Errorf("the builder's log cannot be read here") + } + keys := "" + if module = strings.TrimSpace(module); module != "" { + k, err := delivery.BuildLogKey(module) + if err != nil { + return out, err + } + keys = k + } + page, err := r.BuildLog(since, keys, MaxHistoryEntries) + if err != nil { + return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuildLogRead, since.Format(time.RFC3339), err) + } + if page == nil { + return out, nil + } + var entries []delivery.BuildLogEntry + for _, le := range page.Entries { + e, err := delivery.DecodeBuildLogEntry(le.Value) + if err != nil { + return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuildLogRead, le.Sequence, err) + } + entries = append(entries, e) + } + starts := map[string][]delivery.BuildLogEntry{} + for _, e := range entries { + if e.Event == delivery.BuildStartedEvent { + starts[e.Module+" "+e.FP] = append(starts[e.Module+" "+e.FP], e) + } + } + for _, e := range delivery.EndedSince(entries, since) { + row := HistoryRow{BuildLogEntry: e} + // the newest start of the same build at or before its end: a fingerprint built again later starts again after it + var start delivery.BuildLogEntry + for _, s := range starts[e.Module+" "+e.FP] { + if !s.At.After(e.At) && s.At.After(start.At) { + start = s + } + } + if d, ok := delivery.BuildDuration(start, e); ok { + row.Seconds = d.Seconds() + } + out.Builds = append(out.Builds, row) + } + out.More = page.Next != 0 + return out, nil +} diff --git a/modules/mesh-delivery-view/internal/view/status.go b/modules/mesh-delivery-view/internal/view/status.go new file mode 100644 index 0000000..d8a0837 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/status.go @@ -0,0 +1,218 @@ +package view + +import ( + "encoding/json" + "fmt" + "sort" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// `status` is the view's derivation (novox/hq ADR 0564 §10, Limits 7, issue 1952 V1): what is wrong, quiet and on its way, +// read from the buckets it watches, never from a summary someone writes (research 072 §2), so the controller's status, its +// summary and the probe D9 go with its lease. Each part comes from one bucket: +// - Conditions: the condition keeper's open conditions (mesh-conditions.conditions), urgent first, then oldest; +// - Quiet: the watchman's presence of each node that is offline, and its stuck keys not moved; +// - Moving: each module whose want and have differ, as mesh-sdk's delivery.MotionOf derives it (moving or stuck); +// - Heals: what mesh-healers did in the last seven days (mesh-healers.attempts), by outcome. +// A bucket that cannot be read says so in its part's Unread, never as none. + +// StatusTool is the view's tool that answers status. +const StatusTool = "view_status" + +// The states status reads, as the view's manifest names them. +const ( + StatusConditionsState = health.KeeperModule + "." + health.ConditionsState + StatusAttemptsState = health.HealersModule + "." + health.AttemptsState + StatusPresenceState = delivery.WatchmanModule + ".presence" + StatusStuckState = delivery.WatchmanModule + ".stuck" +) + +// HealsWithin is how far back status counts the healers' acts. +const HealsWithin = 7 * 24 * time.Hour + +// StatusRead is the five buckets status reads, as a tool reads them. +type StatusRead struct { + Conditions, Attempts, Presence, Stuck, Wants, Haves StateRead +} + +// Status is the view's answer to status. +type Status struct { + Conditions []StatusCondition `json:"conditions"` + ConditionsUnread string `json:"conditions-unread,omitempty"` + Quiet []StatusQuiet `json:"quiet"` + QuietUnread string `json:"quiet-unread,omitempty"` + Moving []StatusMoving `json:"moving"` + MovingUnread string `json:"moving-unread,omitempty"` + Heals map[string]int `json:"heals-this-week"` + HealsUnread string `json:"heals-unread,omitempty"` + // Well says nothing is open, quiet or stuck; a part unread is not well. + Well bool `json:"well"` +} + +// StatusCondition is one open condition in brief. +type StatusCondition struct { + Key string `json:"key"` + Severity string `json:"severity"` + Machine string `json:"machine,omitempty"` + Summary string `json:"summary"` + Raised time.Time `json:"raised"` + Silenced bool `json:"silenced,omitempty"` +} + +// StatusQuiet is a node offline, or a key the watchman says stopped moving. +type StatusQuiet struct { + Key string `json:"key"` + What string `json:"what"` + Since time.Time `json:"since,omitzero"` +} + +// StatusMoving is a module whose want and have differ on a node. +type StatusMoving struct { + Node string `json:"node"` + Module string `json:"module"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Words string `json:"words,omitempty"` + FromFP string `json:"from_fp,omitempty"` + ToFP string `json:"to_fp"` +} + +func values(s StateRead) (map[string]json.RawMessage, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("not read here") + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + out := map[string]json.RawMessage{} + for _, k := range keys { + v, err := s.Get(k) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + if v != nil { + out[k] = v + } + } + return out, nil +} + +// DeriveStatus is status from the buckets r reads, at now. +func DeriveStatus(r StatusRead, now time.Time) Status { + st := Status{Conditions: []StatusCondition{}, Quiet: []StatusQuiet{}, Moving: []StatusMoving{}, Heals: map[string]int{}} + + if raw, err := values(r.Conditions); err != nil { + st.ConditionsUnread = "the condition keeper's conditions cannot be read: " + err.Error() + } else { + var open []health.Condition + for _, v := range raw { + var c health.Condition + if json.Unmarshal(v, &c) == nil && c.Key != "" { + open = append(open, c) + } + } + health.Order(open) + for _, c := range open { + st.Conditions = append(st.Conditions, StatusCondition{Key: c.Key, Severity: string(c.Severity), Machine: c.Subject.Machine, + Summary: c.Summary, Raised: c.Raised, Silenced: c.SilencedAt(now)}) + } + } + + presence, perr := values(r.Presence) + stuck, serr := values(r.Stuck) + switch { + case perr != nil: + st.QuietUnread = "the watchman's presence cannot be read: " + perr.Error() + case serr != nil: + st.QuietUnread = "the watchman's stuck keys cannot be read: " + serr.Error() + default: + for k, v := range presence { + if p, err := delivery.DecodePresence(v); err == nil && !p.Online { + st.Quiet = append(st.Quiet, StatusQuiet{Key: k, What: "offline"}) + } + } + for k, v := range stuck { + if s, err := delivery.DecodeStuck(v); err == nil && !s.Moved { + st.Quiet = append(st.Quiet, StatusQuiet{Key: k, What: "stopped moving: it waited for " + s.Waited, Since: s.Since}) + } + } + sort.Slice(st.Quiet, func(i, j int) bool { return st.Quiet[i].Key < st.Quiet[j].Key }) + } + + wants, werr := values(r.Wants) + haves, herr := values(r.Haves) + switch { + case werr != nil: + st.MovingUnread = "the assigner's wants cannot be read: " + werr.Error() + case herr != nil: + st.MovingUnread = "the nodes' haves cannot be read: " + herr.Error() + default: + for wk, v := range wants { + node, module, err := delivery.ParseWantKey(wk) + if err != nil { + continue + } + w, err := delivery.DecodeWant(v) + if err != nil { + continue + } + hk, _ := delivery.HaveKey(node, module) + var h delivery.Have + hv, has := haves[hk] + if has { + if h, err = delivery.DecodeHave(hv); err != nil { + has = false + } + } + words, stopped := "", false + if sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk); err == nil && serr == nil { + if s, err := delivery.DecodeStuck(stuck[sk]); err == nil && !s.Moved { + words, stopped = s.Waited, true + } + } + m := delivery.MotionOf(w, true, h, has, words, stopped) + if m.State == "" || m.State == delivery.MotionSettled { + continue + } + row := StatusMoving{Node: node, Module: module, State: m.State, Outcome: m.Outcome, Words: m.Words, ToFP: w.FP} + if has { + row.FromFP = h.FP + } + st.Moving = append(st.Moving, row) + } + sort.Slice(st.Moving, func(i, j int) bool { + if st.Moving[i].Node != st.Moving[j].Node { + return st.Moving[i].Node < st.Moving[j].Node + } + return st.Moving[i].Module < st.Moving[j].Module + }) + } + + if raw, err := values(r.Attempts); err != nil { + st.HealsUnread = "mesh-healers' attempts cannot be read: " + err.Error() + } else { + for _, v := range raw { + var a health.Attempt + if json.Unmarshal(v, &a) != nil || now.Sub(a.Begun) > HealsWithin { + continue + } + outcome := a.Outcome + if outcome == "" { + outcome = health.OutcomeActing + } + st.Heals[outcome]++ + } + } + + stuckMoving := false + for _, m := range st.Moving { + stuckMoving = stuckMoving || m.State == delivery.MotionStuck + } + st.Well = len(st.Conditions) == 0 && len(st.Quiet) == 0 && !stuckMoving && + st.ConditionsUnread == "" && st.QuietUnread == "" && st.MovingUnread == "" + return st +} diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index ccdeb3b..a447fe7 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -4,6 +4,7 @@ package view import ( "encoding/json" "fmt" + "maps" "slices" "sort" "strings" @@ -102,8 +103,8 @@ type WantRow struct { Newest string `json:"newest,omitempty"` Rule int `json:"rule,omitempty"` Words string `json:"words,omitempty"` - // Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or - // cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14). + // Judgement is the node agent's judgement of the move that waits for a person's yes (its have in waiting-yes, ADR 0547 + // §3), and Judged its words; empty otherwise. The controller's composed copy is no longer read (ADR 0571 §6). Judgement string `json:"judgement,omitempty"` Judged string `json:"judged,omitempty"` } @@ -215,7 +216,7 @@ type UnreadableRow struct { // Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values. type Model struct { mu sync.Mutex - assigns map[string]assignment + placed *delivery.Placed // the seat's placement (the assigner's state, ADR 0567, ADR 0549 §3) builds map[string]buildEntry // by .build. groups map[string]groupEntry // the indexer's candidates, by group modules map[string]moduleEntry // the indexer's modules, by module @@ -263,22 +264,20 @@ type stuckEntry struct { err string } -// assignment holds only the fields of the controller's Assignment the view reads. -type assignment struct { - Node string `json:"node"` - Module string `json:"module"` - Build string `json:"build"` - Commit string `json:"commit"` - Since time.Time `json:"since"` - Ended time.Time `json:"ended"` - State string `json:"state"` - PulledIn bool `json:"pulled-in"` - RequiredBy string `json:"required-by"` -} +// pair is a module the seat's placement puts on a node. +type pair struct{ Node, Module string } -// open says whether the value is an assignment that stands for the module on its node, as the controller's -// Assignment.Stands does. A module pulled in by another stands too. -func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" } +// placedPairs is every module the seat's placement puts on a node, by node then module. The caller holds mu. +func (m *Model) placedPairs() []pair { + byNode := m.placed.ByNode() + var out []pair + for _, node := range slices.Sorted(maps.Keys(byNode)) { + for _, module := range byNode[node] { + out = append(out, pair{Node: node, Module: module}) + } + } + return out +} // build holds only the fields of the controller's Build record the view reads. A record with no state and no failure // is a done build. @@ -356,7 +355,7 @@ type nodeEntry struct { // New is an empty model. func New() *Model { - return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{}, + return &Model{placed: delivery.NewPlaced(), builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}, modHist: map[string][]delivery.Module{}, srcHist: map[uint64]map[string]string{}, taken: map[string]delivery.Taken{}, @@ -529,16 +528,9 @@ func (m *Model) apply(e Entry) { } st, err := delivery.DecodeStuck(e.Value) m.stuck[e.Key] = stuckEntry{value: st, err: errText(err)} - case AssignmentsBucket: - if e.Deleted { - delete(m.assigns, e.Key) - return - } - var a assignment - if json.Unmarshal(e.Value, &a) != nil || a.Node == "" || a.Module == "" { - return - } - m.assigns[a.Node+"."+a.Module] = a + case delivery.PlacementBucket: + // A key that does not read places nothing on its node; the seat's verb placement says why. + _, _, _ = m.placed.Put(e.Key, e.Value, e.Deleted, e.At) case delivery.CandidatesBucket: if e.Deleted { delete(m.groups, e.Key) @@ -597,13 +589,6 @@ func (m *Model) apply(e Entry) { } w, err := delivery.DecodeWant(e.Value) m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At} - case delivery.ControllerComposedBucket: - if e.Deleted { - delete(m.composed, e.Key) - return - } - d, err := delivery.DecodeComposedDigest(e.Value) - m.composed[e.Key] = composedEntry{value: d, err: errText(err)} case delivery.TakenBucket: if e.Deleted { delete(m.taken, e.Key) @@ -651,34 +636,6 @@ func (m *Model) apply(e Entry) { return } m.keys[e.Key] = keyEntry{value: b} - case BuildsBucket: - module, rest, ok := strings.Cut(e.Key, ".") - if !ok { - return - } - switch { - case rest == "newest": - if e.Deleted { - delete(m.newest, module) - return - } - m.newest[module] = strings.TrimSpace(string(e.Value)) - case strings.HasPrefix(rest, "build."): - if e.Deleted { - delete(m.builds, e.Key) - return - } - var b build - if json.Unmarshal(e.Value, &b) != nil { - return - } - // The source fingerprint is read with the SDK's record, not with the view's own struct. - var rec delivery.BuildRecord - if json.Unmarshal(e.Value, &rec) != nil { - return - } - m.builds[e.Key] = buildEntry{build: b, rec: rec} - } } } @@ -714,18 +671,12 @@ func (m *Model) snapshot() Snapshot { m.mu.Lock() defer m.mu.Unlock() rows := []Row{} - for key, a := range m.assigns { - if !a.open() { - continue - } - r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned, Assigned: a.Build} - if a.PulledIn { - r.RequiredBy = a.RequiredBy - } + for _, a := range m.placedPairs() { + r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned} var waits []string hk, herr := delivery.HaveKey(a.Node, a.Module) if herr != nil { - hk = key + continue } h, has := m.haves[hk] // The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node. @@ -745,17 +696,8 @@ func (m *Model) snapshot() Snapshot { r.Have = h.have.FP } r.Since, r.Detail = h.at, h.have.Detail - if wanted { - if h.have.FP != w.value.FP { - waits = append(waits, "the node has not applied "+w.value.FP) - } - } else if a.Build != "" && h.have.Build != a.Build { - waits = append(waits, "the node has not applied "+a.Build) - } - } - if id := m.newest[a.Module]; id != "" && a.Build != "" && id != a.Build { - if b, ok := m.builds[a.Module+".build."+id]; ok && b.done() { - waits = append(waits, a.Module+" has a newer build "+id+" not yet assigned") + if wanted && h.have.FP != w.value.FP { + waits = append(waits, "the node has not applied "+w.value.FP) } } r.Waits = strings.Join(waits, "; ") @@ -766,7 +708,7 @@ func (m *Model) snapshot() Snapshot { if perr != nil { continue } - if a, ok := m.assigns[node+"."+module]; ok && a.open() { + if _, ok := m.placed.Since(node, module); ok { continue } r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err} @@ -892,38 +834,22 @@ func (m *Model) groupRows() []GroupRow { // successful trunk build (delivery.BuildRecord.IsTrunkBuild: an outcome, not failed, on main or with no ref) that recorded one. The newest key is any state and ref, so it is not // used: the records are. func (m *Model) fingerprintRows() []FingerprintRow { - newest := map[string]buildEntry{} - for _, b := range m.builds { - if b.rec.Module == "" || !b.rec.IsTrunkBuild() || b.rec.SourceFingerprint == "" { - continue - } - if old, ok := newest[b.rec.Module]; !ok || b.rec.At.After(old.rec.At) || (b.rec.At.Equal(old.rec.At) && b.rec.ID > old.rec.ID) { - newest[b.rec.Module] = b - } - } out := []FingerprintRow{} for name, mod := range m.modules { if mod.err != "" { continue } row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit} - if mod.value.FP2 != "" || mod.value.None2 != "" { - // The new form (R4e): the indexer's fp2 against the builder's key of it. - out = append(out, m.newFormRow(name, row, mod.value)) + if mod.value.FP2 == "" && mod.value.None2 == "" { + // The builder builds the new form alone, and its done keys are the build history the view reads (ADR 0571 §1). + row.State, row.Reason = FPNone, mod.value.None + if row.Reason == "" { + row.Reason = "the indexer holds no fingerprint of the new form for it, so no build of the builder's is compared" + } + out = append(out, row) continue } - b, hasBuild := newest[name] - switch { - case mod.value.None != "": - row.State, row.Reason = FPNone, mod.value.None - case !hasBuild: - row.State, row.Indexer = FPNoBuild, mod.value.FP - case b.rec.SourceFingerprint == mod.value.FP: - row.State, row.Indexer, row.Build = FPSame, mod.value.FP, b.rec.SourceFingerprint - default: - row.State, row.Indexer, row.Build = FPDiffers, mod.value.FP, b.rec.SourceFingerprint - } - out = append(out, row) + out = append(out, m.newFormRow(name, row, mod.value)) } sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) return out @@ -982,12 +908,6 @@ func (s Snapshot) Counted() int { return n } -// The buckets the view reads besides the have bucket (which mesh-sdk's delivery package names). -const ( - AssignmentsBucket = "mesh-controller_assignments" - BuildsBucket = "mesh-controller_builds" -) - // The statuses of a row. const ( StatusAssigned = "assigned" @@ -1113,10 +1033,8 @@ func (m *Model) wantRows() []WantRow { if has && h.err == "" { row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health) } - if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed { - if c, ok := m.composed[ckey]; ok && c.err == "" { - row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words - } + if has && h.err == "" && h.have.Phase == delivery.PhaseWaitingYes && !v.Removed { + row.Judgement, row.Judged = string(h.have.Judgement), h.have.Detail } switch { case v.Removed: @@ -1176,13 +1094,10 @@ func (m *Model) sourceRow() *SourceRow { waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP))) } } - for key, a := range m.assigns { - if !a.open() { - continue - } + for _, a := range m.placedPairs() { wk, werr := delivery.HaveKey(a.Node, a.Module) if werr != nil { - wk = key + continue } if _, ok := m.wants[wk]; ok { continue @@ -1212,9 +1127,9 @@ func splitKey(key string) (node, module string) { // module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it // is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition. const ( - MotionMoving = "moving" - MotionSettled = "settled" - MotionStuck = "stuck" + MotionMoving = delivery.MotionMoving + MotionSettled = delivery.MotionSettled + MotionStuck = delivery.MotionStuck // MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound). MotionBoundSeconds = 1800 ) @@ -1343,7 +1258,6 @@ func (m *Model) motionOf(hk string) []MotionEvent { id := func(kind string) string { return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:") } - applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed) var out []MotionEvent say := func(kind string, b MotionBody) { if m.told[id(kind)] { @@ -1352,31 +1266,25 @@ func (m *Model) motionOf(hk string) []MotionEvent { m.told[id(kind)] = true out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b}) } - switch { - case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK: + // The derivation is mesh-sdk's (delivery.MotionOf, issue 1952 S7), shared with the condition keeper's hold. + words, stuck := m.stuckWant(w.value, node, module) + mo := delivery.MotionOf(w.value, true, h.have, has, words, stuck) + if mo.State == delivery.MotionSettled { if m.told[id(MotionMoving)] { b := body - b.Outcome = "running" + b.Outcome = mo.Outcome say(MotionSettled, b) } return out - case !m.told[id(MotionMoving)]: + } + if !m.told[id(MotionMoving)] { say(MotionMoving, body) } - stop := body - switch { - case has && h.have.Phase == delivery.PhaseHeld: - stop.Outcome, stop.Words = "stopped", h.have.Detail - case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP: - stop.Outcome, stop.Words = "failed", h.have.Detail - default: - if s, ok := m.stuckWant(w.value, node, module); ok { - stop.Outcome, stop.Words = "stopped", s - } else { - return out - } + if mo.State == delivery.MotionStuck { + stop := body + stop.Outcome, stop.Words = mo.Outcome, mo.Words + say(MotionStuck, stop) } - say(MotionStuck, stop) return out } diff --git a/modules/mesh-delivery-view/internal/view/view_test.go b/modules/mesh-delivery-view/internal/view/view_test.go index d228cab..a68ad20 100644 --- a/modules/mesh-delivery-view/internal/view/view_test.go +++ b/modules/mesh-delivery-view/internal/view/view_test.go @@ -1,6 +1,7 @@ package view import ( + "encoding/json" "errors" "reflect" "strings" @@ -8,6 +9,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) @@ -25,17 +27,22 @@ func haveE(t *testing.T, node, module string, v []byte, at time.Time) Entry { return Entry{Bucket: delivery.HaveBucket, Key: node + "." + module, Value: v, At: at} } -func assign(node, module, build, extra string) Entry { - v := `{"node":"` + node + `","module":"` + module + `","build":"` + build + `","commit":"c","since":"2026-10-18T11:00:00Z"` + extra + `}` - return Entry{Bucket: AssignmentsBucket, Key: node + "." + module, Value: []byte(v), At: t0} -} - -func buildRec(module, id, extra string) []Entry { - rec := `{"id":"` + id + `","module":"` + module + `","commit":"c"` + extra + `}` - return []Entry{ - {Bucket: BuildsBucket, Key: module + ".build." + id, Value: []byte(rec), At: t0}, - {Bucket: BuildsBucket, Key: module + ".newest", Value: []byte(id), At: t0}, +// place is the seat's placement of node, as the assigner keeps it (ADR 0567): the modules placed there. +func place(t *testing.T, node string, modules ...string) Entry { + t.Helper() + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) } + key, err := delivery.PlacementKey(node) + if err != nil { + t.Fatal(err) + } + v, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.PlacementBucket, Key: key, Value: v, At: t0} } func find(t *testing.T, s Snapshot, node, module string) Row { @@ -58,14 +65,11 @@ func fill(m *Model, es ...Entry) { // base: two nodes (a, b), three modules (x, y, z). func base(t *testing.T) []Entry { es := []Entry{ - assign("a", "x", "b1", ""), assign("a", "y", "b2", ""), assign("b", "x", "b1", ""), assign("b", "z", "b3", ""), + place(t, "a", "x", "y"), place(t, "b", "x", "z"), haveE(t, "a", "x", have(t, "b1", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(1*time.Minute)), haveE(t, "b", "x", have(t, "b1", delivery.PhaseChecking, delivery.HealthWaits, "starting"), t0.Add(2*time.Minute)), haveE(t, "b", "z", have(t, "b3", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(3*time.Minute)), } - es = append(es, buildRec("x", "b1", "")...) - es = append(es, buildRec("y", "b2", "")...) - es = append(es, buildRec("z", "b3", "")...) return es } @@ -77,68 +81,35 @@ func TestRowsPerAssignedModuleAndNode(t *testing.T) { // 1134/AC1 t.Fatalf("want 4 rows, got %d counted %d: %+v", len(s.Rows), s.Counted(), s.Rows) } r := find(t, s, "b", "x") - want := Row{Node: "b", Module: "x", Status: StatusAssigned, Assigned: "b1", Have: "b1", Phase: "checking", Health: "waits", + want := Row{Node: "b", Module: "x", Status: StatusAssigned, Have: "b1", Phase: "checking", Health: "waits", Since: t0.Add(2 * time.Minute), Detail: "starting"} if !reflect.DeepEqual(r, want) { t.Fatalf("row = %+v, want %+v", r, want) } y := find(t, s, "a", "y") - if y.Waits != "no have yet" || y.Assigned != "b2" || y.Have != "" { + if y.Waits != "no have yet" || y.Have != "" { t.Fatalf("a.y = %+v, want 'no have yet'", y) } } -func TestWaitsForTheNodeAndForANewerBuild(t *testing.T) { // 1134/AC2 +func TestLeftBehindIsWhatThePlacementDoesNotPut(t *testing.T) { // 1134/AC3, issue 774 (the seat's placement) m := New() fill(m, base(t)...) - fill(m, haveE(t, "a", "x", have(t, "b0", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) - if w := find(t, m.Snapshot(), "a", "x").Waits; w != "the node has not applied b1" { - t.Fatalf("waits = %q", w) - } - fill(m, buildRec("z", "b4", "")...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "z has a newer build b4 not yet assigned" { - t.Fatalf("waits = %q", w) - } - // a newest build that is not done is not a newer build to assign - fill(m, buildRec("z", "b5", `,"failed":"boom"`)...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" { - t.Fatalf("a failed newest build must not be waited on, waits = %q", w) - } - fill(m, buildRec("z", "b6", `,"state":"asked"`)...) - if w := find(t, m.Snapshot(), "b", "z").Waits; w != "" { - t.Fatalf("an asked newest build must not be waited on, waits = %q", w) - } -} - -func TestLeftBehindIsNotRunningAndPulledInIs(t *testing.T) { // 1134/AC3 (both halves) - for name, extra := range map[string]string{"ended": `,"ended":"2026-10-18T11:30:00Z"`, "state": `,"state":"pending"`} { - m := New() - fill(m, base(t)...) - fill(m, assign("b", "z", "b3", extra)) - s := m.Snapshot() - r := find(t, s, "b", "z") - if r.Status != StatusLeftBehind { - t.Errorf("%s: status = %q, want %q", name, r.Status, StatusLeftBehind) - } - if s.Counted() != 3 { - t.Errorf("%s: counted = %d, want 3", name, s.Counted()) - } - } - m := New() - fill(m, base(t)...) - fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // no assignment at all + fill(m, place(t, "b", "x")) // z taken off b s := m.Snapshot() - if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 { - t.Fatalf("a have with no assignment: %+v counted %d", find(t, s, "a", "w"), s.Counted()) + if r := find(t, s, "b", "z"); r.Status != StatusLeftBehind || s.Counted() != 3 { + t.Errorf("status = %q counted %d, want %q and 3", r.Status, s.Counted(), StatusLeftBehind) } - // A module pulled in by another stands: shown and counted, with what required it. m = New() fill(m, base(t)...) - fill(m, assign("b", "z", "b3", `,"pulled-in":true,"required-by":"x"`)) + fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // placed nowhere s = m.Snapshot() - r := find(t, s, "b", "z") - if r.Status != StatusAssigned || r.RequiredBy != "x" || r.Have != "b3" || s.Counted() != 4 { - t.Fatalf("pulled-in row = %+v counted %d, want shown and counted with required-by x", r, s.Counted()) + if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 { + t.Fatalf("a have with no placement: %+v counted %d", find(t, s, "a", "w"), s.Counted()) + } + fill(m, Entry{Bucket: delivery.PlacementBucket, Key: "placement.b", Deleted: true}) + if s := m.Snapshot(); s.Counted() != 2 || find(t, s, "b", "x").Status != StatusLeftBehind { + t.Fatalf("b's placement deleted: counted %d, %+v", s.Counted(), s.Rows) } } @@ -181,18 +152,13 @@ func TestOnlyTheAffectedRowChanges(t *testing.T) { // 1134/AC5 if r := find(t, after, "b", "x"); r.Phase != "running" || r.Health != "ok" || !r.Since.Equal(t0.Add(9*time.Minute)) { t.Fatalf("changed row = %+v", r) } - // an assignment change and a build change touch only their rows - fill(m, assign("a", "y", "b7", "")) - if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != "b7" { - t.Fatalf("assignment change not seen: %+v", r) + // a want's change touches only its row + fill(m, wantE(t, "a", "y", delivery.Want{FP: fpNew})) + if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != fpNew { + t.Fatalf("want change not seen: %+v", r) } if !reflect.DeepEqual(find(t, m.Snapshot(), "b", "z"), find(t, after, "b", "z")) { - t.Fatal("an assignment of another row changed b.z") - } - fill(m, buildRec("x", "b8", "")...) - s := m.Snapshot() - if find(t, s, "a", "x").Waits == "" || find(t, s, "b", "x").Waits == "" || find(t, s, "b", "z").Waits != "" { - t.Fatalf("a build of x must touch x's rows only: %+v", s.Rows) + t.Fatal("a want of another row changed b.z") } // a deleted key (a bucket's expiry) removes its value m.Apply(Entry{Bucket: delivery.HaveBucket, Key: "a.x", Deleted: true}) @@ -237,7 +203,13 @@ func stuckE(t *testing.T, bucket, key, waited string, since time.Time, moved boo return Entry{Bucket: delivery.StuckBucket, Key: bucket + "." + key, Value: b, At: since} } -func TestOfflineAndStuckAreListed(t *testing.T) { // 1168/AC2 +func TestTheWatchmansKeysAndStatus(t *testing.T) { + t.Run("offline and stuck are listed", offlineAndStuckAreListed) + t.Run("an unknown watchman version is unreadable, not dropped", unknownWatchmanVersionIsUnreadableNotDropped) + t.Run("1952/V1: status derives each part from its bucket, and says a part it cannot read", statusDerivesEachPartFromItsBucket) +} + +func offlineAndStuckAreListed(t *testing.T) { // 1168/AC2 m := New() fill(m, base(t)...) fill(m, @@ -265,7 +237,7 @@ func TestOfflineAndStuckAreListed(t *testing.T) { // 1168/AC2 } } -func TestUnknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC3 +func unknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC3 m := New() fill(m, Entry{Bucket: delivery.PresenceBucket, Key: "c", Value: []byte(`{"v":9,"online":false}`), At: t0}, @@ -292,3 +264,70 @@ func TestUnknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC t.Fatal("a readable value must clear the unreadable mark") } } + +// 1952/V1 (ADR 0564 §10): status reads the keeper's open conditions, the watchman's presence and stuck keys, the wants against +// the haves and the healers' attempts of the week; nothing writes a summary, and a part it cannot read is said, never well. +func statusDerivesEachPartFromItsBucket(t *testing.T) { + now := t0.Add(time.Hour) + composedC := "sha256:" + strings.Repeat("c", 64) + fp1, fp2, fp3 := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64), delivery.SourcePrefix2+strings.Repeat("3", 64) + enc := func(v any) string { b, _ := json.Marshal(v); return string(b) } + want := func(fp string) string { + b, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: composedC}) + return string(b) + } + have := func(fp string, phase delivery.Phase, health delivery.Health, detail string) string { + b, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: composedC, Phase: phase, Health: health, Detail: detail}) + if err != nil { + t.Fatal(err) + } + return string(b) + } + presence := func(on bool) string { b, _ := delivery.EncodePresence(delivery.Presence{Online: on}); return string(b) } + stuckOf := func(bucket, key, waited string, moved bool) string { + b, _ := delivery.EncodeStuck(delivery.Stuck{Bucket: bucket, Key: key, Waited: waited, Since: t0, Moved: moved}) + return string(b) + } + stuckWant, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.b.web") + fin := t0.Add(time.Minute) + r := StatusRead{ + Conditions: states(map[string]string{ + "machine.b.silent": enc(health.Condition{Key: "machine.b.silent", Severity: health.Warning, Summary: "b silent", Raised: t0}), + "core.doctor.self-check": enc(health.Condition{Key: "core.doctor.self-check", Severity: health.Urgent, Summary: "no run", Raised: t0.Add(time.Minute)}), + }), + Presence: states(map[string]string{"a": presence(true), "b": presence(false)}), + Stuck: states(map[string]string{stuckWant: stuckOf(delivery.AssignerWantsBucket, "want.b.web", "b to run web", false), + "x.moved": stuckOf("x", "moved", "y", true)}), + Wants: states(map[string]string{"want.a.web": want(fp2), "want.a.mail": want(fp1), "want.b.web": want(fp2), "want.a.db": want(fp3)}), + Haves: states(map[string]string{"a.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.mail": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), + "b.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.db": have(fp3, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed")}), + Attempts: states(map[string]string{"1": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: t0, Finished: &fin}), + "2": enc(health.Attempt{Outcome: health.OutcomeEscalated, Begun: t0}), "old": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: now.Add(-8 * 24 * time.Hour)})}), + } + st := DeriveStatus(r, now) + if len(st.Conditions) != 2 || st.Conditions[0].Key != "core.doctor.self-check" || st.Conditions[1].Key != "machine.b.silent" { + t.Errorf("conditions = %+v, want the urgent one first", st.Conditions) + } + if len(st.Quiet) != 2 || st.Quiet[0].Key != "b" || st.Quiet[1].Key != stuckWant { + t.Errorf("quiet = %+v", st.Quiet) + } + wantMoving := []StatusMoving{ + {Node: "a", Module: "db", State: delivery.MotionStuck, Outcome: delivery.OutcomeFailed, Words: "the unit failed", FromFP: fp3, ToFP: fp3}, + {Node: "a", Module: "web", State: delivery.MotionMoving, FromFP: fp1, ToFP: fp2}, + {Node: "b", Module: "web", State: delivery.MotionStuck, Outcome: delivery.OutcomeStopped, Words: "b to run web", FromFP: fp1, ToFP: fp2}, + } + if !reflect.DeepEqual(st.Moving, wantMoving) { + t.Errorf("moving = %+v\nwant %+v", st.Moving, wantMoving) + } + if !reflect.DeepEqual(st.Heals, map[string]int{health.OutcomeActed: 1, health.OutcomeEscalated: 1}) || st.Well { + t.Errorf("heals = %v, well %v", st.Heals, st.Well) + } + // A part it cannot read is said, and the mesh is not well; with nothing wrong and every part read it is. + if st := DeriveStatus(StatusRead{}, now); st.ConditionsUnread == "" || st.QuietUnread == "" || st.MovingUnread == "" || st.HealsUnread == "" || st.Well { + t.Errorf("nothing read = %+v", st) + } + empty := states(map[string]string{}) + if st := DeriveStatus(StatusRead{Conditions: empty, Presence: empty, Stuck: empty, Wants: empty, Haves: empty, Attempts: empty}, now); !st.Well { + t.Errorf("an empty mesh = %+v, want well", st) + } +} diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go index 6019daa..07ad6dc 100644 --- a/modules/mesh-delivery-view/internal/view/wants_test.go +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -101,7 +101,7 @@ func TestEveryWantIsARowAgainstItsHave(t *testing.T) { func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { m := New() fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew), - assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""), + place(t, "ace", "web"), place(t, "g14", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK)) @@ -119,7 +119,7 @@ func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") { t.Errorf("a want that is not the module's fingerprint: %+v", s) } - fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", "")) + fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), place(t, "novox", "web")) s = m.Snapshot().Source if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") { t.Errorf("a node assigned the module with no want: %+v", s) @@ -143,16 +143,15 @@ func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) { // 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words. func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) { + // 1955/L7 (ADR 0571 §6): the judgement is the node agent's, in its have waiting for a yes; the controller's composed copy + // is not read. m := New() - key, err := delivery.ComposedKey("ace", "web", fpNew) + h, err := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK, + Judgement: delivery.JudgedCannot, Waits: "sha256:" + strings.Repeat("c", 64), Detail: "the composition it runs is gone"}) if err != nil { t.Fatal(err) } - v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}) - if err != nil { - t.Fatal(err) - } - fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0}) + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: h, At: t0}) rows := m.Snapshot().Wants if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") { t.Fatalf("rows = %+v", rows) @@ -177,7 +176,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { said = append(said, e) return nil }) - fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""), + fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), place(t, "ace", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew})) if len(said) != 0 { t.Fatalf("said %v before any have", said) @@ -213,7 +212,7 @@ func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { // have: the row shows the fingerprint wanted and waits while the node runs another. func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { m := New() - fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + fill(m, place(t, "ace", "web"), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) r := find(t, m.Snapshot(), "ace", "web") if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") { t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r) @@ -228,10 +227,12 @@ func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { // and a have nothing is assigned to keeps the module's name. func TestADottedModuleNameIsFoundAndKept(t *testing.T) { m := New() - fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK), + // The seat's placement refuses a dotted module name (mesh-sdk delivery.Placement), so such a module is never placed: + // its have is a row left behind under its own name. + fill(m, place(t, "ace"), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK), haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK)) s := m.Snapshot() - if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" { + if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Status != StatusLeftBehind { t.Errorf("the dotted module's row = %+v", r) } if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind { diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 50985b4..72c912c 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-vault.signers", "node-engine.have"} // view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -119,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 475f6ca..6f16694 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -4,16 +4,21 @@ "moved": true, "slug": "view", "tools": [ + "view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", + "indexer_predefined", + "indexer_config_candidates", "builder_build", "builder_builds", + "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", + "node_trusts", "build_usage", "view_reach", "indexer_module_history", @@ -30,8 +35,6 @@ { "name": "reach", "history": 1 } ], "reads": [ - "mesh-controller.assignments", - "mesh-controller.builds", "mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", @@ -39,21 +42,27 @@ "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", - "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", + "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", + "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", + "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", + "mesh-conditions.conditions", + "mesh-healers.attempts", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", + "mesh-vault.signers", "node-engine.have" ], "own-secrets": { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go new file mode 100644 index 0000000..1ae2d78 --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go @@ -0,0 +1,170 @@ +// Command mesh-delivery-watchman-tools serves the seat mesh-watchdogs' verbs and the watchman's read tools (novox/hq ADR 0564 +// §4, §5, §12, issue 1952 WM1): the runtime launches it for a call. It reads every state the watchman keeps (presence, pulse, +// stuck, findings: its keys, one key, how many it holds) and asks the watchman's program, on its socket, for what only the +// program reads or does: DEAD_LETTERS, a stream, and resync. A process never serves (ADR 0436). +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" +) + +// Seat is the seat the watchman holds. +const Seat = "mesh-watchdogs" + +// The seat's verbs and the module's own tools, as the manifest names them. +const ( + VerbSignals = "signals" + VerbStuck = "stuck" + VerbPresence = "presence" + VerbDeadLetters = "dead-letters" + VerbResync = "resync" + ToolFindings = "watchman_findings" +) + +// Verbs are the seat's verbs, in the manifest's order. +var Verbs = []string{VerbSignals, VerbStuck, VerbPresence, VerbDeadLetters, VerbResync} + +// ModuleTools are the module's own tools, in the manifest's order. +var ModuleTools = []string{ToolFindings} + +// Reads names, per tool, the states it reads whole; Acts, per tool, the program's act it starts. The clause-12 test holds +// that every state the manifest keeps is read by one, and every act the program takes by hand is started by one. +var ( + Reads = map[string][]string{VerbSignals: {"pulse"}, VerbStuck: {"stuck"}, VerbPresence: {"presence"}, ToolFindings: {"findings"}} + Acts = map[string][]string{VerbDeadLetters: {socket.AskDeadLetters}, VerbResync: {socket.AskResync}} +) + +// state is what the tools need of a state: the runtime's, or a test's. +type state interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// keyed is one key as a read answers it. +type keyed struct { + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` +} + +// readAll answers a state whole, or one key, with how many keys it holds; prefix narrows the keys. +func readAll(st state, name, key, prefix string) (any, error) { + if key != "" { + e, err := st.Get(key) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", name, key, err) + } + if e == nil { + return nil, fmt.Errorf("%s holds no key %s", name, key) + } + return keyed{Key: key, Value: e.Value, Revision: e.Revision}, nil + } + keys, err := st.Keys() + if err != nil { + return nil, fmt.Errorf("%s cannot be listed: %w", name, err) + } + sort.Strings(keys) + out := []keyed{} + for _, k := range keys { + if prefix != "" && !strings.HasPrefix(k, prefix) { + continue + } + e, err := st.Get(k) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", name, k, err) + } + if e != nil { + out = append(out, keyed{Key: k, Value: e.Value, Revision: e.Revision}) + } + } + return map[string]any{"state": name, "holds": len(keys), "keys": out}, nil +} + +func str(args map[string]any, name string) string { + v, _ := args[name].(string) + return strings.TrimSpace(v) +} + +var keyArgs = map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "one key only"}, + "prefix": map[string]any{"type": "string", "description": "only the keys that begin with it"}}} + +// Tools are the seat's verbs and the module's tools, over the states states gives and the program asked through ask. +func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, error)) []stdio.Tool { + read := func(name string) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + return readAll(states(name), name, str(args, "key"), str(args, "prefix")) + } + } + signals := func(args map[string]any) (any, error) { + pulses, err := readAll(states("pulse"), "pulse", str(args, "key"), str(args, "prefix")) + if err != nil { + return nil, err + } + return map[string]any{"rows": watch.SignalRows, "pulses": pulses}, nil + } + dead := func(args map[string]any) (any, error) { + a := socket.Ask{Ask: socket.AskDeadLetters, Consumer: str(args, "consumer"), Act: str(args, "act"), Why: str(args, "why")} + if v, ok := args["limit"].(float64); ok { + a.Limit = int(v) + } + if v, ok := args["id"].(float64); ok && v > 0 { + a.ID = uint64(v) + } + return ask(a) + } + resync := func(map[string]any) (any, error) { return ask(socket.Ask{Ask: socket.AskResync}) } + verb := func(v string) string { return Seat + "." + v } + return []stdio.Tool{ + {Name: verb(VerbSignals), Input: keyArgs, Run: signals, + Description: "The signals table's rows the watchman holds (S1, S8, S9, S10, S11, S21 and the haves' health), each a pulse key " + + "whose time to live is its bound and whose expiry is its finding (novox/hq ADR 0564 §4), and every pulse the watchman " + + "keeps: the moves it mirrors (a build step, a want, a presence) and the signals (signal.., with what their " + + "expiry does). Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbStuck), Input: keyArgs, Run: read("stuck"), + Description: "What stopped moving: every key of the watchman's stuck bucket, . with what it waited for since " + + "when, and whether it moved since (never deleted). A key not moved is also the watchman's finding " + + "delivery..stuck, a node's presence machine..silent. Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbPresence), Input: keyArgs, Run: read("presence"), + Description: "Who is online on the bus as the watchman hears it: each node's agent () and each delivery part's instance " + + "(part..), online or not since the entry's time. Give key for one, prefix to narrow. Read-only."}, + {Name: verb(VerbDeadLetters), Run: dead, + Description: "What a consumer gave up on, kept in DEAD_LETTERS (novox/hq ADR 0564 §5, ADR 0264, ADR 0376): newest first, " + + "for one consumer (its name or .), at most limit; one whole by id (a restricted one's body " + + "withheld). Who keeps them now: the controller until the lease names no controller, the watchman from then. " + + "Delivering again and dropping (act deliver or drop, with why) are refused here until the hand-act log's home " + + "(issue 1953): before the lease write use mesh-controller.dead-letters.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "consumer": map[string]any{"type": "string", "description": "only this consumer's, its name or ."}, + "limit": map[string]any{"type": "integer", "description": "at most this many, newest first"}, + "id": map[string]any{"type": "integer", "description": "one dead letter whole, by its id"}, + "act": map[string]any{"type": "string", "enum": []string{"deliver", "drop"}, "description": "deliver again or drop (refused until issue 1953)"}, + "why": map[string]any{"type": "string", "description": "why, with act"}}}}, + {Name: verb(VerbResync), Run: resync, + Description: "Rebuild what the watchman holds by hand (novox/hq ADR 0564 §12): presence from the bus's open connections, " + + "the dead letters' count from DEAD_LETTERS and whether to take the notices, and every finding it observes written " + + "again to its bucket. Answers how many findings it wrote."}, + {Name: ToolFindings, Input: keyArgs, Run: read("findings"), + Description: "The watchman's findings (novox/hq ADR 0564 §2): each thing it observes wrong now, .., with its " + + "severity, subject, evidence and the row that observed it; the condition keeper raises and clears the conditions from " + + "them. Give key for one, prefix to narrow. Read-only."}, + } +} + +func main() { + path := socket.PathOf(os.Getenv) + tools := Tools(func(name string) state { return stdio.State(name) }, func(a socket.Ask) (json.RawMessage, error) { return socket.On(path, a) }) + if err := stdio.Serve("", tools); err != nil { + fmt.Fprintln(os.Stderr, "mesh-delivery-watchman-tools:", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go new file mode 100644 index 0000000..97a9262 --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "slices" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-watchman/internal/socket" +) + +type fakeState map[string]string + +func (f fakeState) Keys() ([]string, error) { + out := []string{} + for k := range f { + out = append(out, k) + } + return out, nil +} + +func (f fakeState) Get(key string) (*stdio.StateEntry, error) { + v, ok := f[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: 3}, nil +} + +// 1952/WM1, ADR 0564 §12: every state the manifest keeps is read by a tool, every act the program starts by hand is +// started by one, and every verb the claim serves and every tool the manifest names is served; and each answers. +func TestEveryStateAndActHasATool(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Tools []string `json:"tools"` + State []struct { + Name string `json:"name"` + } `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + asked := []socket.Ask{} + states := map[string]fakeState{"findings": {"machine.ace.silent": `{"v":1}`, "core.doctor.self-check-silent": `{"v":1}`}} + tools := Tools(func(name string) state { return states[name] }, func(a socket.Ask) (json.RawMessage, error) { + asked = append(asked, a) + if a.Act != "" { + return nil, errors.New("refused") + } + return json.RawMessage(`{}`), nil + }) + served := map[string]stdio.Tool{} + for _, tool := range tools { + served[tool.Name] = tool + } + var names []string + for _, c := range m.Claims { + if c.Name != Seat || !slices.Equal(c.Serves, Verbs) { + t.Errorf("claim %+v, want %s serving %v", c, Seat, Verbs) + } + for _, v := range c.Serves { + names = append(names, Seat+"."+v) + } + } + if !slices.Equal(m.Tools, ModuleTools) { + t.Errorf("tools = %v, want %v", m.Tools, ModuleTools) + } + names = append(names, m.Tools...) + if len(served) != len(names) { + t.Errorf("served %d tools, the manifest names %v", len(served), names) + } + for _, n := range names { + if _, ok := served[n]; !ok { + t.Errorf("%s is named and not served", n) + } + } + read := map[string]bool{} + for _, r := range Reads { + for _, s := range r { + read[s] = true + } + } + for _, s := range m.State { + if !read[s.Name] { + t.Errorf("the state %s is read by no tool", s.Name) + } + } + acts := map[string]bool{} + for _, a := range Acts { + for _, s := range a { + acts[s] = true + } + } + for _, a := range []string{socket.AskDeadLetters, socket.AskResync} { + if !acts[a] { + t.Errorf("the program's act %s is started by no tool", a) + } + } + t.Run("a read answers every key and how many", func(t *testing.T) { + out, err := served[ToolFindings].Run(map[string]any{"prefix": "machine."}) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(out) + var got struct { + Holds int `json:"holds"` + Keys []struct { + Key string `json:"key"` + } `json:"keys"` + } + _ = json.Unmarshal(b, &got) + if got.Holds != 2 || len(got.Keys) != 1 || got.Keys[0].Key != "machine.ace.silent" { + t.Fatalf("answer = %s", b) + } + if _, err := served[ToolFindings].Run(map[string]any{"key": "no.such.key"}); err == nil { + t.Fatal("a key not held is answered") + } + }) + t.Run("dead-letters and resync ask the program", func(t *testing.T) { + if _, err := served[Seat+"."+VerbDeadLetters].Run(map[string]any{"consumer": "ace_web", "limit": float64(3), "id": float64(7)}); err != nil { + t.Fatal(err) + } + if _, err := served[Seat+"."+VerbDeadLetters].Run(map[string]any{"act": "drop", "why": "x"}); err == nil { + t.Fatal("a refused act answered") + } + if _, err := served[Seat+"."+VerbResync].Run(nil); err != nil { + t.Fatal(err) + } + want := []socket.Ask{{Ask: socket.AskDeadLetters, Consumer: "ace_web", Limit: 3, ID: 7}, {Ask: socket.AskDeadLetters, Act: "drop", Why: "x"}, {Ask: socket.AskResync}} + if !slices.Equal(asked, want) { + t.Fatalf("asked %+v", asked) + } + }) +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index 27cf64c..b32baa3 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -17,8 +17,13 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-watchman/internal/deadletters" + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" ) // busConf is the composed accounts file's shape (mesh-controller internal/broker): the account MESH imports the system @@ -103,6 +108,7 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -111,6 +117,31 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 kvs[c.Bucket] = kv } + // The controller's lease, its consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one + // message a consumer will give up on (1952/WM5). + lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket}) + if err != nil { + t.Fatal(err) + } + for _, c := range []jetstream.StreamConfig{ + {Name: buscompose.DeadLetterNoticesStream, Subjects: []string{buscompose.MaxDeliveriesAdvisories}, Retention: jetstream.WorkQueuePolicy}, + {Name: buscompose.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}, Duplicates: 24 * time.Hour, AllowMsgTTL: true}, + {Name: "SRC", Subjects: []string{"src.>"}}, + } { + if _, err := js.CreateStream(ctx, c); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateConsumer(ctx, buscompose.DeadLetterNoticesStream, jetstream.ConsumerConfig{Durable: buscompose.ControllerName, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + given := nats.NewMsg("src.ask") + given.Header.Set("X-Asker", "n1") + given.Data = []byte("do it") + if _, err := js.PublishMsg(ctx, given); err != nil { + t.Fatal(err) + } + // A node already connected when the watchman starts: its presence comes from the one connections request. connect("node.n0") @@ -119,7 +150,13 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_PULSE_BOUND": "3s"} + // A unix socket's path is at most 108 bytes, which a test's TMPDIR may pass. + sockDir, err := os.MkdirTemp("/tmp", "ws") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_PULSE_BOUND": "3s", "MESH_WATCHMAN_SOCKET": filepath.Join(sockDir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() @@ -224,6 +261,63 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 t.Errorf("a stuck key %s was written for a move that happened before the bound", k) } } + + // 1952/WM4: what stopped moving is the watchman's own finding: the ask no node ended, and node n1 silent. + finding := func(key string) bool { _, ok := has(watch.FindingsBucket, key); return ok } + until("the stuck ask's finding", func() bool { + return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck)) + }) + until("node n1's silence", func() bool { return finding("machine.n1.silent") }) + + // 1952/WM5: while the lease names a controller the watchman takes no notice; handed away while the controller's consumer + // stands, it says the notices wait; the consumer removed (the lease write's step), it makes its own and keeps a notice + // with its headers. + if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil { + t.Fatal("the watchman made its consumer while the lease names a controller") + } + if _, err := lease.Put(ctx, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator","why":"the lease write"}`)); err != nil { + t.Fatal(err) + } + waits := watch.FindingKey(watch.ScopeBus, "DEAD_LETTER_NOTICES.controller", watch.KindNoticesWaiting) + until("the notices said waiting behind the controller's consumer", func() bool { return finding(waits) }) + if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil { + t.Fatal("the watchman made its consumer while the controller's stands") + } + if err := js.DeleteConsumer(ctx, buscompose.DeadLetterNoticesStream, buscompose.ControllerName); err != nil { + t.Fatal(err) + } + until("the watchman's own consumer on the notices", func() bool { + _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer) + return err == nil + }) + until("the wait no longer said", func() bool { return !finding(waits) }) + notice, _ := json.Marshal(map[string]any{"stream": "SRC", "consumer": "c1", "stream_seq": 1, "deliveries": 5, "timestamp": time.Now().UTC()}) + if err := ctl.Publish(buscompose.MaxDeliveriesAdvisories[:len(buscompose.MaxDeliveriesAdvisories)-1]+"SRC.c1", notice); err != nil { + t.Fatal(err) + } + dl, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + t.Fatal(err) + } + var kept *jetstream.RawStreamMsg + until("the notice kept in DEAD_LETTERS", func() bool { + kept, err = dl.GetLastMsgForSubject(ctx, buscompose.DeadLetterSubject("SRC", "c1")) + return err == nil + }) + if kept.Header.Get(deadletters.HeaderStream) != "SRC" || kept.Header.Get(deadletters.HeaderSequence) != "1" || + kept.Header.Get(deadletters.HeaderSubject) != "src.ask" || kept.Header.Get("X-Asker") != "n1" || string(kept.Data) != "do it" { + t.Fatalf("kept = %v %q", kept.Header, kept.Data) + } + until("the consumer's dead letters said", func() bool { return finding("bus.SRC.c1.max-deliveries") }) + // The tools' socket lists it, and refuses to drop it. + answer, err := socket.On(env["MESH_WATCHMAN_SOCKET"], socket.Ask{Ask: socket.AskDeadLetters, Consumer: "c1"}) + if err != nil || !strings.Contains(string(answer), `"total":1`) || !strings.Contains(string(answer), "the watchman") { + t.Fatalf("dead-letters = %s, %v", answer, err) + } + if _, err := socket.On(env["MESH_WATCHMAN_SOCKET"], socket.Ask{Ask: socket.AskDeadLetters, ID: kept.Sequence, Act: "drop", Why: "test"}); err == nil || + !strings.Contains(err.Error(), "issue 1953") { + t.Fatalf("a drop was not refused naming the hand-act log's home: %v", err) + } } // TestAnAskRunningBeforeTheStartGetsItsPulse puts a running step in the build module's steps before the watchman starts, and @@ -271,6 +365,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -291,7 +386,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker} + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_WATCHMAN_SOCKET": filepath.Join(dir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() @@ -355,6 +450,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, + {Bucket: watch.FindingsBucket, History: 5}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -393,7 +489,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - env := map[string]string{"MESH_BROKER_FILE": broker} + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_WATCHMAN_SOCKET": filepath.Join(dir, "tools.sock")} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index c349da7..660ea74 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,8 +22,11 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" + "mesh-delivery-watchman/internal/socket" "mesh-delivery-watchman/internal/watch" ) @@ -44,12 +47,19 @@ func main() { } } -// kvOut writes the watchman's three buckets. +// kvOut writes the watchman's four buckets. type kvOut struct { - ctx context.Context - presence, pulse, stuk jetstream.KeyValue + ctx context.Context + presence, pulse, stuk, findings jetstream.KeyValue } +func (o *kvOut) PutFinding(key string, value []byte) error { + _, err := o.findings.Put(o.ctx, key, value) + return err +} + +func (o *kvOut) DeleteFinding(key string) error { return o.findings.Delete(o.ctx, key) } + func (o *kvOut) PutPresence(key string, online bool) error { value, err := delivery.EncodePresence(delivery.Presence{Online: online}) if err != nil { @@ -161,11 +171,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants // and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build // seats' steps, and says that moves are not watched (issue 1945). - var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue } + var kvs struct{ presence, pulse, stuck, findings, wants, haves, lease jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue - }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { + }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}, {watch.FindingsBucket, &kvs.findings}} { if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil { return err } @@ -176,7 +186,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it } - w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now) + kvs.lease = process.OptionalBucket(ctx, js, buscompose.LeaseBucket, "the lease is read as naming a controller, so the dead letters stay the controller's", logger.Printf) + w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now) w.Logf(logger.Printf) // A test shortens how long a move may take; the mesh never sets it. if v := strings.TrimSpace(getenv("MESH_DELIVERY_PULSE_BOUND")); v != "" { @@ -197,6 +208,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro w.SeedPresence(e.Key(), p.Online, e.Created()) } }}, + {kvs.findings, func(e jetstream.KeyValueEntry) { + if opOf(e) == watch.Put { + w.SeedFinding(e.Key(), e.Value()) + } + }}, {kvs.stuck, func(e jetstream.KeyValueEntry) { if opOf(e) == watch.Put { w.SeedStuck(e.Key(), e.Value()) @@ -212,6 +228,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro pulses, err := settle(ctx, kvs.pulse, func(e jetstream.KeyValueEntry) { if opOf(e) == watch.Put { w.SeedPulse(e.Key()) + w.SeedSignal(e.Key(), e.Value()) } }) if err != nil { @@ -281,9 +298,92 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := w.Rebuild(ask); err != nil { return err } + // The seat mesh-provisioner's provisions (ADR 0570 §7): a provision in waits is needs-operator. Read before the findings are + // reconciled when the bucket is there; bound once it appears when it is not (mesh-provisioner comes in its own delivery). + provisionsWatch := func(kv jetstream.KeyValue) (jetstream.KeyWatcher, error) { + return settle(ctx, kv, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + } + var provWatch jetstream.KeyWatcher + if kv, err := js.KeyValue(ctx, provisions.ProvisionsBucket); err == nil { + if provWatch, err = provisionsWatch(kv); err != nil { + process.NotWatched(logger.Printf, provisions.ProvisionsBucket, err, "no provision waiting for the operator is said until the watchman starts again") + } + } else { + process.AwaitBucket(ctx, js, conn, provisions.ProvisionsBucket, "no provision waiting for the operator is said", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + pw, err := provisionsWatch(kv) + if err != nil { + process.NotWatched(logger.Printf, provisions.ProvisionsBucket, err, "no provision waiting for the operator is said until the watchman starts again") + return + } + go func() { + _ = follow(ctx, pw, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + }() + }) + } + // Each node's own health report (the node-engine's state health): read before the findings are reconciled when the + // bucket is there, bound once it appears when it is not; each change is heard on the watch, never polled. + healthEach := func(e jetstream.KeyValueEntry) { w.NodeHealth(e.Key(), e.Value(), opOf(e)) } + var healthWatch jetstream.KeyWatcher + if kv, err := js.KeyValue(ctx, delivery.NodeHealthBucket); err == nil { + if healthWatch, err = settle(ctx, kv, healthEach); err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + } + } else { + process.AwaitBucket(ctx, js, conn, delivery.NodeHealthBucket, "no node's own health is said", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + hw, err := settle(ctx, kv, healthEach) + if err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + return + } + go func() { _ = follow(ctx, hw, healthEach) }() + }) + } + dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} + w.SetRecount(dead.count) + dead.count() + dead.ages() + dead.check() + provisioned(ctx, js, w, logger.Printf) w.Reconcile() + if err := hear(conn, w, dead, logger.Printf); err != nil { + return err + } - errs := make(chan error, 8) + errs := make(chan error, 16) + if healthWatch != nil { + go func() { errs <- follow(ctx, healthWatch, healthEach) }() + } + if provWatch != nil { + go func() { + errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) + }() + } + if kvs.lease != nil { + lw, err := kvs.lease.Watch(ctx, buscompose.ControllerLeaseKey) + if err != nil { + process.NotWatched(logger.Printf, buscompose.LeaseBucket, err, "the lease is read as naming a controller, so the dead letters stay the controller's") + } else { + go func() { + errs <- follow(ctx, lw, func(e jetstream.KeyValueEntry) { + var raw []byte + if opOf(e) == watch.Put { + raw = e.Value() + } + on, _ := buscompose.LeaseHandedAway(raw, len(raw) > 0, nil) + dead.lease(on) + }) + }() + } + } + if l, err := socket.Listen(socket.PathOf(getenv)); err != nil { + logger.Printf("the tools' socket cannot be opened (%v): dead-letters and resync answer nothing until the watchman starts again", err) + } else { + go func() { + errs <- socket.Serve(ctx, l, func(a socket.Ask) (any, error) { return answer(a, w, dead, ask) }) + }() + } go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 66ce007..892c16a 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -24,6 +24,20 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Errorf("an essential bucket refused: %v, want its grant named", err) } }) + t.Run("1952/WM2: every subject its grant lets it hear goes to a row", func(t *testing.T) { + if len(HeardSubjects) < 7 { + t.Fatalf("hears %v", HeardSubjects) + } + for _, s := range HeardSubjects { + concrete := strings.ReplaceAll(strings.ReplaceAll(s, "*", "x"), ">", "x.y") + if routeOf(concrete) == "" { + t.Errorf("%s (%s) goes to no row", s, concrete) + } + } + if routeOf("mesh.mod.web.event.started") != "" { + t.Error("another module's event goes to a row") + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go new file mode 100644 index 0000000..501800a --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go @@ -0,0 +1,282 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" + + "mesh-delivery-watchman/internal/deadletters" + "mesh-delivery-watchman/internal/socket" + "mesh-delivery-watchman/internal/watch" +) + +// The subjects the watchman hears for the signals table's rows (novox/hq ADR 0564 §4, Limits 3 and 11), each on the bus, +// none in the controller's store. Its grant names them (mesh-sdk buscompose, the watchman's principal). +const CheckedSubject = "mesh.seat." + health.SelfCheckModule + ".event." + health.EventChecked + +// HeardSubjects is every subject the watchman subscribes for its rows, beside the bus's connect and disconnect: the ones +// its grant lets it hear (buscompose.WatchmanHears), so the two cannot drift. +var HeardSubjects = buscompose.WatchmanHears() + +// The rows a heard subject goes to. +const ( + routePower = "power" + routeStanding = "standing" + routeTools = "tools-alive" + routeChecked = "checked" + routeAdvisory = "advisory" + routeDead = "dead-letter" + routeBusFault = "bus-fault" + routeProv = "provisioned" +) + +// The bus module's events of the connections the bus closed (S9). +const ( + SlowConsumerSubject = "mesh.mod.nats.event.bus.slow-consumer" + ConnectionClosedSubject = "mesh.mod.nats.event.bus.connection-closed" +) + +// routeOf is the row a heard subject goes to; empty for one the watchman does not read. +func routeOf(subject string) string { + switch { + case subject == SlowConsumerSubject || subject == ConnectionClosedSubject: + return routeBusFault + case strings.HasPrefix(subject, "prov.applied."): + return routeProv + case strings.HasPrefix(subject, "mesh.mod.power.event."): + return routePower + case strings.HasSuffix(subject, ".event.provisioner.failing") || strings.HasSuffix(subject, ".event.provisioner.recovered"): + return routeStanding + case strings.HasSuffix(subject, ".tools-alive"): + return routeTools + case subject == CheckedSubject: + return routeChecked + case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."): + return routeAdvisory + case strings.HasPrefix(subject, "mesh.events.dead."): + return routeDead + } + return "" +} + +// heard hands one message of a row's subject to the watchman. +func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, ...any)) { + switch routeOf(m.Subject) { + case routePower: + w.Power(m.Header.Get("x-node"), strings.TrimPrefix(m.Subject, "mesh.mod.power.event.")) + case routeStanding: + st, err := watch.ReadStanding(m.Subject, m.Data) + if err != nil { + logf("%v; left", err) + return + } + w.Stood(st) + case routeTools: + var body struct { + Every float64 `json:"every"` + } + _ = json.Unmarshal(m.Data, &body) + w.ToolsAlive(watch.ToolsNode(m.Subject), time.Duration(body.Every*float64(time.Second))) + case routeChecked: + w.Checked(m.Header.Get("x-node")) + case routeAdvisory: + a, ok := watch.ReadAdvisory(m.Subject, m.Data, deadletters.ConsumerInWords) + if !ok { + return + } + if dead.isTheControllersNotices(a.ID) { + dead.check() // the lease write's step removed it: the watchman may take the notices now + return + } + w.Advised(a) + case routeBusFault: + var c watch.BusClosure + if err := json.Unmarshal(m.Data, &c); err != nil { + logf("the bus module's word on %s cannot be read, left: %v", m.Subject, err) + return + } + kind := watch.KindSlowConsumer + if m.Subject == ConnectionClosedSubject { + kind = watch.KindConnectionClosed + } + w.Closed(kind, c) + case routeProv: + var r watch.ProvReport + if err := json.Unmarshal(m.Data, &r); err != nil { + logf("the provisioner client's report on %s cannot be read, left: %v", m.Subject, err) + return + } + w.Provisioned(r) + case routeDead: + if stream, consumer, ok := buscompose.DeadLetterOf(m.Subject); ok { + if at, err := time.Parse(time.RFC3339Nano, m.Header.Get(deadletters.HeaderAgesOut)); err == nil { + w.DeadAging(stream+"."+consumer, at) + } + } + dead.count() + } +} + +// hear subscribes every row's subject; a subject the bus refuses stops the start, naming it. +func hear(conn *nats.Conn, w *watch.Watchman, dead *deadKeeper, logf func(string, ...any)) error { + for _, subject := range HeardSubjects { + if _, err := conn.Subscribe(subject, func(m *nats.Msg) { heard(w, dead, m, logf) }); err != nil { + return fmt.Errorf("the watchman cannot hear %s: %w", subject, err) + } + } + return conn.Flush() +} + +// deadKeeper keeps what consumers gave up on from the lease write (ADR 0564 §5): it counts DEAD_LETTERS into findings, +// and takes DEAD_LETTER_NOTICES on its own consumer only once the lease names no controller and `controller` is gone. +type deadKeeper struct { + ctx context.Context + js jetstream.JetStream + w *watch.Watchman + logf func(string, ...any) + + mu sync.Mutex + handedAway bool + taking jetstream.ConsumeContext +} + +// ages pulses each consumer's restricted letters by when they age out, read from DEAD_LETTERS' headers (at the start and +// at resync; each letter kept later is pulsed as it is heard). +func (d *deadKeeper) ages() { + letters, _, err := deadletters.List(d.ctx, d.js, "", 0) + if err != nil { + d.logf("the dead letters' ages cannot be read, so one aging out ends its finding only at the next count: %v", err) + return + } + for _, l := range letters { + if l.Restricted && !l.AgesOut.IsZero() { + d.w.DeadAging(l.Stream+"."+l.Consumer, l.AgesOut) + } + } +} + +// provisioned reads each provisioner client's newest report kept by PROV_APPLIED (as the controller's +// ProvAppliedReports): S21 judges what stood at the start as what is said later. +func provisioned(ctx context.Context, js jetstream.JetStream, w *watch.Watchman, logf func(string, ...any)) { + s, err := js.Stream(ctx, buscompose.ProvAppliedStream) + if err == nil { + var info *jetstream.StreamInfo + if info, err = s.Info(ctx, jetstream.WithSubjectFilter(buscompose.ProvAppliedSubjects)); err == nil { + for subject := range info.State.Subjects { + msg, gerr := s.GetLastMsgForSubject(ctx, subject) + if gerr != nil { + continue // aged out between the two reads + } + var r watch.ProvReport + if json.Unmarshal(msg.Data, &r) == nil { + w.Provisioned(r) + } + } + return + } + } + logf("the provisioner clients' reports cannot be read (%v): S21 hears only those said from now", err) +} + +// count reads DEAD_LETTERS' count per consumer into the findings. +func (d *deadKeeper) count() { + held, err := deadletters.Held(d.ctx, d.js) + if err != nil { + d.logf("the dead letters cannot be counted, their findings are left as they are: %v", err) + return + } + d.w.DeadLettersHeld(held, deadletters.ConsumerInWords) +} + +// lease is the lease's change heard: handed away or not. +func (d *deadKeeper) lease(handedAway bool) { + d.mu.Lock() + d.handedAway = handedAway + d.mu.Unlock() + d.check() +} + +func (d *deadKeeper) away() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.handedAway +} + +// isTheControllersNotices says an advisory is about the controller's consumer on DEAD_LETTER_NOTICES once the lease is +// handed away: its removal is the lease write's step, not a loss. +func (d *deadKeeper) isTheControllersNotices(id string) bool { + return d.away() && id == buscompose.DeadLetterNoticesStream+"."+buscompose.ControllerName +} + +// check takes the notices, or says they wait, as Gate has it. +func (d *deadKeeper) check() { + away := d.away() + stands := true + if away { + var err error + if stands, err = deadletters.ControllerStands(d.ctx, d.js); err != nil { + d.logf("whether the controller's consumer on %s stands cannot be read (%v): it is taken to stand", buscompose.DeadLetterNoticesStream, err) + stands = true + } + } + take, waiting := deadletters.Gate(away, stands) + d.w.NoticesWaiting(waiting) + d.mu.Lock() + defer d.mu.Unlock() + if !take || d.taking != nil { + return + } + cc, err := deadletters.TakeNotices(d.ctx, d.js, func(deadletters.DeadLetter) { go d.count() }, d.logf) + if err != nil { + d.logf("the dead-letter notices cannot be taken yet: %v", err) + return + } + d.taking = cc + d.logf("the lease names no controller and its consumer on %s is gone: the watchman keeps the dead letters", buscompose.DeadLetterNoticesStream) +} + +// answer is the program's answer to the tools' ask on its socket. +func answer(a socket.Ask, w *watch.Watchman, dead *deadKeeper, ask func([]byte) ([]byte, error)) (any, error) { + switch a.Ask { + case socket.AskResync: + if err := w.Rebuild(ask); err != nil { + return nil, err + } + dead.count() + dead.ages() + dead.check() + return map[string]any{"findings-written": w.Rewrite(), "presence": "rebuilt from the bus's connections"}, nil + case socket.AskDeadLetters: + switch { + case a.Act == "deliver" || a.Act == "drop": + if strings.TrimSpace(a.Why) == "" { + return nil, errors.New("a dead letter is delivered again or dropped only with why") + } + return nil, deadletters.Refusal(dead.away(), a.Act) + case a.Act != "": + return nil, fmt.Errorf("%q is not deliver or drop", a.Act) + case a.ID > 0: + return deadletters.Named(dead.ctx, dead.js, a.ID) + } + list, total, err := deadletters.List(dead.ctx, dead.js, a.Consumer, a.Limit) + if err != nil { + return nil, err + } + keeper := "the controller (it holds its lease)" + if dead.away() { + keeper = "the watchman" + } + return map[string]any{"dead-letters": list, "total": total, "kept-by": keeper}, nil + } + return nil, fmt.Errorf("refused: the watchman's socket answers %q and %q", socket.AskDeadLetters, socket.AskResync) +} diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d42c521..ea23a95 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index 084deeb..cfa6f9b 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go b/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go new file mode 100644 index 0000000..4e4a228 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/deadletters/deadletters.go @@ -0,0 +1,358 @@ +// Package deadletters is what a consumer gave up on, kept by the watchman from the lease write (novox/hq ADR 0564 §5, +// ADR 0264, ADR 0376, issue 1952 WM5): ported from mesh-controller internal/link/deadletters.go at e04846e9 with nothing +// changed in what is kept, aged or withheld, on the jetstream API. +// +// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The controller's consumer `controller` never expires, so +// the watchman makes its own (Consumer) only when the lease names no controller and `controller` is gone (Gate); while +// `controller` still stands after the lease write it says so as a finding. Removing `controller` is a step of the lease +// write, with the bus module's tool. Before the lease write the watchman only lists and shows DEAD_LETTERS, which the +// controller still fills, and refuses to deliver or drop, naming the controller's verb; from it, delivering and dropping +// wait for the hand-act log's home (issue 1953), so they are refused naming that. +package deadletters + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// Consumer is the watchman's own consumer on DEAD_LETTER_NOTICES, made only by Gate's leave. +const Consumer = "mesh-delivery-watchman" + +// The headers a kept message carries beside its own (mesh-controller link.DeadHeader*: the same names, so the +// controller's copies and the watchman's are read alike). +const ( + HeaderStream = "Mesh-Dead-Stream" + HeaderConsumer = "Mesh-Dead-Consumer" + HeaderSequence = "Mesh-Dead-Sequence" + HeaderSubject = "Mesh-Dead-Subject" + HeaderDeliveries = "Mesh-Dead-Deliveries" + HeaderGaveUp = "Mesh-Dead-Gave-Up" + HeaderPublished = "Mesh-Dead-Published" + HeaderLost = "Mesh-Dead-Lost" + HeaderMsgID = "Mesh-Dead-Msg-Id" + HeaderRestricted = "Mesh-Dead-Restricted" + HeaderAgesOut = "Mesh-Dead-Ages-Out" + HeaderWorkQueue = "Mesh-Dead-Work-Queue" +) + +// DeadLetter is one message a consumer gave up on, as DEAD_LETTERS keeps it (the controller's JSON). +type DeadLetter struct { + ID uint64 `json:"id"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Who string `json:"who"` + Subject string `json:"subject"` + Sequence uint64 `json:"sequence"` + Deliveries uint64 `json:"deliveries"` + GaveUp time.Time `json:"gave_up"` + Published time.Time `json:"published,omitzero"` + Lost string `json:"lost,omitempty"` + Size int `json:"size"` + Restricted bool `json:"restricted,omitempty"` + AgesOut time.Time `json:"ages_out,omitzero"` + Withheld string `json:"withheld,omitempty"` + WorkQueue bool `json:"work-queue,omitempty"` + Body string `json:"body,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` +} + +// Now is the clock dead letters age by: a variable, so a test can move it. +var Now = time.Now + +// WithheldWords is what is said in place of a restricted dead letter's body. +const WithheldWords = "its body and its own headers are not shown: it came from a restricted stream, which only " + + "the modules its grant names may read (novox/hq ADR 0376)" + +// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what. +func ConsumerInWords(stream, name string) string { + switch { + case name == buscompose.ControllerName || name == buscompose.MergesConsumerName: + return "the controller's consumer on " + stream + case stream == "NODES": + return "how " + name + " hears what it should be (its declaration consumer)" + case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"): + seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-")) + return "the worker every holder of " + seat + " takes its asks from" + case stream == buscompose.EventsStream: + if node, module, ok := strings.Cut(name, "_"); ok { + return "how " + module + " on " + node + " hears what it consumes" + } + } + return "the consumer " + name + " on " + stream +} + +// sourceAge is whether a stream is restricted, a module's declared work queue, and its maximum age; a stream gone is +// neither. +func sourceAge(ctx context.Context, js jetstream.JetStream, stream string) (restricted, workQueue bool, maxAge time.Duration, err error) { + s, err := js.Stream(ctx, stream) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return false, false, 0, nil + } + if err != nil { + return false, false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err) + } + cfg := s.CachedInfo().Config + return cfg.Metadata[buscompose.RestrictedMetadata] == "true", cfg.Metadata[buscompose.WorkQueueMetadata] == "true", cfg.MaxAge, nil +} + +// notice is the part of the server's maximum-deliveries advisory a kept message is made from. +type notice struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + StreamSeq uint64 `json:"stream_seq"` + Deliveries uint64 `json:"deliveries"` + Timestamp time.Time `json:"timestamp"` +} + +// Keep keeps the message one maximum-deliveries notice is about. An error keeps nothing and the notice is to be taken +// again; a source that no longer holds the message is no error: the record is kept with why. +func Keep(ctx context.Context, js jetstream.JetStream, body []byte) (DeadLetter, error) { + var a notice + if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" || a.StreamSeq == 0 { + return DeadLetter{}, fmt.Errorf("not a maximum-deliveries notice the mesh can read: %.200s", body) + } + gaveUp := a.Timestamp + if gaveUp.IsZero() { + gaveUp = Now() + } + kept := &nats.Msg{Subject: buscompose.DeadLetterSubject(a.Stream, a.Consumer), Header: nats.Header{}} + restricted, workQueue, maxAge, err := sourceAge(ctx, js, a.Stream) + if err != nil { + return DeadLetter{}, err + } + if restricted { + kept.Header.Set(HeaderRestricted, "true") + } + if workQueue { + kept.Header.Set(HeaderWorkQueue, "true") + } + var raw *jetstream.RawStreamMsg + src, err := js.Stream(ctx, a.Stream) + if err == nil { + raw, err = src.GetMsg(ctx, a.StreamSeq) + } + agesOut := time.Time{} + if err == nil && restricted && maxAge > 0 { + agesOut = raw.Time.Add(maxAge) + kept.Header.Set(HeaderAgesOut, agesOut.UTC().Format(time.RFC3339Nano)) + } + switch { + case err == nil && !agesOut.IsZero() && agesOut.Sub(Now()) < time.Second: + kept.Header.Set(HeaderSubject, raw.Subject) + kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano)) + kept.Header.Set(HeaderLost, fmt.Sprintf("message %d of %s had passed its stream's maximum age of %s when the notice "+ + "was taken, so it is kept without its body (novox/hq ADR 0376)", a.StreamSeq, a.Stream, maxAge)) + case err == nil: + for k, v := range raw.Header { + if !strings.HasPrefix(k, "Nats-") { + kept.Header[k] = v + } + } + if id := raw.Header.Get(nats.MsgIdHdr); id != "" { + kept.Header.Set(HeaderMsgID, id) + } + kept.Header.Set(HeaderSubject, raw.Subject) + kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano)) + kept.Data = raw.Data + if !agesOut.IsZero() { + kept.Header.Set(nats.MsgTTLHdr, strconv.FormatInt(int64(agesOut.Sub(Now())/time.Second), 10)+"s") + } + case errors.Is(err, jetstream.ErrMsgNotFound) || errors.Is(err, jetstream.ErrStreamNotFound): + kept.Header.Set(HeaderLost, fmt.Sprintf("%s no longer held message %d when the notice was taken: %v", a.Stream, a.StreamSeq, err)) + default: + return DeadLetter{}, fmt.Errorf("message %d of %s could not be read: %w", a.StreamSeq, a.Stream, err) + } + kept.Header.Set(HeaderStream, a.Stream) + kept.Header.Set(HeaderConsumer, a.Consumer) + kept.Header.Set(HeaderSequence, strconv.FormatUint(a.StreamSeq, 10)) + kept.Header.Set(HeaderDeliveries, strconv.FormatUint(a.Deliveries, 10)) + kept.Header.Set(HeaderGaveUp, gaveUp.UTC().Format(time.RFC3339Nano)) + // One copy per message given up, however often its notice is taken, by whichever keeper takes it. + kept.Header.Set(nats.MsgIdHdr, fmt.Sprintf("%s.%s.%d", a.Stream, a.Consumer, a.StreamSeq)) + ack, err := js.PublishMsg(ctx, kept) + if err != nil { + return DeadLetter{}, fmt.Errorf("message %d of %s could not be kept in %s: %w", a.StreamSeq, a.Stream, buscompose.DeadLettersStream, err) + } + return Of(ack.Sequence, kept.Subject, kept.Header, kept.Data, false), nil +} + +// Of reads a kept message; whole gives its body and its own headers. +func Of(id uint64, subject string, h nats.Header, data []byte, whole bool) DeadLetter { + d := DeadLetter{ID: id, Stream: h.Get(HeaderStream), Consumer: h.Get(HeaderConsumer), Subject: h.Get(HeaderSubject), + Lost: h.Get(HeaderLost), Size: len(data)} + if d.Stream == "" || d.Consumer == "" { + d.Stream, d.Consumer, _ = buscompose.DeadLetterOf(subject) + } + d.Who = ConsumerInWords(d.Stream, d.Consumer) + d.Sequence, _ = strconv.ParseUint(h.Get(HeaderSequence), 10, 64) + d.Deliveries, _ = strconv.ParseUint(h.Get(HeaderDeliveries), 10, 64) + d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderGaveUp)) + d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderPublished)) + d.Restricted = h.Get(HeaderRestricted) == "true" + d.WorkQueue = h.Get(HeaderWorkQueue) == "true" + d.AgesOut, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderAgesOut)) + if whole { + d.Body = string(data) + d.Headers = map[string][]string{} + for k, v := range h { + if !strings.HasPrefix(k, "Mesh-Dead-") && !strings.HasPrefix(k, "Nats-") { + d.Headers[k] = append([]string(nil), v...) + } + } + } + return d +} + +// Held is how many messages DEAD_LETTERS holds for each consumer, by `.`. +func Held(ctx context.Context, js jetstream.JetStream) (map[string]int, error) { + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + info, err := s.Info(ctx, jetstream.WithSubjectFilter(">")) + if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + held := map[string]int{} + for subject, n := range info.State.Subjects { + if stream, consumer, ok := buscompose.DeadLetterOf(subject); ok && n > 0 { + held[stream+"."+consumer] += int(n) + } + } + return held, nil +} + +// List is what DEAD_LETTERS holds, newest first, at most most (all when most is not positive); for one consumer when +// consumer names one (its name, or `.`). A restricted one's body is withheld. The total is the stream's +// own count. +func List(ctx context.Context, js jetstream.JetStream, consumer string, most int) ([]DeadLetter, int, error) { + held, err := Held(ctx, js) + if err != nil { + return nil, 0, err + } + total := 0 + for key, n := range held { + if consumer == "" || key == consumer || strings.HasSuffix(key, "."+consumer) { + total += n + } + } + if total == 0 { + return nil, 0, nil + } + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return nil, 0, err + } + info, err := s.Info(ctx) + if err != nil { + return nil, 0, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err) + } + var out []DeadLetter + for seq := info.State.LastSeq; seq >= info.State.FirstSeq && seq > 0; seq-- { + if most > 0 && len(out) >= most || len(out) >= total { + break + } + raw, err := s.GetMsg(ctx, seq) + if errors.Is(err, jetstream.ErrMsgNotFound) { + continue + } + if err != nil { + return out, total, fmt.Errorf("dead letter %d cannot be read: %w", seq, err) + } + d := Of(seq, raw.Subject, raw.Header, raw.Data, false) + if consumer != "" && consumer != d.Consumer && consumer != d.Stream+"."+d.Consumer { + continue + } + if d.Restricted { + d.Withheld = WithheldWords + } + out = append(out, d) + } + return out, total, nil +} + +// ErrNoDeadLetter is an id DEAD_LETTERS does not hold. +var ErrNoDeadLetter = errors.New("no such dead letter") + +// Named is one kept message whole, but for a restricted one, whose body and own headers are never shown. +func Named(ctx context.Context, js jetstream.JetStream, id uint64) (DeadLetter, error) { + s, err := js.Stream(ctx, buscompose.DeadLettersStream) + if err != nil { + return DeadLetter{}, err + } + raw, err := s.GetMsg(ctx, id) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return DeadLetter{}, fmt.Errorf("%w: %s holds no message %d — delivered again or dropped already, or never kept; "+ + "dead-letters lists what it holds", ErrNoDeadLetter, buscompose.DeadLettersStream, id) + } + if err != nil { + return DeadLetter{}, fmt.Errorf("dead letter %d cannot be read: %w", id, err) + } + d := Of(id, raw.Subject, raw.Header, raw.Data, true) + if d.Restricted { + d.Body, d.Headers, d.Withheld = "", nil, WithheldWords + } + return d, nil +} + +// Refusal is why the watchman delivers nothing again and drops nothing: before the lease write the controller keeps them +// and its verb acts; from it the act waits for the hand-act log's home (issue 1953), where a person's act is recorded. +// act is deliver or drop. +func Refusal(handedAway bool, act string) error { + done := map[string]string{"deliver": "delivered again", "drop": "dropped"}[act] + if done == "" { + return fmt.Errorf("%q is not deliver or drop", act) + } + if !handedAway { + return fmt.Errorf("the controller still holds its lease and keeps the dead letters: %s it with "+ + "mesh-controller.dead-letters (novox/hq ADR 0564 §5); nothing is %s here", act, done) + } + return fmt.Errorf("a dead letter is %s only as a person's act recorded in the hand-act log, whose home is not "+ + "delivered yet (issue 1953): nothing is %s", done, done) +} + +// Gate is whether the watchman takes the notices itself: only when the lease names no controller (handedAway) and the +// controller's consumer is gone. waiting says the lease is handed away while `controller` still stands, which is the +// watchman's finding: each notice waits in the queue until the lease write's step removes it. +func Gate(handedAway, controllerStands bool) (take, waiting bool) { + return handedAway && !controllerStands, handedAway && controllerStands +} + +// ControllerStands asks the bus whether the controller's consumer on DEAD_LETTER_NOTICES stands. +func ControllerStands(ctx context.Context, js jetstream.JetStream) (bool, error) { + _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, buscompose.ControllerName) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + return false, nil + } + return err == nil, err +} + +// TakeNotices makes the watchman's own durable consumer on DEAD_LETTER_NOTICES (never touching `controller`) and keeps +// each notice it takes; a notice not kept is handed back to be taken again. kept hears each one kept. +func TakeNotices(ctx context.Context, js jetstream.JetStream, kept func(DeadLetter), logf func(string, ...any)) (jetstream.ConsumeContext, error) { + c, err := js.CreateOrUpdateConsumer(ctx, buscompose.DeadLetterNoticesStream, jetstream.ConsumerConfig{ + Durable: Consumer, AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Minute}) + if err != nil { + return nil, fmt.Errorf("the consumer %s on %s cannot be made: %w", Consumer, buscompose.DeadLetterNoticesStream, err) + } + return c.Consume(func(m jetstream.Msg) { + d, err := Keep(ctx, js, m.Data()) + if err != nil { + logf("a dead-letter notice was not kept, offered again in a minute: %v", err) + _ = m.NakWithDelay(time.Minute) + return + } + _ = m.Ack() + kept(d) + }) +} diff --git a/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go b/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go new file mode 100644 index 0000000..2926eca --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/deadletters/deadletters_test.go @@ -0,0 +1,64 @@ +package deadletters + +import ( + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// 1952/WM5, ADR 0564 §5: the watchman makes no consumer while `controller` exists or the lease names a controller, makes +// one once both are false, and says the wait meanwhile; it delivers and drops nothing, naming who does; and a kept message +// reads as the controller's copies do, a restricted one's age and work queue kept. +func TestTheWatchmanKeepsTheDeadLettersOnlyFromTheLeaseWrite(t *testing.T) { + for _, c := range []struct { + handedAway, controllerStands, take, waiting bool + }{ + {false, true, false, false}, + {false, false, false, false}, + {true, true, false, true}, + {true, false, true, false}, + } { + take, waiting := Gate(c.handedAway, c.controllerStands) + if take != c.take || waiting != c.waiting { + t.Errorf("Gate(%v, %v) = %v, %v", c.handedAway, c.controllerStands, take, waiting) + } + } + for _, c := range []struct { + away bool + act string + says string + }{ + {false, "deliver", "mesh-controller.dead-letters"}, + {false, "drop", "nothing is dropped"}, + {true, "drop", "issue 1953"}, + {true, "deliver", "delivered again"}, + {true, "send", "not deliver or drop"}, + } { + if err := Refusal(c.away, c.act); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("Refusal(%v, %s) = %v, want it to say %q", c.away, c.act, err, c.says) + } + } + at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + h := nats.Header{} + h.Set(HeaderStream, "EVENTS") + h.Set(HeaderConsumer, "ace_web") + h.Set(HeaderSequence, "41") + h.Set(HeaderDeliveries, "5") + h.Set(HeaderGaveUp, at.Format(time.RFC3339Nano)) + h.Set(HeaderRestricted, "true") + h.Set(HeaderWorkQueue, "true") + h.Set(HeaderAgesOut, at.Add(time.Hour).Format(time.RFC3339Nano)) + h.Set("X-Own", "kept") + h.Set("Nats-Msg-Id", "EVENTS.ace_web.41") + d := Of(9, "mesh.events.dead.EVENTS.ace_web", h, []byte("body"), true) + if d.ID != 9 || d.Sequence != 41 || d.Deliveries != 5 || !d.GaveUp.Equal(at) || !d.Restricted || !d.WorkQueue || + !d.AgesOut.Equal(at.Add(time.Hour)) || d.Who != "how web on ace hears what it consumes" || + d.Body != "body" || len(d.Headers) != 1 || d.Headers["X-Own"][0] != "kept" { + t.Fatalf("dead letter = %+v", d) + } + if d := Of(10, "mesh.events.dead.SEAT_X.y_worker", nats.Header{}, nil, false); d.Stream != "SEAT_X" || d.Consumer != "y_worker" || d.Body != "" { + t.Fatalf("a copy without headers reads its stream and consumer from its subject: %+v", d) + } +} diff --git a/modules/mesh-delivery-watchman/internal/socket/socket.go b/modules/mesh-delivery-watchman/internal/socket/socket.go new file mode 100644 index 0000000..42cf799 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/socket/socket.go @@ -0,0 +1,125 @@ +// Package socket is how the watchman's tools ask its program what only the program reads (novox/hq ADR 0564 §12, as the +// view's tools ask the view, mesh-delivery-view internal/view/serve.go): DEAD_LETTERS is a stream, which the tool runtime +// grants no bundle, and resync is an act of the program. One line of JSON each way, on a unix socket on the program's +// machine; the socket keeps nothing. +package socket + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +// Path is where the program listens unless MESH_WATCHMAN_SOCKET says otherwise. +const Path = "/var/lib/mesh-delivery-watchman-tools/tools.sock" + +// PathOf is the socket's path as getenv says it. +func PathOf(getenv func(string) string) string { + if p := strings.TrimSpace(getenv("MESH_WATCHMAN_SOCKET")); p != "" { + return p + } + return Path +} + +// The asks the program answers. +const ( + AskDeadLetters = "dead-letters" + AskResync = "resync" +) + +// Ask is one ask of the tools: dead-letters (consumer, limit, id; deliver or drop with why) or resync. +type Ask struct { + Ask string `json:"ask"` + Consumer string `json:"consumer,omitempty"` + Limit int `json:"limit,omitempty"` + ID uint64 `json:"id,omitempty"` + Act string `json:"act,omitempty"` + Why string `json:"why,omitempty"` +} + +// Listen listens on the unix socket at path, in a directory of mode 0755, the socket 0666. +func Listen(path string) (net.Listener, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + l, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + if err := os.Chmod(path, 0o666); err != nil { + l.Close() + return nil, err + } + return l, nil +} + +// Serve answers each ask on l with answer until ctx ends. +func Serve(ctx context.Context, l net.Listener, answer func(Ask) (any, error)) error { + go func() { <-ctx.Done(); l.Close() }() + for { + c, err := l.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + go func() { + defer c.Close() + _ = c.SetDeadline(time.Now().Add(30 * time.Second)) + line, err := bufio.NewReader(io.LimitReader(c, 4096)).ReadBytes('\n') + if err != nil && len(line) == 0 { + return + } + var a Ask + enc := json.NewEncoder(c) + if json.Unmarshal(line, &a) != nil { + _ = enc.Encode(map[string]any{"error": "refused: an ask is one line of JSON"}) + return + } + out, err := answer(a) + if err != nil { + _ = enc.Encode(map[string]any{"error": err.Error()}) + return + } + _ = enc.Encode(map[string]any{"answer": out}) + }() + } +} + +// On asks the program at path and returns its answer. +func On(path string, a Ask) (json.RawMessage, error) { + c, err := net.DialTimeout("unix", path, 5*time.Second) + if err != nil { + return nil, fmt.Errorf("cannot reach the watchman at %s: %w", path, err) + } + defer c.Close() + _ = c.SetDeadline(time.Now().Add(30 * time.Second)) + line, _ := json.Marshal(a) + if _, err := c.Write(append(line, '\n')); err != nil { + return nil, err + } + var reply struct { + Answer json.RawMessage `json:"answer"` + Error string `json:"error"` + } + if err := json.NewDecoder(c).Decode(&reply); err != nil { + return nil, fmt.Errorf("the watchman's answer cannot be read: %w", err) + } + if reply.Error != "" { + return nil, errors.New(reply.Error) + } + return reply.Answer, nil +} diff --git a/modules/mesh-delivery-watchman/internal/watch/advisories.go b/modules/mesh-delivery-watchman/internal/watch/advisories.go new file mode 100644 index 0000000..d4d0057 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/advisories.go @@ -0,0 +1,56 @@ +package watch + +import ( + "encoding/json" + "fmt" + "strings" +) + +// The bus's advisories the watchman hears (S9, mesh-controller internal/link/advisories.go at e04846e9): a durable consumer +// the mesh defines deleted is `consumer-lost`. A message given up on is said from DEAD_LETTERS, where it is kept, for as +// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects end +// with it (ADR 0564 §4). + +// Advisory subjects the watchman subscribes. +const ( + ConsumerDeletedAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>" + MaxDeliveriesAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>" +) + +// KindConsumerLost is a durable consumer of the mesh deleted. +const KindConsumerLost = "consumer-lost" + +// ReadAdvisory is one JetStream advisory as the mesh says it; false for one the watchman does not say. who words a +// consumer. +func ReadAdvisory(subject string, body []byte, who func(stream, consumer string) string) (Advisory, bool) { + var a struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + } + if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" { + return Advisory{}, false + } + if !strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.DELETED.") || !MeshNamed(a.Stream, a.Consumer) { + return Advisory{}, false + } + return Advisory{Kind: KindConsumerLost, ID: a.Stream + "." + a.Consumer, Machine: consumerMachine(a.Stream, a.Consumer), + Said: fmt.Sprintf("%s was deleted from the bus", who(a.Stream, a.Consumer))}, true +} + +// MeshNamed says a consumer is one of the durable consumers the mesh defines, by its name's shape; every other is a +// reader's own, named at random and deleted when its read is done. +func MeshNamed(stream, name string) bool { + switch { + case strings.HasPrefix(stream, "KV_") || stream == "ASSIGNMENTS": + return false + case stream == "NODES": + return true + case strings.HasPrefix(stream, "SEAT_"): + return strings.HasSuffix(name, "_worker") + case name == "controller" || name == "controller-merges": + return true + case stream == "EVENTS": + return strings.Contains(name, "_") + } + return false +} diff --git a/modules/mesh-delivery-watchman/internal/watch/dead.go b/modules/mesh-delivery-watchman/internal/watch/dead.go new file mode 100644 index 0000000..51b0747 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/dead.go @@ -0,0 +1,91 @@ +package watch + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// The dead letters' findings (novox/hq ADR 0564 §5, issue 330): a consumer DEAD_LETTERS holds a message for is said for as +// long as it holds any, `bus...max-deliveries`, and the notices waiting behind the controller's consumer +// after the lease write are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`. + +// DeadRecountEvery is how often DEAD_LETTERS is counted again while a consumer's finding stands: a letter delivered again +// or dropped by the controller's verb, which the watchman does not hear, ends its finding within it. The expiry of the +// watchman's own pulse, never a loop. +const DeadRecountEvery = 5 * time.Minute + +// KindNoticesWaiting is the finding of notices waiting while the controller's consumer stands after the lease write. +const KindNoticesWaiting = "notices-waiting" + +// DeadLettersHeld is DEAD_LETTERS' count per consumer, `.`, read whole: each consumer held writes its +// finding, each one no longer held has it deleted. who words a consumer. +func (w *Watchman) DeadLettersHeld(held map[string]int, who func(stream, consumer string) string) { + w.mu.Lock() + defer w.mu.Unlock() + want := map[string]bool{} + keys := make([]string, 0, len(held)) + for k := range held { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + n := held[key] + stream, consumer, ok := strings.Cut(key, ".") + if !ok || n <= 0 { + continue + } + messages, them := "a message", "it is" + if n > 1 { + messages, them = fmt.Sprintf("%d messages", n), "they are" + } + f := Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Source: "S9", + Machine: consumerMachine(stream, consumer), + Summary: fmt.Sprintf("%s gave up on %s; %s kept in DEAD_LETTERS until delivered again or dropped, with why", + who(stream, consumer), messages, them), + Said: fmt.Sprintf("%d held for %s (counted again every %s while any is held, so one delivered again or dropped "+ + "ends this within it)", n, key, DeadRecountEvery), + Needs: "deliver them again or drop them, from the mesh MCP server."} + want[f.Key()] = true + w.find(f) + } + for key := range w.findings { + if strings.HasPrefix(key, ScopeBus+".") && strings.HasSuffix(key, "."+KindDeadLetters) && !want[key] { + w.unfind(key) + } + } + if len(want) == 0 { + w.unsignal("dead", "recount") + return + } + w.signal("dead", "recount", DeadRecountEvery, signalDue{Do: dueRecount, + Finding: Finding{Scope: ScopeBus, ID: "DEAD_LETTERS", Kind: KindDeadLetters, Severity: Warning, Summary: "counted again", Source: "S9"}}) +} + +// consumerMachine is the machine a module's consumer on EVENTS is on; empty for the others. +func consumerMachine(stream, consumer string) string { + if stream == "EVENTS" { + if node, _, ok := strings.Cut(consumer, "_"); ok { + return node + } + } + return "" +} + +// NoticesWaiting says whether the lease is handed away while the controller's consumer on DEAD_LETTER_NOTICES still +// stands: every notice then waits in the queue, kept a week, until the lease write's step removes the consumer. +func (w *Watchman) NoticesWaiting(waiting bool) { + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: "DEAD_LETTER_NOTICES.controller", Kind: KindNoticesWaiting, Severity: Warning, Source: "dead-letters", + Summary: "the lease names no controller but the controller's consumer on DEAD_LETTER_NOTICES still stands, so every " + + "notice of a message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS", + Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the lease write", + Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, the lease write's step."} + if waiting { + w.find(f) + return + } + w.unfind(f.Key()) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/finding.go b/modules/mesh-delivery-watchman/internal/watch/finding.go new file mode 100644 index 0000000..fce809b --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/finding.go @@ -0,0 +1,159 @@ +package watch + +import ( + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The watchman is an observer of the seat mesh-watchdogs (novox/hq ADR 0564 §2, §4): what it observes wrong it writes as a +// finding, one key `..` of its own bucket `findings` in mesh-sdk's layout (health.Finding), only when what +// it says changes, and it deletes the key when it no longer observes it. The condition keeper (mesh-conditions) raises and +// clears the conditions from them; the watchman writes no condition and reads none (Limits 11). + +// FindingsBucket is the watchman's findings. +var FindingsBucket = health.FindingsBucket(health.WatchmanModule) + +// Observer is the name a finding of the watchman carries. +const Observer = health.WatchmanModule + +// The scopes and severities a finding of the watchman names, as the controller's conditions name them, so the keeper's +// `compare` meets the same keys. +const ( + ScopeMachine = health.ScopeMachine + ScopeDelivery = health.ScopeDelivery + ScopeModule = health.ScopeModule + ScopeProvider = health.ScopeProvider + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + + Urgent = string(health.Urgent) + Warning = string(health.Warning) +) + +// Finding is what the watchman builds of one thing it observes wrong: its subject (scope, id, the machine and the others +// it concerns), kind and severity, its evidence (Summary, and Said with Since, its one observation), the words the +// operator reads (ADR 0253; empty leaves the keeper's wording of the kind), and the row or fact it comes from (Source). +// It is written as health.Finding. +type Finding struct { + Scope, ID, Token, Kind, Machine string + Also []string + Severity string + Summary, Said string + Headline, Explanation string + Needs, Resolved string + Source string + Observer string + // Since is when what it says began, as the watchman saw it: the finding's time and its observation's, so the same + // thing observed again writes the same bytes. + Since time.Time +} + +// FindingKey composes a finding's key as the controller composes a condition's (health.Key). +func FindingKey(scope, id, token string) string { return health.FindingKey(scope, id, token) } + +// Key is the finding's key: its token, or its kind when it has none. +func (f Finding) Key() string { + token := f.Token + if token == "" { + token = f.Kind + } + return FindingKey(f.Scope, f.ID, token) +} + +// EncodeFinding is f in mesh-sdk's layout, refusing one the keeper could not make a condition of. +func EncodeFinding(f Finding) ([]byte, error) { + said := f.Said + if said == "" { + said = f.Summary + } + return health.EncodeFinding(health.Finding{Key: f.Key(), Kind: f.Kind, Token: f.Token, + Subject: health.Subject{Scope: f.Scope, ID: f.ID, Machine: f.Machine, Also: f.Also}, + Severity: health.Severity(f.Severity), Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, + Resolved: f.Resolved, Needs: f.Needs, Evidence: []health.Evidence{{At: f.Since.UTC(), Said: said}}, + Source: f.Source, Observer: Observer, At: f.Since.UTC()}) +} + +// DecodeFinding reads a finding the watchman wrote. +func DecodeFinding(data []byte) (Finding, error) { + h, err := health.DecodeFinding(data) + if err != nil { + return Finding{}, err + } + f := Finding{Scope: h.Subject.Scope, ID: h.Subject.ID, Token: h.Token, Kind: h.Kind, Machine: h.Subject.Machine, + Also: h.Subject.Also, Severity: string(h.Severity), Summary: h.Summary, Headline: h.Headline, Explanation: h.Explanation, + Needs: h.Needs, Resolved: h.Resolved, Source: h.Source, Observer: h.Observer, Since: h.At} + if len(h.Evidence) > 0 { + f.Said = h.Evidence[0].Said + } + return f, nil +} + +// find writes a finding when what it says differs from what the bucket holds (ADR 0289 §1). +func (w *Watchman) find(f Finding) { + value, err := EncodeFinding(f) + if err != nil { + w.say("no finding: %v", err) + return + } + key := f.Key() + delete(w.seededFindings, key) + if string(w.findings[key]) == string(value) { + return + } + if err := w.out.PutFinding(key, value); err != nil { + w.say("cannot write the finding %s: %v", key, err) + return + } + w.findings[key] = value +} + +// unfind deletes a finding the watchman no longer observes. +func (w *Watchman) unfind(key string) { + if _, ok := w.findings[key]; !ok { + return + } + if err := w.out.DeleteFinding(key); err != nil { + w.say("cannot delete the finding %s: %v", key, err) + return + } + delete(w.findings, key) +} + +// SeedFinding tells the watchman a finding that stands in its bucket at its start; ReconcileFindings deletes it unless what +// the watchman reads at its start observes it again. +func (w *Watchman) SeedFinding(key string, value []byte) { + w.mu.Lock() + defer w.mu.Unlock() + w.findings[key] = value + w.seededFindings[key] = true +} + +// Findings is every finding the watchman holds now, by key. +func (w *Watchman) Findings() map[string]Finding { + w.mu.Lock() + defer w.mu.Unlock() + out := map[string]Finding{} + for k, v := range w.findings { + if f, err := DecodeFinding(v); err == nil { + out[k] = f + } + } + return out +} + +// Rewrite writes every finding the watchman holds again (resync, ADR 0564 §12): a bucket cleared or a key lost by hand is +// written back as the watchman observes it. It answers how many it wrote. +func (w *Watchman) Rewrite() int { + w.mu.Lock() + defer w.mu.Unlock() + n := 0 + for key, value := range w.findings { + if err := w.out.PutFinding(key, value); err != nil { + w.say("cannot write the finding %s again: %v", key, err) + continue + } + n++ + } + return n +} diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go new file mode 100644 index 0000000..03f8cac --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -0,0 +1,365 @@ +package watch + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +func haveOf(t *testing.T, phase delivery.Phase, health delivery.Health, detail string) []byte { + t.Helper() + b, err := delivery.EncodeHave(delivery.Have{FP: fpNew, Phase: phase, Health: health, Detail: detail}) + if err != nil { + t.Fatal(err) + } + return b +} + +// expire hears the expiry of the pulse at pk. +func expire(w *Watchman, c *clock, pk string, after time.Duration) { + c.t = c.t.Add(after) + w.Pulse(pk, Expired, c.t) +} + +// The watchman's findings (novox/hq ADR 0564 §2, §4, §5, issue 1952 WM2-WM5): per row of the signals table a key not renewed +// writes its finding and a renewed one none; the haves' rows; stuck keys and presence as its own findings; the dead letters. +func TestTheWatchmansFindings(t *testing.T) { + cases := []struct { + name string + run func(t *testing.T, w *Watchman, o *fakeOut, c *clock) + // want are the finding keys standing at the end, with their severity. + want map[string]string + }{ + {"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Step("go", "n1.ask1", runningStep("ask1", t0), Put) + expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second) + k := FindingKey(ScopeDelivery, stuckKeyOf(t, checksKey, "ask1"), KindStuck) + if f := o.findings[k]; f.Kind != KindStuck || !strings.Contains(f.Summary, "the ask ask1 to end on n1") { + t.Fatalf("findings = %v", o.findings) + } + w.Step("go", "n1.ask1", endedStep("ask1", delivery.StepPassed, t0), Put) + }, map[string]string{}}, + {"WM2 S1: a node offline past its bound is silent, and back online it is not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") + w.Disconnect("node.ace") + expire(w, c, pulseKeyOf(t, presenceNS, "ace"), 120*time.Second) + if _, ok := o.findings["machine.ace.silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + w.Connect("node.ace") + }, map[string]string{}}, + {"WM2 S1: a node that said it sleeps is not silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.lap") + w.Power("lap", "sleeping") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + }, map[string]string{}}, + {"WM2 S1: a node back from sleep and still offline is silent after its bound", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.lap") + w.Power("lap", "sleeping") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + w.Power("lap", "woke") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + }, map[string]string{"machine.lap.silent": Warning}}, + {"WM2 S11: a tool runner not heard for three intervals is silent; a heartbeat ends it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.ToolsAlive("ace", 0) + pk := signalKey("S11", "ace") + if o.ttls[pk] != 3*HeartbeatEvery { + t.Fatalf("ttl = %v", o.ttls[pk]) + } + expire(w, c, pk, 3*HeartbeatEvery) + if _, ok := o.findings["machine.ace.tools-silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + w.ToolsAlive("ace", 20*time.Second) + if o.ttls[pk] != time.Minute { + t.Fatalf("ttl = %v, want three of the 20 s it says", o.ttls[pk]) + } + w.ToolsAlive("g14", 0) // renewed, never expired: no finding + }, map[string]string{}}, + {"WM2 S10: the self-check not heard for three runs is urgent; not before its first run", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + if len(o.pulses) != 0 { + t.Fatal("a pulse before the first run") + } + w.Checked("novox") + expire(w, c, signalKey("S10", "doctor"), 3*SelfCheckEvery) + }, map[string]string{"core.doctor.self-check-silent": Urgent}}, + {"WM3 S8: a failing word is its finding; not said again for 30 min it is silent; recovered ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + st, err := ReadStanding("mesh.mod.keycloak.event.provisioner.failing", + []byte(`{"provider":"oidc","provider-node":"ace","consumer":"photos","node":"g14","class":"refused","attempts":4,"since":"2026-10-18T11:00:00Z"}`)) + if err != nil { + t.Fatal(err) + } + w.Stood(st) + if f := o.findings["provider.keycloak.ace.photos.failing"]; f.Kind != KindProviderFailing || len(f.Also) != 1 { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S8", "keycloak.ace.photos"), ProviderSaysAgainWithin) + if _, ok := o.findings["provider.keycloak.ace.photos.silent"]; !ok { + t.Fatalf("findings = %v", o.findings) + } + st.Failing = false + w.Stood(st) + }, map[string]string{}}, + {"WM3 S9: an advisory is a finding counted, ended by an hour's quiet", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + a, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.EVENTS.ace_web", []byte(`{"stream":"EVENTS","consumer":"ace_web"}`), + func(s, c string) string { return c + " on " + s }) + if !ok { + t.Fatal("a mesh consumer's deletion is not read") + } + if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.KV_x.abc", []byte(`{"stream":"KV_x","consumer":"abc"}`), nil); ok { + t.Fatal("a reader's own consumer is said") + } + w.Advised(a) + w.Advised(a) + k := "bus.EVENTS.ace_web.consumer-lost" + if f := o.findings[k]; !strings.Contains(f.Summary, "2 times") || f.Machine != "ace" { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S9", k), AdvisoryQuiet) + }, map[string]string{}}, + {"WM3 S21: a heard machine's have failed for one apply interval is apply-failing, naming the package manager; applied again it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") + w.Have("ace.web", haveOf(t, delivery.PhaseFailed, delivery.HealthUnknown, "refused: the package manager (pacman) does not answer"), Put) + if len(o.findings) != 0 { + t.Fatalf("raised before its bound: %v", o.findings) + } + expire(w, c, signalKey("S21", "ace"), ApplyEvery+time.Second) + if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "pacman does not answer") || f.Headline != "ace cannot install packages" { + t.Fatalf("findings = %v", o.findings) + } + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + }, map[string]string{}}, + {"WM3 S21: a provisioner client's failed files are apply-failing past the bound; a silent machine's are not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Connect("node.ace") + w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: ProvFailed, Reason: "disk full", At: t0.Add(-10 * time.Minute)}) + if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "provisioner has failed to place the files of web") { + t.Fatalf("findings = %v", o.findings) + } + w.Provisioned(ProvReport{Node: "g14", Module: "web", Status: ProvFailed, Reason: "x", At: t0.Add(-10 * time.Minute)}) + w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: "applied", At: t0}) + }, map[string]string{}}, + {"WM3: unhealthy only at the second write in a row, urgent after four hours", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed"), Put) + if len(o.findings) != 0 { + t.Fatalf("one write raised %v", o.findings) + } + w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed"), Put) + if o.findings["module.web.ace.unhealthy"].Severity != Warning { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) // needs-operator is not observed here + }, map[string]string{"module.web.ace.unhealthy": Urgent}}, + {"1954/D2: a provision in waits is its consumer's needs-operator; provided, it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + enc := func(state, why string) []byte { + b, err := provisions.Encode(provisions.Provision{Provision: "oidc", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "keycloak", Node: "novox"}, State: state, Why: why, At: t0}) + if err != nil { + t.Fatal(err) + } + return b + } + w.Provision("ace.photos.oidc", enc(provisions.Waits, "the operator gives the client's redirect"), Put) + w.Provision("ace.photos.db", enc(provisions.Provided, ""), Put) + if f := o.findings["module.photos.ace.needs-operator"]; !strings.Contains(f.Summary, "redirect") || f.Machine != "ace" { + t.Fatalf("findings = %v", o.findings) + } + w.Provision("ace.photos.oidc", enc(provisions.Provided, ""), Put) + w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) + w.Provision("ace.web.oidc", nil, Delete) + }, map[string]string{}}, + {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + report := func(h delivery.NodeHealth) []byte { + b, err := delivery.EncodeNodeHealth(h) + if err != nil { + t.Fatal(err) + } + return b + } + w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{ + {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, + {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, + {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, + Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, + Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, + Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put) + for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, + "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} { + if o.findings[key].Severity != sev { + t.Errorf("%s = %+v", key, o.findings[key]) + } + } + if f := o.findings["machine.novox.network"]; strings.Contains(f.Summary, "names") { + t.Errorf("the names failing through the rewritten file are said twice: %s", f.Summary) + } + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox"}), Put) // all well again + }, map[string]string{}}, + {"774: a failing part is held at the machines it points at, as the controller holds it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + toward := func(node, part string, targets ...string) []byte { + b, _ := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: node, Network: &delivery.NetworkHealth{State: "unhealthy", + Parts: []delivery.NetworkPart{{Part: part, State: "unhealthy", Reason: part + " fails", Said: "x", Targets: targets}}}}) + return b + } + // own part: one machine alone failing toward a healthy one is its own. + w.NodeHealth("ace", toward("ace", "names", "dns1"), Put) + if _, ok := o.findings["machine.ace.network"]; !ok || len(o.findings) != 1 { + t.Fatalf("own part: findings %v", o.findings) + } + // two resolvers' machines: a second machine failing toward dns1 holds both there, said once as unreachable. + w.NodeHealth("g14", toward("g14", "names", "dns1"), Put) + if f, ok := o.findings["machine.dns1.unreachable"]; !ok || len(f.Also) != 2 || len(o.findings) != 1 { + t.Fatalf("two toward one resolver: findings %v", o.findings) + } + w.NodeHealth("ace", toward("ace", "names", "dns1", "dns2"), Put) // toward two resolvers, one not down: its own + if _, ok := o.findings["machine.ace.network"]; !ok { + t.Fatalf("two resolvers, one up: findings %v", o.findings) + } + w.NodeHealth("ace", nil, Delete) + w.NodeHealth("g14", nil, Delete) + // the hub silent on the record: a tunnel toward it is held there, and its silence says it. + w.Connect("node.hub") + w.Disconnect("node.hub") + expire(w, c, pulseKeyOf(t, presenceNS, "hub"), 120*time.Second) + w.NodeHealth("lap", toward("lap", "tunnel", "hub"), Put) + if _, own := o.findings["machine.lap.network"]; own { + t.Errorf("a tunnel toward the silent hub is the reporter's own: %v", o.findings) + } + if _, ok := o.findings["machine.hub.unreachable"]; ok { + t.Error("the silent hub is said unreachable as well") + } + w.Connect("node.hub") // back: lap's tunnel failing toward a hub up is its own again + if _, own := o.findings["machine.lap.network"]; !own { + t.Errorf("hub back: findings %v", o.findings) + } + w.NodeHealth("lap", nil, Delete) + }, map[string]string{}}, + {"774: a node two nodes' tunnels no longer see is unreachable, said once there; not while it is silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for node, key := range map[string]string{"ace": "kA", "g14": "kG", "lap": "kL"} { + b, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, TunnelKey: key}}) + w.Have(node+".node", b, Put) + } + peers := func(from string, seen bool) []byte { + b, _ := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: from, Peers: []delivery.PeerSeen{{Key: "kL", Seen: seen}}}) + return b + } + w.NodeHealth("ace", peers("ace", false), Put) + if len(o.findings) != 0 { + t.Fatalf("one node alone said it: %v", o.findings) + } + w.NodeHealth("g14", peers("g14", false), Put) + if f := o.findings["machine.lap.unreachable"]; len(f.Also) != 2 { + t.Fatalf("findings = %v", o.findings) + } + w.Connect("node.lap") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + if _, ok := o.findings["machine.lap.unreachable"]; ok { + t.Error("a silent node is also said unreachable") + } + w.Connect("node.lap") + w.NodeHealth("g14", peers("g14", true), Put) + }, map[string]string{}}, + {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put) + }, map[string]string{}}, + {"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + w.Connect("node.novox") + w.Disconnect("node.novox") + expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second) + if o.findings["machine.novox.silent"].Severity != Warning { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter) + }, map[string]string{"machine.novox.silent": Urgent}}, + {"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0} + w.Closed(KindSlowConsumer, cl) + w.Closed(KindSlowConsumer, cl) // said again when the bus module starts + cl.CID = 8 + w.Closed(KindSlowConsumer, cl) + f := o.findings["bus.ace.web.slow-consumer"] + if f.Machine != "ace" || !strings.Contains(f.Summary, "web on ace") || !strings.Contains(f.Summary, "2 times") { + t.Fatalf("findings = %v", o.findings) + } + expire(w, c, signalKey("S9", "bus.ace.web.slow-consumer"), AdvisoryQuiet) + }, map[string]string{}}, + {"WM5: while letters are held DEAD_LETTERS is counted again at its own pulse's expiry, and at a letter's age", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + n := 0 + w.SetRecount(func() { n++ }) + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c }) + if o.ttls[signalKey("dead", "recount")] != DeadRecountEvery || !strings.Contains(o.findings["bus.EVENTS.ace_web.max-deliveries"].Said, "every 5m0s") { + t.Fatalf("pulses %v findings %v", o.ttls, o.findings) + } + w.DeadAging("EVENTS.ace_web", c.t.Add(time.Minute)) + expire(w, c, signalKey("dead", "EVENTS.ace_web"), time.Minute) + expire(w, c, signalKey("dead", "recount"), DeadRecountEvery) + time.Sleep(10 * time.Millisecond) + if n != 2 { + t.Fatalf("counted again %d times", n) + } + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c }) // the count renews its pulse + w.DeadLettersHeld(map[string]int{}, nil) + if _, ok := o.pulses[signalKey("dead", "recount")]; ok { + t.Fatal("still counted again with nothing held") + } + }, map[string]string{}}, + {"WM4: resync after a cleared bucket writes the same findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) + o.findings = map[string]Finding{} + if n := w.Rewrite(); n != 1 { + t.Fatalf("rewrote %d", n) + } + }, map[string]string{"module.mail.ace.unhealthy": Warning}}, + {"WM4: a finding read at the start and not observed again is deleted; a signal's stays", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for _, f := range []Finding{ + {Scope: ScopeDelivery, ID: "x.y", Kind: KindStuck, Severity: Warning, Summary: "old", Source: "stuck"}, + {Scope: ScopeMachine, ID: "ace", Kind: KindToolsSilent, Severity: Warning, Summary: "silent", Source: "S11"}, + } { + v, _ := EncodeFinding(f) + o.findings[f.Key()], _ = DecodeFinding(v) + w.SeedFinding(f.Key(), v) + } + w.Reconcile() + }, map[string]string{"machine.ace.tools-silent": Warning}}, + {"WM5: a consumer DEAD_LETTERS holds for is its finding until it holds none", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + who := func(s, c string) string { return c } + w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 2, "SEAT_X.y_worker": 1}, who) + if f := o.findings["bus.EVENTS.ace_web.max-deliveries"]; f.Machine != "ace" || !strings.Contains(f.Summary, "2 messages") { + t.Fatalf("findings = %v", o.findings) + } + w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who) + w.NoticesWaiting(true) + }, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + tc.run(t, w, o, c) + got := map[string]string{} + for k, f := range o.findings { + got[k] = f.Severity + if f.Key() != k || f.Observer != Observer { + t.Errorf("finding %s = %+v", k, f) + } + } + if len(got) != len(tc.want) { + t.Fatalf("findings = %v, want %v", got, tc.want) + } + for k, sev := range tc.want { + if got[k] != sev { + t.Fatalf("findings = %v, want %v", got, tc.want) + } + } + }) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index 78e54c5..fe9f262 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -106,16 +106,28 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { // fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. func (w *Watchman) Have(key string, value []byte, op Op) { node, module, isNode, err := delivery.ParseHaveKey(key) - if err != nil || isNode { + if err != nil { return } w.mu.Lock() defer w.mu.Unlock() + if isNode { + // The node's own record: its tunnel key, whose peers' sight of it is network-unreachable. + var record *delivery.NodeRecord + if op == Put { + if hn, err := delivery.DecodeHaveNode(value); err == nil { + record = hn.Record + } + } + w.tunnelKey(node, record) + return + } switch op { case Expired: return case Delete: delete(w.haves, key) + w.haveFindings(key, node, module, nil) return } have, err := delivery.DecodeHave(value) @@ -124,6 +136,7 @@ func (w *Watchman) Have(key string, value []byte, op Op) { return } w.haves[key] = have.FP + w.haveFindings(key, node, module, &have) wk, err := delivery.WantKey(node, module) if err != nil { return diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go new file mode 100644 index 0000000..d3b6dd3 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -0,0 +1,432 @@ +package watch + +import ( + "fmt" + "maps" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A node's own health report (novox/hq ADR 0241, issue 315, the operator's alert table of 2026-10-29): each node's engine +// writes `mesh-host_health.` when what it says changes, and the watchman raises from it what the controller raised +// from the passing health message, under the same keys and words (mesh-control cmd/mesh-controller machine_network.go, +// machine_units.go, module_health.go at e04846e9): +// - machine-network `machine..network`: the machine's own unhealthy parts; urgent on the control node or for the bus; +// - network-rewritten `machine.[.].rewritten`: the resolver file rewritten by another program; +// - network-unreachable `machine..unreachable`: two or more nodes' tunnels no longer see it (their peers' handshake), +// said once there, never while its own network or its silence says it; +// - machine-units `machine..units`: failed units no module places; +// - relogin-needed `module...relogin-needed` and directory-used-as-found +// `module...directory-used-as-found`. +// +// A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held +// there, as the controller holds it: when every machine it points at is down on the record (silent, its own network +// unhealthy, or two machines failing toward it), it is that machine's `unreachable`, said once there, and no finding of +// the reporter's. A tunnel peer two or more nodes no longer see points at its node the same way. + +// The kinds a node's health report raises. +const ( + KindMachineNetwork = "machine-network" + KindNetworkRewritten = "network-rewritten" + KindNetworkUnreachable = "network-unreachable" + KindMachineUnits = "machine-units" + KindReloginNeeded = "relogin-needed" + KindUsedAsFound = "directory-used-as-found" +) + +// The parts and states a report names (mesh-control internal/link). +const ( + partResolvConf = "resolv-conf" + partNames = "names" + partTunnel = "tunnel" + partBus = "bus" + stateUnhealthy = "unhealthy" +) + +func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } + +// NodeHealth is a change of a node's report: written, or deleted (value nil). +func (w *Watchman) NodeHealth(node string, value []byte, op Op) { + w.mu.Lock() + defer w.mu.Unlock() + if op == Put { + h, err := delivery.DecodeNodeHealth(value) + if err != nil { + w.say("the health report of %s cannot be read, and is left: %v", node, err) + return + } + w.health[node] = h + } else { + delete(w.health, node) + } + w.nodeFindings(node) + w.networks() +} + +// TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it. +func (w *Watchman) tunnelKey(node string, record *delivery.NodeRecord) { + key := "" + if record != nil { + key = record.TunnelKey + } + if w.tunnels[node] == key { + return + } + if key == "" { + delete(w.tunnels, node) + } else { + w.tunnels[node] = key + } + w.networks() +} + +// nodeFindings writes and deletes one node's own findings from its report. +func (w *Watchman) nodeFindings(node string) { + h, ok := w.health[node] + want := map[string]bool{} + if ok { + if f, raise := unitsOf(node, h.Units); raise { + want[f.Key()] = true + w.find(f) + } + for _, f := range reloginOf(node, h.Relogin) { + want[f.Key()] = true + w.find(f) + } + for _, f := range foundOf(node, h.Found) { + want[f.Key()] = true + w.find(f) + } + } + for key := range w.findings { + if !want[key] && w.ownsHealthKey(key, node) { + w.unfind(key) + } + } +} + +// ownsHealthKey says a finding key is one of node's own report findings. +func (w *Watchman) ownsHealthKey(key, node string) bool { + f, err := DecodeFinding(w.findings[key]) + if err != nil || f.Machine != node || f.Source != "health-report" { + return false + } + return f.Kind != KindNetworkUnreachable && f.Kind != KindMachineNetwork && f.Kind != KindNetworkRewritten +} + +func (w *Watchman) rewrittenOf(m string, p delivery.NetworkPart, all []delivery.NetworkPart) Finding { + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names still resolve through what it wrote" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == partNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + id := m + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ + "back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + id = m + "." + p.Owner + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ + "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + severity := Warning + if w.control[m] { + severity = Urgent + } + return Finding{Scope: ScopeMachine, ID: id, Token: "rewritten", Kind: KindNetworkRewritten, Machine: m, Severity: severity, + Source: "health-report", Since: p.Since, Summary: summary, + Said: fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))} +} + +// unitsOf is a machine's failed units no module places, one finding. +func unitsOf(node string, u *delivery.UnitsHealth) (Finding, bool) { + if u == nil || len(u.Failed) == 0 { + return Finding{}, false + } + var names, said []string + since := u.Failed[0].Since + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + scope, result := f.Scope, f.Result + if scope == "" { + scope = "system" + } + if result == "" { + result = "no reason said" + } + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + said = append(said, line+", since "+stampSec(f.Since)) + if f.Since.Before(since) { + since = f.Since + } + } + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + return Finding{Scope: ScopeMachine, ID: node, Token: "units", Kind: KindMachineUnits, Machine: node, Severity: Warning, + Source: "health-report", Since: since, + Summary: fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ + "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), + Said: strings.Join(said, "; "), + Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node, + Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node), + Resolved: "No failed services on " + node + " any more"}, true +} + +// reloginOf is one finding per module whose accounts wait for a new login. +func reloginOf(node string, accounts []delivery.NodeAccount) []Finding { + by := map[string][]delivery.NodeAccount{} + for _, a := range accounts { + by[a.Module] = append(by[a.Module], a) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var names []string + since := by[module][0].Since + for _, a := range by[module] { + names = append(names, a.Account) + if a.Since.Before(since) { + since = a.Since + } + } + sort.Strings(names) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ + "before it was; log out of %[1]s completely and log in again, or restart it", node, orModule(module), strings.Join(names, ", ")) + out = append(out, Finding{Scope: ScopeModule, ID: orModule(module) + "." + node, Kind: KindReloginNeeded, Machine: node, + Severity: Warning, Source: "health-report", Since: since, Summary: said, Said: said}) + } + return out +} + +// foundOf is one finding per module with a directory used as found. +func foundOf(node string, dirs []delivery.NodeDirectory) []Finding { + by := map[string][]delivery.NodeDirectory{} + for _, d := range dirs { + by[d.Module] = append(by[d.Module], d) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var ids, why []string + since := by[module][0].Since + for _, d := range by[module] { + ids = append(ids, d.Resource) + why = append(why, d.Reason) + if d.Since.Before(since) { + since = d.Since + } + } + m := orModule(module) + said := fmt.Sprintf("%s on %s uses %s as found: %s", m, node, strings.Join(ids, ", "), strings.Join(why, "; ")) + out = append(out, Finding{Scope: ScopeModule, ID: m + "." + node, Kind: KindUsedAsFound, Machine: node, Severity: Warning, + Source: "health-report", Since: since, Summary: said, Said: said, + Headline: fmt.Sprintf("%s waits for a directory on %s", m, node), + Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ + "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m), + Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type.", + Resolved: fmt.Sprintf("%s's directory on %s is the mesh's", m, node)}) + } + return out +} + +func orModule(m string) string { + if m == "" { + return "_" + } + return m +} + +// pointed is one machine's failing part that points at another machine. +type pointed struct { + from string + part delivery.NetworkPart +} + +// networks is every machine's network findings from every report together (the controller's networkObservations): the +// resolver file rewritten, each machine's own unhealthy parts, and the machine others point at, held there. +func (w *Watchman) networks() { + machines := slices.Sorted(maps.Keys(w.health)) + unhealthy := func(m string) []delivery.NetworkPart { + h := w.health[m] + if h.Network == nil { + return nil + } + var out []delivery.NetworkPart + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + out = append(out, p) + } + } + return out + } + targets := func(m string, p delivery.NetworkPart) ([]string, bool) { + var out []string + for _, x := range p.Targets { + if x == "" || x == m { + return nil, false + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out, len(out) > 0 + } + pointing := map[string][]pointed{} + for _, m := range machines { + for _, p := range unhealthy(m) { + if xs, ok := targets(m, p); ok { + for _, x := range xs { + pointing[x] = append(pointing[x], pointed{from: m, part: p}) + } + } + } + } + // A tunnel peer a node no longer sees points at that peer's node, as a tunnel part toward it would. + byKey := map[string]string{} + for node, key := range w.tunnels { + byKey[key] = node + } + for _, m := range machines { + for _, p := range w.health[m].Peers { + if x, ok := byKey[p.Key]; ok && x != m && !p.Seen { + pointing[x] = append(pointing[x], pointed{from: m, part: delivery.NetworkPart{Part: partTunnel, State: stateUnhealthy, + Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}}) + } + } + } + from := func(x string) []string { + var out []string + for _, pt := range pointing[x] { + if !slices.Contains(out, pt.from) { + out = append(out, pt.from) + } + } + sort.Strings(out) + return out + } + silent := func(x string) bool { + _, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)] + return ok + } + down := func(x string) bool { return silent(x) || len(unhealthy(x)) > 0 || len(from(x)) >= 2 } + allDown := func(xs []string) bool { + for _, x := range xs { + if !down(x) { + return false + } + } + return len(xs) > 0 + } + want := map[string]bool{} + saysOwn := map[string]bool{} + for _, m := range machines { + parts := unhealthy(m) + if len(parts) == 0 { + continue + } + var own []delivery.NetworkPart + var rewritten *delivery.NetworkPart + for i, p := range parts { + if p.Part == partResolvConf { + rewritten = &parts[i] + continue + } + if xs, ok := targets(m, p); ok && allDown(xs) { + continue // held at the machines it points at + } + own = append(own, p) + } + if rewritten != nil { + f := w.rewrittenOf(m, *rewritten, parts) + want[f.Key()] = true + w.find(f) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) == 0 { + continue + } + var words, said []string + severity := Warning + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = Urgent + } + } + if w.control[m] { + severity = Urgent + } + summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) + if waiting := from(m); len(waiting) > 0 { + severity = Urgent + summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) + } + f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Severity: severity, + Source: "health-report", Since: w.health[m].Network.Since, Summary: summary, Said: strings.Join(said, " | ")} + want[f.Key()] = true + saysOwn[m] = true + w.find(f) + } + for _, x := range slices.Sorted(maps.Keys(pointing)) { + if !down(x) || saysOwn[x] || silent(x) { + continue + } + var what, said []string + for _, pt := range pointing[x] { + wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[pt.part.Part] + if wd == "" { + wd = pt.part.Part + } + if !slices.Contains(what, wd) { + what = append(what, wd) + } + said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) + } + severity := Warning + if w.control[x] || slices.Contains(what, "the bus on it") { + severity = Urgent + } + f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from(x), + Severity: severity, Source: "health-report", + Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from(x), ", "), strings.Join(what, ", ")), + Said: strings.Join(said, " | ")} + want[f.Key()] = true + w.find(f) + } + for key, value := range w.findings { + f, err := DecodeFinding(value) + if err != nil || f.Source != "health-report" { + continue + } + if (f.Kind == KindMachineNetwork || f.Kind == KindNetworkRewritten || f.Kind == KindNetworkUnreachable) && !want[key] { + w.unfind(key) + } + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/presence.go b/modules/mesh-delivery-watchman/internal/watch/presence.go index b6bcab2..369e73c 100644 --- a/modules/mesh-delivery-watchman/internal/watch/presence.go +++ b/modules/mesh-delivery-watchman/internal/watch/presence.go @@ -87,6 +87,9 @@ func (w *Watchman) settle(key string) { return } w.presence[key] = presence{online: online, since: w.now()} + if node, part, _, err := delivery.ParsePresenceKey(key); err == nil && part == "" && node != "" { + defer w.applyCheck(node) // S21 is judged only while the machine is heard + } if online { w.moved(delivery.PresenceBucket, key) return diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go new file mode 100644 index 0000000..214ee3f --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -0,0 +1,857 @@ +package watch + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The rows of the signals table the watchman holds (novox/hq ADR 0564 §4, Limits 3 and 11). Each is a key of the pulse +// bucket whose time to live is the row's bound, renewed by the signal's own event: its expiry is the finding, so no loop +// looks every half minute. A signal's pulse is kept under the pulse bucket's pseudo-bucket `signal`, `signal..`, +// and its value says what its expiry does (signalDue), so a pulse that stood across a restart still acts when it expires. +// +// What each row hears is on the bus, never in the controller's store: +// - S1, a machine silent: the watchman's own presence of `node.` (its stuck key), not while the machine said it +// sleeps or shuts down (the power seat's events, ADR 0211); +// - S8, a provider's failing word repeated: the providers' `provisioner.failing` and `provisioner.recovered` (ADR 0224); +// the failing word is itself the finding `provider-failing`, and its pulse of 30 minutes, renewed by each repeat, is S8; +// - S9, the bus's advisories: each occurrence a finding, cleared by the expiry of an hour's pulse renewed by the next; +// - S10, the self-check's `checked`: a pulse of three runs' length, from the first run heard; +// - S11, a machine's tool runner heartbeat on `mesh.control..tools-alive`: a pulse of three intervals; +// - S21, a module's have failed: a pulse of one apply interval from the first failure, whose expiry while it still fails +// is the finding `apply-failing`; +// - a module's have unhealthy in two writes in a row, `unhealthy`, urgent after four hours; and a have that waits for +// the operator, `needs-operator` (ADR 0240 §12, §17). + +// The bounds of the rows. +const ( + // ProviderSaysAgainWithin is how long a provider's failing word stays current without being said again (S8): twice + // the quarter of an hour it repeats it at. + ProviderSaysAgainWithin = 30 * time.Minute + // AdvisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9). + AdvisoryQuiet = time.Hour + // SelfCheckEvery is the self-check's run interval; S10's bound is three of them. + SelfCheckEvery = 5 * time.Minute + // HeartbeatEvery is a tool runner's interval when it says none; S11's bound is three of them. + HeartbeatEvery = 60 * time.Second + // ApplyEvery is one apply interval (S21): a failure still there after it is not one the next apply clears. + ApplyEvery = 5 * time.Minute + // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. + UnhealthyUrgentAfter = 4 * time.Hour + // SilentBound is the bound S1 says, three of the intervals a machine has always used, and ControlNodeUrgentAfter + // how long a control node may be silent before it is urgent (the controller's silentBound, controlNodeUrgentAfter). + SilentBound = 3 * HeartbeatEvery + ControlNodeUrgentAfter = 30 * time.Minute +) + +// signalBucket is the pseudo-bucket of a signal's pulse key. +const signalBucket = "signal" + +// signalDue is what a signal's pulse does when it expires: raise the finding (find), or end it (end, an advisory's quiet +// hour). Kept as the pulse's Owes, so it is read back at a restart. +type signalDue struct { + Do string `json:"do"` + Finding Finding `json:"finding"` + // While names what must still stand at the expiry for a find to be written: a have's key still failing or unhealthy; + // empty for a row whose expiry alone is the finding. + While string `json:"while,omitempty"` +} + +const ( + dueFind = "find" + dueEnd = "end" + // dueApply judges a machine's S21 again (applyCheck); dueRecount counts DEAD_LETTERS again (a restricted letter aged out). + dueApply = "apply" + dueRecount = "recount" +) + +// signalKey is the pulse key of a row's signal about id. +func signalKey(row, id string) string { + parts := []string{signalBucket, row} + for _, p := range strings.Split(id, ".") { + if p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_"); p == "" { + p = "_" + } + parts = append(parts, p) + } + return strings.Join(parts, ".") +} + +// unsafeKey is what a key's token may not hold: the bus takes letters, digits and `-_/=`. +var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) + +// signal writes (or renews) the pulse of a row's signal, whose expiry does due. +func (w *Watchman) signal(row, id string, ttl time.Duration, due signalDue) { + pk := signalKey(row, id) + owes, err := json.Marshal(due) + if err != nil { + w.say("no signal %s: %v", pk, err) + return + } + value, err := delivery.EncodePulse(delivery.Pulse{Bucket: signalBucket, Key: strings.TrimPrefix(pk, signalBucket+"."), Owes: string(owes)}) + if err != nil { + w.say("no signal %s: %v", pk, err) + return + } + if err := w.out.PutPulse(pk, value, ttl); err != nil { + w.say("cannot write the signal %s: %v", pk, err) + return + } + w.signals[pk] = due +} + +// unsignal ends a row's signal about id: its pulse deleted, so it never expires. +func (w *Watchman) unsignal(row, id string) { + pk := signalKey(row, id) + if _, ok := w.signals[pk]; !ok { + return + } + delete(w.signals, pk) + if err := w.out.DeletePulse(pk); err != nil { + w.say("cannot delete the signal %s: %v", pk, err) + } +} + +// SeedSignal tells the watchman a signal's pulse that stands at its start, with its value: it is kept, since its event +// renews it and its expiry still acts. +func (w *Watchman) SeedSignal(key string, value []byte) { + w.mu.Lock() + defer w.mu.Unlock() + if due, ok := dueOf(value); ok { + w.signals[key] = due + } +} + +func dueOf(value []byte) (signalDue, bool) { + p, err := delivery.DecodePulse(value) + if err != nil || p.Bucket != signalBucket { + return signalDue{}, false + } + var due signalDue + if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd && due.Do != dueApply && due.Do != dueRecount) { + return signalDue{}, false + } + return due, true +} + +// signalExpired is a signal's pulse expired: its due is done. +func (w *Watchman) signalExpired(pk string) { + due, ok := w.signals[pk] + if !ok { + return + } + delete(w.signals, pk) + switch due.Do { + case dueApply: + w.applyCheck(due.While) + case dueRecount: + if w.recount != nil { + go w.recount() + } + case dueEnd: + w.unfind(due.Finding.Key()) + case dueFind: + if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { + return + } + w.find(due.Finding) + } +} + +// stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for +// unhealthy. +func (w *Watchman) stillWrong(haveKey, kind string) bool { + if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { + _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] + return open + } + h, ok := w.haveState[haveKey] + if !ok { + return false + } + switch kind { + case KindApplyFailing: + return h.Phase == delivery.PhaseFailed + case KindUnhealthy: + return h.Health == delivery.HealthFailed + } + return false +} + +// The kinds the watchman's rows write, as the controller's conditions name them. +const ( + KindSilent = "silent" + KindStuck = "stuck" + KindProviderFailing = "provider-failing" + KindProviderSilent = "provider-silent" + KindSelfCheckSilent = "self-check-silent" + KindToolsSilent = "tools-silent" + KindApplyFailing = "apply-failing" + KindUnhealthy = "module-unhealthy" + KindNeedsOperator = "needs-operator" + KindDeadLetters = "max-deliveries" +) + +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } + +// ---- S1 and the power seat ---- + +// PowerEvents are the power seat's events that say whether a machine is about to be away or is back (ADR 0211). +var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true} + +// Power is a machine's power event: sleeping and shutting-down say it is away, booted and woke that it is back. A machine +// that said it is away is not silent; one back that is still offline is watched again from now. +func (w *Watchman) Power(node, state string) { + if !PowerEvents[state] || node == "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + away := state == "sleeping" || state == "shutting-down" + w.away[node] = away + key, err := delivery.PresenceNodeKey(node) + if err != nil { + return + } + defer w.applyCheck(node) + if away { + w.unfind(FindingKey(ScopeMachine, node, KindSilent)) + w.unsignal("S1", node) + return + } + if p, ok := w.presence[key]; ok && !p.online { + p.since = w.now() + w.presence[key] = p + w.ensureOffline(key) + } +} + +// stuckFinding is the finding of a stuck key as it stands (ADR 0433 §2, §3, now the watchman's own): a node's presence not +// moved is S1, the machine silent, unless it said it is away; any other key not moved is `delivery..stuck`; a key +// moved ends its finding. +func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { + if s.Bucket == delivery.PresenceBucket { + if node, part, _, err := delivery.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { + key := FindingKey(ScopeMachine, node, KindSilent) + defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence + if s.Moved || w.away[node] { + w.unfind(key) + w.unsignal("S1", node) + w.applyCheck(node) + return + } + f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", + Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound), + Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since} + // A control node, the machine running the controller, is urgent after half an hour silent, as the controller + // says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop. + if w.control[node] { + if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter { + f.Severity = Urgent + } else { + u := f + u.Severity = Urgent + w.signal("S1", node, ControlNodeUrgentAfter-age+time.Second, signalDue{Do: dueFind, While: "silent:" + node, Finding: u}) + } + } + w.find(f) + w.applyCheck(node) + return + } + } + key := FindingKey(ScopeDelivery, sk, KindStuck) + if s.Moved { + w.unfind(key) + return + } + waited := s.Waited + if waited == "" { + waited = "a move" + } + w.find(Finding{Scope: ScopeDelivery, ID: sk, Kind: KindStuck, Severity: Warning, Source: "stuck", + Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", sk, stamp(s.Since), waited), + Said: fmt.Sprintf("%s: waiting for %s since %s", sk, waited, s.Since.UTC().Format(time.RFC3339)), Since: s.Since}) +} + +// ---- S8 ---- + +// Standing is a provider's word about one consumer (ADR 0224): Module is the emitter, read from the subject. +type Standing struct { + Module string `json:"-"` + Failing bool `json:"-"` + Provider string `json:"provider"` + ProviderNode string `json:"provider-node"` + Consumer string `json:"consumer"` + Node string `json:"node"` + Class string `json:"class,omitempty"` + Error string `json:"error,omitempty"` + Since time.Time `json:"since"` + Attempts int `json:"attempts"` + Why string `json:"why,omitempty"` +} + +// ReadStanding reads a provider's word from its subject, `mesh.mod..event.provisioner.failing` or `.recovered`. +func ReadStanding(subject string, body []byte) (Standing, error) { + rest, ok := strings.CutPrefix(subject, "mesh.mod.") + if !ok { + return Standing{}, fmt.Errorf("%s is not a provider's word", subject) + } + module, event, ok := strings.Cut(rest, ".event.") + if !ok || module == "" || (event != "provisioner.failing" && event != "provisioner.recovered") { + return Standing{}, fmt.Errorf("%s is not a provider's word", subject) + } + var st Standing + if err := json.Unmarshal(body, &st); err != nil { + return Standing{}, fmt.Errorf("%s's word could not be read: %w", module, err) + } + if st.Consumer == "" { + return Standing{}, fmt.Errorf("%s's word names no consumer", module) + } + st.Module, st.Failing = module, event == "provisioner.failing" + return st, nil +} + +// Stood is a provider's word: failing writes `provider....failing` and renews S8's pulse, whose +// expiry writes `provider-silent`; recovered deletes both (Limits 11). +func (w *Watchman) Stood(st Standing) { + w.mu.Lock() + defer w.mu.Unlock() + id := st.Module + "." + st.ProviderNode + "." + st.Consumer + failing := FindingKey(ScopeProvider, id, "failing") + silentKey := FindingKey(ScopeProvider, id, "silent") + if !st.Failing { + w.unfind(failing) + w.unfind(silentKey) + w.unsignal("S8", id) + return + } + class := st.Class + if class == "" { + class = "failing" + } + whom := st.Consumer + if st.Node != "" { + whom += " on " + st.Node + } + said := class + if st.Error != "" { + said += ": " + firstLine(st.Error) + } + var also []string + if st.Node != "" && st.Node != st.ProviderNode { + also = []string{st.Node} + } + w.find(Finding{Scope: ScopeProvider, ID: id, Token: "failing", Kind: KindProviderFailing, Machine: st.ProviderNode, Also: also, + Severity: Warning, Source: "provisioner.failing", Since: st.Since, Said: said, + Summary: fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode, whom, class, + st.Attempts, stamp(st.Since))}) + w.unfind(silentKey) + now := w.now() + w.signal("S8", id, ProviderSaysAgainWithin, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeProvider, ID: id, Token: "silent", + Kind: KindProviderSilent, Machine: st.ProviderNode, Severity: Warning, Source: "S8", Since: now, + Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped saying anything, so "+ + "its last word is all the mesh has", st.Module, st.ProviderNode, st.Consumer, stamp(now)), + Said: fmt.Sprintf("not said again within %s", ProviderSaysAgainWithin)}}) +} + +func firstLine(s string) string { + s, _, _ = strings.Cut(strings.TrimSpace(s), "\n") + return s +} + +// ---- S9 ---- + +// Advisory is one thing the bus said (S9): its kind (slow-consumer, connection-closed, max-deliveries, refused, +// consumer-lost), what it is about, its words, and the machine when it names one. +type Advisory struct { + Kind, ID, Said, Machine string +} + +// Advised is an advisory heard: its finding `bus..` is written at once, counting the times since its first, and +// an hour's pulse renewed by each, whose expiry ends it. +func (w *Watchman) Advised(a Advisory) { + if a.Kind == "" || a.ID == "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: a.ID, Kind: a.Kind, Machine: a.Machine, Severity: Warning, Source: "S9", Said: a.Said} + key := f.Key() + n := w.advised[key] + 1 + if _, open := w.findings[key]; !open { + n = 1 + w.advisedSince[key] = w.now() + } + w.advised[key] = n + since := w.advisedSince[key] + f.Since, f.Summary = since, a.Said + if n > 1 { + f.Summary = fmt.Sprintf("%s (%d times since %s)", a.Said, n, since.UTC().Format("15:04 MST")) + } + w.find(f) + w.signal("S9", key, AdvisoryQuiet, signalDue{Do: dueEnd, Finding: f}) +} + +// ---- S10 ---- + +// Checked is the self-check's `checked` heard: its pulse of three runs' length is renewed, and its silence ended. Before +// the first run heard there is no pulse, so a self-check not yet placed is not said silent. +func (w *Watchman) Checked(machine string) { + w.mu.Lock() + defer w.mu.Unlock() + w.unfind(FindingKey(ScopeCore, "doctor", KindSelfCheckSilent)) + now := w.now() + w.signal("S10", "doctor", 3*SelfCheckEvery, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeCore, ID: "doctor", + Kind: KindSelfCheckSilent, Machine: machine, Severity: Urgent, Source: "S10", Since: now, + Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's invariants are not "+ + "being checked", stamp(now), SelfCheckEvery), + Said: fmt.Sprintf("no run within %s", 3*SelfCheckEvery)}}) +} + +// ---- S11 ---- + +// ToolsAlive is a machine's tool runner heartbeat, saying its interval (zero: HeartbeatEvery): its pulse of three +// intervals is renewed and its silence ended. +func (w *Watchman) ToolsAlive(node string, every time.Duration) { + if node == "" { + return + } + if every <= 0 { + every = HeartbeatEvery + } + w.mu.Lock() + defer w.mu.Unlock() + w.unfind(FindingKey(ScopeMachine, node, KindToolsSilent)) + if w.away[node] { + w.unsignal("S11", node) + return + } + now := w.now() + w.signal("S11", node, 3*every, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindToolsSilent, + Machine: node, Severity: Warning, Source: "S11", Since: now, + Summary: fmt.Sprintf("the node tools on %s have not said they are there since %s (bound %s): nothing can ask that "+ + "machine anything", node, stamp(now), 3*every), + Said: fmt.Sprintf("silent for more than %s", 3*every)}}) +} + +// ToolsNode is the machine a tool runner heartbeat's subject names, `mesh.control..tools-alive`. +func ToolsNode(subject string) string { + rest, ok := strings.CutPrefix(subject, "mesh.control.") + if !ok { + return "" + } + node, ok := strings.CutSuffix(rest, ".tools-alive") + if !ok || strings.Contains(node, ".") { + return "" + } + return node +} + +// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ---- + +// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck), +// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have +// removed or deleted ends its part of them. +// +// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state +// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7). +func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) { + unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") + if module == ControllerModule { + w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved + } + if h == nil || h.Phase == delivery.PhaseRemoved { + delete(w.haveState, key) + delete(w.unhealthy, key) + delete(w.failedSince, key) + w.unsignal("unhealthy", module+"."+node) + w.unfind(unhealthyKey) + w.applyCheck(node) + return + } + w.haveState[key] = *h + now := w.now() + + // S21: a have refused or applied in part says phase failed, with the node's words in its detail. + if h.Phase == delivery.PhaseFailed { + if _, ok := w.failedSince[key]; !ok { + w.failedSince[key] = now + } + } else { + delete(w.failedSince, key) + } + w.applyCheck(node) + + // unhealthy: two writes in a row that say it failed. + if h.Health == delivery.HealthFailed { + w.unhealthy[key]++ + if w.unhealthy[key] == 2 { + f := Finding{Scope: ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: KindUnhealthy, Machine: node, + Severity: Warning, Source: "health", Since: now, + Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, orWords(h.Detail, "its check failed")), + Said: firstLine(orWords(h.Detail, "failed")), + Headline: fmt.Sprintf("%s not working on %s", module, node)} + w.find(f) + f.Severity = Urgent + w.signal("unhealthy", module+"."+node, UnhealthyUrgentAfter, signalDue{Do: dueFind, While: key, Finding: f}) + } + } else { + delete(w.unhealthy, key) + w.unsignal("unhealthy", module+"."+node) + w.unfind(unhealthyKey) + } +} + +// ControllerModule is the module whose have names a control node. +const ControllerModule = "mesh-controller" + +// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its +// consumer waiting for the operator, said as `module...needs-operator` naming each waiting provision and +// why; the finding ends when no provision of that consumer on that node waits. A key deleted, or one that cannot be read, +// waits no more. +func (w *Watchman) Provision(key string, value []byte, op Op) { + node, consumer, _, _, err := provisions.ParseKey(key) + if err != nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + id := consumer + "." + node + if w.waiting[id] == nil { + w.waiting[id] = map[string]string{} + } + delete(w.waiting[id], key) + if op == Put { + if p, err := provisions.Decode(value); err != nil { + w.say("the provision %s cannot be read, and is taken as not waiting: %v", key, err) + } else if p.State == provisions.Waits { + w.waiting[id][key] = fmt.Sprintf("%s from %s on %s: %s", p.Provision, p.Provider.Module, p.Provider.Node, p.Why) + } + } + fkey := FindingKey(ScopeModule, id, KindNeedsOperator) + if len(w.waiting[id]) == 0 { + delete(w.waiting, id) + w.unfind(fkey) + return + } + var said []string + for _, k := range sortedKeys(w.waiting[id]) { + said = append(said, w.waiting[id][k]) + } + w.find(Finding{Scope: ScopeModule, ID: id, Kind: KindNeedsOperator, Machine: node, Severity: Warning, Source: "provisions", + Summary: fmt.Sprintf("%s on %s waits for the operator: %s", consumer, node, strings.Join(said, "; ")), + Said: strings.Join(said, "; ")}) +} + +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// ProvReport is a machine's provisioner client's word on one module's files (`prov.applied..`, ADR 0362 §3). +type ProvReport struct { + Node string `json:"node"` + Module string `json:"module"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + At time.Time `json:"at"` +} + +// ProvFailed is the status of a report whose files were not placed. +const ProvFailed = "failed" + +// Provisioned is a provisioner client's newest report: a failure is kept for S21, any other word ends it. +func (w *Watchman) Provisioned(r ProvReport) { + if r.Node == "" || r.Module == "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if w.provFailed[r.Node] == nil { + w.provFailed[r.Node] = map[string]ProvReport{} + } + if r.Status == ProvFailed { + w.provFailed[r.Node][r.Module] = r + } else { + delete(w.provFailed[r.Node], r.Module) + } + w.applyCheck(r.Node) +} + +// heard says the watchman hears the machine now: online, and not said away. S21 is judged only while it is (the +// controller's watchApplyFailing): S1 says silent, and asleep is not lost. +func (w *Watchman) heard(node string) bool { + key, err := delivery.PresenceNodeKey(node) + if err != nil || w.away[node] { + return false + } + p, ok := w.presence[key] + return ok && p.online +} + +// applyCheck is S21 for one machine, `machine..apply-failing` (the controller's watchApplyFailing): raised once the +// first failure it still has is older than one apply interval while the machine is heard, naming the modules its node +// failed to apply (their haves failed), or else those whose files its provisioner client failed to place; ended by the +// clean apply. Before the bound a pulse judges it again at the bound. +func (w *Watchman) applyCheck(node string) { + key := FindingKey(ScopeMachine, node, KindApplyFailing) + type failure struct{ module, words string } + var applied, placed []failure + var since time.Time + for hk, at := range w.failedSince { + n, module, _, err := delivery.ParseHaveKey(hk) + if err != nil || n != node { + continue + } + applied = append(applied, failure{module, firstLine(orWords(w.haveState[hk].Detail, "no reason given"))}) + if since.IsZero() || at.Before(since) { + since = at + } + } + if len(applied) == 0 { + for module, r := range w.provFailed[node] { + placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))}) + if since.IsZero() || r.At.Before(since) { + since = r.At + } + } + } + if since.IsZero() || !w.heard(node) { + w.unsignal("S21", node) + w.unfind(key) + return + } + if age := w.now().Sub(since); age <= ApplyEvery { + if _, pending := w.signals[signalKey("S21", node)]; !pending { + w.signal("S21", node, ApplyEvery-age+time.Second, signalDue{Do: dueApply, While: node, + Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Severity: Warning, Summary: "judged again at its bound", Source: "S21"}}) + } + return + } + failed := append(applied, placed...) + sort.Slice(failed, func(i, j int) bool { return failed[i].module < failed[j].module }) + var names, words []string + tool := "" + for _, f := range failed { + names = append(names, f.module) + words = append(words, f.module+": "+f.words) + if rest, ok := strings.CutPrefix(f.words, "refused: "); ok { + f.words = rest + } + if i := strings.Index(f.words, PackageManagerKey); i >= 0 && tool == "" { + tool = strings.TrimSuffix(strings.SplitN(f.words[i+len(PackageManagerKey):], ")", 2)[0], ")") + } + } + f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since, + Said: strings.Join(words, "; "), Resolved: node + " applies what it is sent again", Headline: node + " is not applying its update"} + switch { + case len(applied) == 0: + f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node, + strings.Join(names, ", "), stamp(since), ApplyEvery) + f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.", + node, strings.Join(names, ", ")) + case tool != "": + f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, "+ + "since %s (bound %s)", node, tool, stamp(since), ApplyEvery) + f.Headline = node + " cannot install packages" + default: + f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", node, strings.Join(names, ", "), stamp(since), ApplyEvery) + f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", node) + } + w.find(f) +} + +// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed. +const PackageManagerKey = "the package manager (" + +// ---- S9: the bus module's closures ---- + +// BusClosure is one connection the bus closed, as the bus module says it (`bus.slow-consumer`, `bus.connection-closed`). +type BusClosure struct { + User string `json:"user"` + Name string `json:"name,omitempty"` + CID uint64 `json:"cid"` + Reason string `json:"reason"` + Opened time.Time `json:"opened"` + Closed time.Time `json:"closed"` + Pending int64 `json:"pending_bytes,omitempty"` +} + +// The kinds of the bus module's closures, as conditions name them. +const ( + KindSlowConsumer = "slow-consumer" + KindConnectionClosed = "connection-closed" +) + +// Closed is one closure the bus module said (S9, the controller's BusClosures and busUserClosed): one finding per user and +// kind, `bus..`, each closure counted once however often it is said, ended an hour after the newest. +func (w *Watchman) Closed(kind string, c BusClosure) { + if c.User == "" || c.Closed.IsZero() || (kind != KindSlowConsumer && kind != KindConnectionClosed) { + return + } + w.mu.Lock() + defer w.mu.Unlock() + f := Finding{Scope: ScopeBus, ID: c.User, Kind: kind, Severity: Warning, Source: "S9"} + key := f.Key() + if _, open := w.findings[key]; !open { + w.closures[key] = map[string]time.Time{} + } + identity := fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano)) + if _, seen := w.closures[key][identity]; seen { + return + } + w.closures[key][identity] = c.Closed + var first, last time.Time + for _, at := range w.closures[key] { + if first.IsZero() || at.Before(first) { + first = at + } + if at.After(last) { + last = at + } + } + n := len(w.closures[key]) + who, machine := BusUserWords(c.User) + f.Machine, f.Since = machine, first + times, count := "once", "once" + if n > 1 { + times = fmt.Sprintf("%d times since %s", n, first.UTC().Format("15:04 MST")) + count = fmt.Sprintf("%d times", n) + } + if kind == KindSlowConsumer { + f.Said = fmt.Sprintf("the bus closed %s's connection %d as a slow consumer: %s", c.User, c.CID, c.Reason) + f.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, newest at %s: it "+ + "could not take what it was sent fast enough, and what was on its way to it was lost", who, times, last.UTC().Format("15:04 MST")) + f.Headline = clip(capital(who)+" was too slow for the bus", 60) + f.Explanation = capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus closed its connection (%s "+ + "in the last hour) and what was on its way was lost. It connects again by itself; this clears an hour after the last time.", who, count)) + f.Resolved = "Resolved: " + who + " kept up for an hour" + } else { + reason := strings.ToLower(c.Reason) + f.Said = fmt.Sprintf("the bus closed %s's connection %d: %s", c.User, c.CID, c.Reason) + f.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, newest at %s: %s", + who, times, last.UTC().Format("15:04 MST"), reason) + f.Headline = clip(capital(who)+" was cut off by the bus", 60) + f.Explanation = capital(fmt.Sprintf("The bus closed the connection of %s because it broke the bus's rules (%s, %s in the "+ + "last hour), so what it was sending was lost. This clears an hour after the last time.", who, reason, count)) + f.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour" + } + w.find(f) + if quiet := AdvisoryQuiet - w.now().Sub(last); quiet > 0 { + w.signal("S9", key, quiet, signalDue{Do: dueEnd, Finding: f}) + } else { + w.unfind(key) + } +} + +// BusUserWords is a bus user as the operator says it, and its machine (the controller's busUserWords). +func BusUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + cut := s[:n] + if i := strings.LastIndex(cut, " "); i > 0 { + cut = cut[:i] + } + return cut +} + +// ---- dead letters aging out ---- + +// SetRecount sets how DEAD_LETTERS is counted again when a restricted letter ages out. +func (w *Watchman) SetRecount(f func()) { + w.mu.Lock() + defer w.mu.Unlock() + w.recount = f +} + +// DeadAging is a restricted letter of a consumer (`.`) that ages out at at: a pulse expiring a second +// after it counts DEAD_LETTERS again, so the consumer's finding ends when its letters are gone (ADR 0376 §6), with no +// timer. The earliest of a consumer's letters is the one pulsed. +func (w *Watchman) DeadAging(key string, at time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + left := at.Sub(w.now()) + if left <= 0 { + left = 0 + } + pk := signalKey("dead", key) + if cur, ok := w.signals[pk]; ok && !cur.Finding.Since.IsZero() && cur.Finding.Since.Before(at) && cur.Finding.Since.After(w.now()) { + return + } + w.signal("dead", key, left+time.Second, signalDue{Do: dueRecount, + Finding: Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Summary: "counted again when it ages out", Source: "S9", Since: at}}) +} + +func orWords(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +// SignalRows are the rows the watchman holds, in the table's order, for `signals`. +var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ + {"S1", "a machine's presence (not while it said it sleeps or shuts down); a control node urgent after " + ControlNodeUrgentAfter.String(), SilentBound.String(), KindSilent}, + {"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent}, + {"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"}, + {"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent}, + {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, + {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, + {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, + {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, +} + +// Signal is one signal's pulse as `signals` shows it. +type Signal struct { + Key string `json:"key"` + Do string `json:"do"` + Finding Finding `json:"finding"` +} + +// Signals is every signal's pulse the watchman holds, sorted by key. +func (w *Watchman) Signals() []Signal { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]Signal, 0, len(w.signals)) + for k, d := range w.signals { + out = append(out, Signal{Key: k, Do: d.Do, Finding: d.Finding}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index e81613b..cc4dcc3 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -31,6 +31,9 @@ type Out interface { // DeletePulse deletes the pulse key (a delete, not an expiry). DeletePulse(key string) error PutStuck(key string, value []byte) error + // PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes. + PutFinding(key string, value []byte) error + DeleteFinding(key string) error } type presence struct { @@ -60,6 +63,32 @@ type Watchman struct { wants map[string]string // the fingerprint of each want that stands, by want key haves map[string]string // the fingerprint each node runs of a module, by have key bound time.Duration + // findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed + // again. + findings map[string][]byte + seededFindings map[string]bool + // signals is every signal's pulse standing, by pulse key, with what its expiry does. + signals map[string]signalDue + // away is each machine that said it sleeps or shuts down, until it says it is back. + away map[string]bool + // haveState is each module's have as last read, unhealthy its writes in a row that said it failed, advised each + // advisory's count while its finding stands and advisedSince since when. + haveState map[string]delivery.Have + unhealthy map[string]int + advised map[string]int + advisedSince map[string]time.Time + // control is each node running the controller; failedSince each have's first failure still standing; provFailed each + // machine's provisioner client's failed reports; closures each bus closure counted, by finding key and identity. + control map[string]bool + failedSince map[string]time.Time + provFailed map[string]map[string]ProvReport + closures map[string]map[string]time.Time + recount func() + // waiting is, by `.`, each of its provisions in waits, with its words. + waiting map[string]map[string]string + // health is each node's own health report, and tunnels each node's tunnel key from its record. + health map[string]delivery.NodeHealth + tunnels map[string]string // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -67,7 +96,11 @@ type Watchman struct { // New is a watchman writing to out, reading the time from now. func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, - pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound} + pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, + haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, + control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, + health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}} } // Logf sets where a write that failed is said; it is silent until set. @@ -178,6 +211,7 @@ func (w *Watchman) putStuck(sk string, s delivery.Stuck) { } s.V = delivery.Version w.stuck[sk] = s + w.stuckFinding(sk, s) } // Pulse is a change heard on the pulse bucket. Only the server's expiry marker of a pulse the watchman made writes a @@ -188,6 +222,10 @@ func (w *Watchman) Pulse(key string, op Op, at time.Time) { if op != Expired { return } + if _, ok := w.signals[key]; ok { + w.signalExpired(key) + return + } p, ok := w.pulses[key] if !ok || p.seeded { return @@ -202,6 +240,7 @@ func (w *Watchman) SeedStuck(key string, value []byte) { defer w.mu.Unlock() if s, err := delivery.DecodeStuck(value); err == nil { w.stuck[key] = s + w.stuckFinding(key, s) } } @@ -211,7 +250,7 @@ func (w *Watchman) SeedPulse(key string) { w.mu.Lock() defer w.mu.Unlock() bucket, k, err := delivery.ParsePulseKey(key) - if err != nil { + if err != nil || bucket == signalBucket { return } w.pulses[key] = &pulse{bucket: bucket, key: k, seeded: true} @@ -227,6 +266,38 @@ func (w *Watchman) Reconcile() { w.deletePulse(pk) } } + w.reconcileFindings() +} + +// reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not +// observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a +// machine's silent tools, a silent self-check): those end by their own events or expiry. +func (w *Watchman) reconcileFindings() { + kept := map[string]bool{} + for _, due := range w.signals { + kept[due.Finding.Key()] = true + } + for key := range w.seededFindings { + delete(w.seededFindings, key) + if kept[key] || signalKind(w.findings[key]) { + continue + } + w.unfind(key) + } +} + +// signalKind says a finding is one a signal row writes and ends by its own event (S8, S9, S10, S11): it outlives a +// restart, since what ends it is heard, not read. +func signalKind(value []byte) bool { + f, err := DecodeFinding(value) + if err != nil { + return false + } + switch f.Source { + case "provisioner.failing", "S8", "S9", "S10", "S11": + return true + } + return false } func describe(format string, args ...any) string { return fmt.Sprintf(format, args...) } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 28ea6f0..edcdec9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -19,10 +19,31 @@ type fakeOut struct { pulseDeletes []string stuck map[string]delivery.Stuck stuckWrites int + findings map[string]Finding + findingWrites int } func newOut() *fakeOut { - return &fakeOut{presence: map[string]bool{}, pulses: map[string]delivery.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]delivery.Stuck{}} + return &fakeOut{presence: map[string]bool{}, pulses: map[string]delivery.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]delivery.Stuck{}, + findings: map[string]Finding{}} +} + +func (o *fakeOut) PutFinding(key string, value []byte) error { + f, err := DecodeFinding(value) + if err != nil { + return err + } + if f.Key() != key { + return errors.New("a finding written under another key than its own: " + key) + } + o.findings[key] = f + o.findingWrites++ + return nil +} + +func (o *fakeOut) DeleteFinding(key string) error { + delete(o.findings, key) + return nil } func (o *fakeOut) PutPresence(key string, online bool) error { @@ -116,7 +137,12 @@ func stuckKeyOf(t *testing.T, bucket, key string) string { return k } -func TestPresenceFromEvents(t *testing.T) { // 1135/AC1 +func TestPresenceFromEventsWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 + t.Run("from events", presenceFromEvents) + t.Run("written only when it differs", presenceWrittenOnlyWhenItDiffers) +} + +func presenceFromEvents(t *testing.T) { // 1135/AC1 w, o, _ := setup() w.Connect("node.ace") w.Connect("ace.mesh-delivery-view") @@ -135,7 +161,7 @@ func TestPresenceFromEvents(t *testing.T) { // 1135/AC1 } } -func TestPresenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 +func presenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1 w, o, _ := setup() w.Connect("node.ace") w.Connect("node.ace") @@ -178,7 +204,12 @@ func connzAnswer(users ...string) []byte { return b } -func TestRebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 +func TestRebuildAsksConnectionsAndTakesWhoIsGone(t *testing.T) { // 1135/AC2 + t.Run("asks the connections with auth", rebuildAsksConnectionsWithAuth) + t.Run("takes the presence of who is gone", rebuildTakesThePresenceOfWhoIsGone) +} + +func rebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 w, o, _ := setup() var body []byte err := w.Rebuild(func(b []byte) ([]byte, error) { @@ -201,7 +232,7 @@ func TestRebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2 } } -func TestRebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2 +func rebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2 w, o, c := setup() w.SeedPresence("ace", true, t0.Add(-time.Hour)) w.SeedPresence("g14", true, t0.Add(-time.Hour)) @@ -416,7 +447,12 @@ func TestPresenceStuckThenBackOnline(t *testing.T) { // 1135/AC4 } } -func TestExpiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 +func TestOnlyTheExpiryOfItsOwnPulseWritesStuck(t *testing.T) { // 1135/AC5 + t.Run("the expiry marker writes stuck", expiryMarkerWritesStuck) + t.Run("its own delete of a pulse writes nothing", ownDeleteOfAPulseWritesNothing) +} + +func expiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 w, o, c := setup() w.Step("go", "n1.askid1", runningStep("askid1", t0), Put) c.t = t0.Add(120 * time.Second) @@ -430,7 +466,7 @@ func TestExpiryMarkerWritesStuck(t *testing.T) { // 1135/AC5 } } -func TestOwnDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5 +func ownDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5 w, o, c := setup() w.Step("go", "n1.askid1", runningStep("askid1", t0), Put) pk := pulseKeyOf(t, checksKey, "askid1") diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 58e2171..a57ddfe 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -25,19 +25,28 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` Context *struct { Repository string `json:"repository"` } `json:"context"` } +type claim struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` +} + type manifest struct { Module string `json:"module"` + Claims []claim `json:"claims"` + Tools []string `json:"tools"` Moved bool `json:"moved"` Version string `json:"version"` Slug string `json:"slug"` @@ -75,16 +84,24 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if m.Module != "mesh-delivery-watchman" || m.Moved || m.Slug != "watchman" { t.Errorf("module %q moved %v (it has no template, so it is not moved) slug %q", m.Module, m.Moved, m.Slug) } - want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}} + want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}, {Name: "findings", History: 5}} if !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } reads := slices.Clone(m.Reads) slices.Sort(reads) - if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) { + if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease", + "mesh-delivery-assigner.wants", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } - for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} { + // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). + if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { + t.Errorf("claims = %+v", m.Claims) + } + if !reflect.DeepEqual(m.Tools, []string{"watchman_findings"}) { + t.Errorf("tools = %v", m.Tools) + } + for _, k := range []string{"emits", "context", "items", "consumes", "provides"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the watchman has none", k) } @@ -97,25 +114,28 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } } -func TestManifestIsWholeInItsRepository(t *testing.T) { // 1135/AC6 - m, _ := load(t) - if len(m.Build.Artifacts) != 1 { - t.Fatalf("artifacts = %+v, want the one bundle program", m.Build.Artifacts) +// The manifest is whole in its repository: the program, and the tools bundle the runtime loads (1952/WM1). +func manifestIsWholeInItsRepository(t *testing.T, m manifest) { // 1135/AC6 + if len(m.Build.Artifacts) != 2 { + t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts) } - a := m.Build.Artifacts[0] - if a.Name != "program" || a.Kind != "bundle" || a.Language != "go" || a.System != "arch" || a.Context != nil { - t.Errorf("artifact = %+v", a) - } - if st, err := os.Stat(a.From); err != nil || !st.IsDir() { - t.Errorf("the artifact builds from %q, which is not a directory of this module", a.From) - } - if a.Binary != "mesh-delivery-watchman" || !strings.HasSuffix(a.From, a.Binary) { - t.Errorf("binary %q from %q", a.Binary, a.From) + for i, name := range []string{"program", "tools"} { + a := m.Build.Artifacts[i] + if a.Name != name || a.Kind != "bundle" || a.Language != "go" || a.System != "arch" || a.Context != nil { + t.Errorf("artifact = %+v", a) + } + if st, err := os.Stat(a.From); err != nil || !st.IsDir() { + t.Errorf("the artifact builds from %q, which is not a directory of this module", a.From) + } + if !strings.HasSuffix(a.From, a.Binary) || (name == "tools") != reflect.DeepEqual(a.Loads, []string{a.Binary}) { + t.Errorf("binary %q from %q loads %v", a.Binary, a.From, a.Loads) + } } } func TestManifestResources(t *testing.T) { // 1135/AC6 m, _ := load(t) + manifestIsWholeInItsRepository(t, m) byType := map[string][]map[string]any{} for _, r := range m.Resources { ty, _ := r["type"].(string) diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index c1c128f..fcc3077 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -2,17 +2,37 @@ "module": "mesh-delivery-watchman", "version": "1", "slug": "watchman", + "claims": [ + { + "name": "mesh-watchdogs", + "scope": "mesh", + "serves": [ + "signals", + "stuck", + "presence", + "dead-letters", + "resync" + ] + } + ], "reads": [ "mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", + "mesh-controller.lease", "mesh-delivery-assigner.wants", + "mesh-host.health", + "mesh-provisioner.provisions", "node-engine.have" ], + "tools": [ + "watchman_findings" + ], "state": [ { "name": "presence", "history": 20 }, { "name": "pulse", "key-ttl": true }, - { "name": "stuck", "history": 20 } + { "name": "stuck", "history": 20 }, + { "name": "findings", "history": 5 } ], "own-secrets": { "broker": "${dir:state}/broker" @@ -34,6 +54,14 @@ "place": ".", "unknown": "program" }, + { + "id": "socket-dir", + "type": "directory", + "path": "/var/lib/mesh-delivery-watchman-tools", + "mode": "0755", + "owner": "mesh-delivery-watchman", + "unknown": "program" + }, { "id": "watchman", "type": "process", @@ -60,6 +88,17 @@ "system": "arch", "from": "cmd/mesh-delivery-watchman", "binary": "mesh-delivery-watchman" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-watchman-tools", + "binary": "mesh-delivery-watchman-tools", + "loads": [ + "mesh-delivery-watchman-tools" + ] } ] } diff --git a/modules/mesh-grants/cmd/mesh-grants/bus.go b/modules/mesh-grants/cmd/mesh-grants/bus.go index e2ac265..7f839ea 100644 --- a/modules/mesh-grants/cmd/mesh-grants/bus.go +++ b/modules/mesh-grants/cmd/mesh-grants/bus.go @@ -214,6 +214,10 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) { return s.c.State(state).Put(key, json.RawMessage(value)) } +func (s ownStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() } + +func (s ownStore) Delete(state, key string) error { return s.c.State(state).Delete(key) } + // emitting is the process connection as the part's emitter: its events, with no options. type emitting struct{ c *process.Conn } diff --git a/modules/mesh-grants/cmd/mesh-grants/create.go b/modules/mesh-grants/cmd/mesh-grants/create.go index c155393..405a56b 100644 --- a/modules/mesh-grants/cmd/mesh-grants/create.go +++ b/modules/mesh-grants/cmd/mesh-grants/create.go @@ -18,6 +18,41 @@ type creating struct { js jetstream.JetStream } +// CreateConsumer creates one durable consumer an account needs, by its stream and name (ADR 0561 §6): the create alone; +// one the bus holds answers ErrHeld and is left as it is. +func (c creating) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error { + ctx, cancel := context.WithTimeout(c.ctx, requestWithin) + defer cancel() + if _, err := c.js.Consumer(ctx, stream, cfg.Durable); err == nil { + return grants.ErrHeld + } + _, err := c.js.CreateConsumer(ctx, stream, cfg) + switch { + case err == nil: + return nil + case errors.Is(err, jetstream.ErrConsumerExists): + return grants.ErrHeld + case errors.Is(err, context.DeadlineExceeded): + return fmt.Errorf("no answer in %s: a create the bus refuses is answered by none (%w)", requestWithin, err) + } + return err +} + +// MatchConsumer brings a held consumer to cfg on a person's ask (grants_consumer_match), keeping where it delivers from: +// the server refuses a changed deliver policy, and a consumer moved would replay or lose what it holds. +func (c creating) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error { + ctx, cancel := context.WithTimeout(c.ctx, requestWithin) + defer cancel() + held, err := c.js.Consumer(ctx, stream, cfg.Durable) + if err != nil { + return err + } + have := held.CachedInfo().Config + cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = have.DeliverPolicy, have.OptStartSeq, have.OptStartTime + _, err = c.js.UpdateConsumer(ctx, stream, cfg) + return err +} + func (c creating) CreateDeclared(cfg jetstream.StreamConfig) error { ctx, cancel := context.WithTimeout(c.ctx, requestWithin) defer cancel() diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index d971edc..6ef39b6 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -62,14 +62,9 @@ func main() { } } -// keptStates is what the part keeps, each readable (ADR 0495 §2-§3): the view and a session read them. -func keptStates() []announce.State { - var out []announce.State - for _, s := range grants.States { - out = append(out, announce.State{Name: s, Readable: true}) - } - return out -} +// keptStates is what the part keeps (ADR 0495 §2-§3): each readable, for the view and a session, but the wanted accounts, +// by the vault alone (ADR 0561 §2). +func keptStates() []announce.State { return grants.Announced() } // run is the process: it returns when ctx ends, or when it has no credential or cannot dial the bus at all. A bucket or // read the bus refuses never ends it. @@ -117,8 +112,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } bus := newBusReader(ctx, js) f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, - Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, - Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf} + Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, + Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf, + Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}} f.Watch = bus // The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on // the warrant once (mesh-sdk asker), after its refusals. @@ -138,8 +134,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) } // The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two - // condition subjects); it is pulled and every message acknowledged, so its pending never grows. Where it is not there - // yet the pass says so and hears again at the next change. + // condition subjects and the vault's account-issued; ADR 0561 §6 gives that to the part from the lease write); it is + // pulled and every message acknowledged, so its pending never grows. Where it is not there yet the pass says so and + // hears again at the next change. The vault's bus-accounts is watched beside it, so a hash is composed either way. f.Hear = func(handle func(string, []byte)) error { cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module) if err != nil { diff --git a/modules/mesh-grants/cmd/mesh-grants/tools.go b/modules/mesh-grants/cmd/mesh-grants/tools.go index 822fb2b..beefe2b 100644 --- a/modules/mesh-grants/cmd/mesh-grants/tools.go +++ b/modules/mesh-grants/cmd/mesh-grants/tools.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "strings" stdio "git.novox.be/novox/mesh-sdk/go" @@ -39,6 +40,10 @@ func (runtimeStore) Put(state, key string, value []byte) (uint64, error) { return stdio.State(state).Put(key, json.RawMessage(value)) } +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + // runtimeBus calls tools and says events through the runtime. type runtimeBus struct{} @@ -113,6 +118,43 @@ func toolsOf(p *grants.Part) []stdio.Tool { Run: func(a map[string]any) (any, error) { return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why")) }}, + {Name: grants.ToolAccounts, Description: "The bus accounts the mesh wants (novox/hq ADR 0561 §2, §9): every account the vault " + + "mints that placement composes (a module's, a process's, a machine's provisioner client's), as the grants part last " + + "wrote it to its state accounts, which the vault alone reads: user, kind, node, module, process, the bus's address and " + + "fingerprint, a module's claims with the verbs each seat promises, and when the key was written; one user's or all; " + + "what the last pass wrote, deleted or kept; and the durable consumers the accounts need as the last pass found them " + + "(missing, differing and why, created; whether the lease is handed away). Never a password. Read-only: " + + "grants_compose_now writes them again.", + Input: schema(map[string]string{"user": "one bus user, e.g. ace.mesh-build-go"}), + Run: func(a map[string]any) (any, error) { return p.Accounts(grants.Answer(a, "user")) }}, + {Name: grants.ToolConsumerMatch, Description: "Bring one durable consumer an account needs (a module's on EVENTS, a process's on " + + "its work queue; novox/hq ADR 0561 §6) to the configuration the bus composer composes, by hand: the recovery when " + + "the condition mesh-grants.consumer-differs names it. Asks the part's process, which alone may, and keeps where the " + + "consumer delivers from. Refused while the controller holds its lease (the consumers are the controller's then) and " + + "for a consumer the records do not compose. grants_user_list's last pass says what became of it.", + Input: schema(map[string]string{"stream": "the stream, e.g. EVENTS", "consumer": "the consumer, e.g. ace_mesh-build-go"}, "stream", "consumer"), + Run: func(a map[string]any) (any, error) { + return p.MatchConsumer(grants.Answer(a, "stream"), grants.Answer(a, "consumer")) + }}, + {Name: grants.ToolAcceptHashes, Description: "Release the hold of a user list that changes several loaded users' hashes at " + + "once (novox/hq ADR 0561 §7; the condition mesh-grants.user-list-held names them), by a person's act with why: names " + + "exactly the users held, bound to the held list's digest, so the part's process writes that list once at its next pass " + + "and says hashes-accepted. It never releases another hold (a failed read, a user dropped for want of a hash, a lost " + + "right), and a list composed otherwise since is held again for a new acceptance.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "users": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "the users held, as the condition names them"}, + "why": map[string]any{"type": "string", "description": "why their hashes change"}}, "required": []string{"users", "why"}}, + Run: func(a map[string]any) (any, error) { + var users []string + if list, ok := a["users"].([]any); ok { + for _, u := range list { + if s, ok := u.(string); ok { + users = append(users, strings.TrimSpace(s)) + } + } + } + return p.AcceptHashes(stdio.CallerOf(a), users, grants.Answer(a, "why")) + }}, {Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " + "every act with what became of it (asked, switched, declined, expired, refused and why).", Input: schema(nil), diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 4927f65..93d9d03 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index e742949..05084f7 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 85b58fd..7a9725e 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "path/filepath" "slices" @@ -21,6 +22,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/outward" "git.novox.be/novox/mesh-sdk/go/seal" @@ -52,11 +54,51 @@ type fakeBus struct { hold chan struct{} // creates are the streams the part asked to create, refused when refuseCreate. creates []string + createdAge map[string]time.Duration refuseCreate bool // refuseUsers refuses the bus module's nats_users: the loaded list cannot be read. refuseUsers bool // rootFree answers the controller's root-free verb: nil is no answerer at all (no responders). rootFree func(machines []string) (any, error) + // consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and + // consumerCreates each consumer the part asked to create, `.`. + consumers map[string][]map[string]any + consumerCreates []string +} + +// MatchConsumer is the bus's update of a held durable consumer to the configuration given. +func (b *fakeBus) MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error { + b.mu.Lock() + defer b.mu.Unlock() + b.consumerCreates = append(b.consumerCreates, "match "+stream+"."+cfg.Durable) + for _, c := range b.consumers[stream] { + if c["name"] == cfg.Durable { + c["max_deliver"] = cfg.MaxDeliver + return nil + } + } + return errors.New("consumer not found") +} + +// CreateConsumer is the bus's create of a durable consumer: held when the bus holds one of that name, else held from now. +func (b *fakeBus) CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error { + b.mu.Lock() + defer b.mu.Unlock() + b.consumerCreates = append(b.consumerCreates, stream+"."+cfg.Durable) + for _, c := range b.consumers[stream] { + if c["name"] == cfg.Durable { + return grants.ErrHeld + } + } + filters := append([]string{}, cfg.FilterSubjects...) + if cfg.FilterSubject != "" { + filters = []string{cfg.FilterSubject} + } + if b.consumers == nil { + b.consumers = map[string][]map[string]any{} + } + b.consumers[stream] = append(b.consumers[stream], map[string]any{"name": cfg.Durable, "filters": filters, "max_deliver": cfg.MaxDeliver}) + return nil } func (b *fakeBus) LastMembership(node, module string) ([]byte, uint64, bool, error) { @@ -92,6 +134,10 @@ func (b *fakeBus) CreateDeclared(cfg jetstream.StreamConfig) error { b.mu.Lock() defer b.mu.Unlock() b.creates = append(b.creates, cfg.Name) + if b.createdAge == nil { + b.createdAge = map[string]time.Duration{} + } + b.createdAge[cfg.Name] = cfg.MaxAge if b.refuseCreate { return errors.New("Permissions Violation for Publish to $JS.API.STREAM.CREATE." + cfg.Name) } @@ -177,6 +223,9 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) { } return json.Marshal(map[string]any{"users": out}) case grants.CallStreams: + if stream, _ := args["stream"].(string); stream != "" { + return json.Marshal(map[string]any{"detail": []map[string]any{{"name": stream, "consumer_detail": b.consumers[stream]}}}) + } return json.Marshal(map[string]any{"detail": b.streams}) case grants.CallRootFree: if b.rootFree == nil { @@ -217,6 +266,27 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) { return b.revs[state+"/"+key], nil } +func (s ownStore) Keys(state string) ([]string, error) { + b := s.b + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for k := range b.own[state] { + out = append(out, k) + } + slices.Sort(out) + return out, nil +} + +func (s ownStore) Delete(state, key string) error { + b := s.b + b.mu.Lock() + defer b.mu.Unlock() + b.writes = append(b.writes, state+" delete") + delete(b.own[state], key) + return nil +} + func (b *fakeBus) Emit(event string, _ any) error { b.mu.Lock() defer b.mu.Unlock() @@ -250,7 +320,7 @@ func (b *fakeBus) assign(node, module string) { const ( natsManifest = `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}` xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` + - `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history"}]}` + `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history","max-age-days":30}]}` yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}` ) @@ -417,8 +487,8 @@ func TestGrants(t *testing.T) { []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"}, {"a manifest missing is a gap and its user is left out", func(b *fakeBus) { delete(b.states[grants.ReadModules], "mod-y") }, []string{"a.mod-x", "a.nats", "controller", "node.a", "node.b"}, "manifest of mod-y"}, - {"a want's built manifest is composed, not the trunk's", func(b *fakeBus) { - built := strings.Replace(xManifest, `"history":3}]`, `"history":3},{"name":"newer"}]`, 1) + {"a want's built manifest is composed, not the trunk's; the list's records carry both (issue 774)", func(b *fakeBus) { + built := strings.Replace(xManifest, `{"name":"kept","history":3}`, `{"name":"newer"}`, 1) digest := delivery.ManifestDigest([]byte(built)) b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "mod-x", JSON: built}) fp := "src2:" + strings.Repeat("cd", 32) @@ -446,8 +516,22 @@ func TestGrants(t *testing.T) { } if strings.HasPrefix(row.name, "a want's") { x := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.mod-x" })] - if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") { - t.Errorf("a.mod-x is not granted the state its wanted build declares: %v", x.Publish) + if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") || slices.Contains(x.Publish, "$KV.mod-x_kept.>") { + t.Errorf("a.mod-x is not granted the state its wanted build declares alone: %v", x.Publish) + } + // The list composes what runs and what the trunk names: a subject only the trunk's manifest names is in + // it before the want moves to the trunk's build (issue 774). + listRecs, _ := grants.ListRecords(in) + listed, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + if got := names(listed); !slices.Equal(got, row.users) { + t.Errorf("the list's users = %v, want %v", got, row.users) + } + lx := listed[slices.IndexFunc(listed, func(u grants.User) bool { return u.User == "a.mod-x" })] + if !slices.Contains(lx.Publish, "$KV.mod-x_newer.>") || !slices.Contains(lx.Publish, "$KV.mod-x_kept.>") { + t.Errorf("the list does not grant a.mod-x both its running and its trunk manifest's state: %v", lx.Publish) } } }) @@ -500,6 +584,34 @@ func TestGrants(t *testing.T) { } }) + t.Run("withheld: a hearing of a restricted stream in the tool runner is written on change only, and deleted once heard (issue 1955 L7)", func(t *testing.T) { + const mail = `{"module":"mesh-mail","version":"1","streams":[{"name":"mail","events":["mail.*.message.added"],"max-age-days":7,` + + `"grant":{"config":"grants","events":"mail.{key}.**"}}],"resources":[{"id":"state","type":"directory","mode":"0700","place":"."},` + + `{"id":"grants","trusted":true,"type":"file","path":"${dir:state}/grants.json","mode":"0600","merge":"json","takes-settings":["grants"],"content":"{\"grants\": {}}"}]}` + b := fixture() + b.manifestOf("mesh-mail", mail) + b.manifestOf("reader", `{"module":"reader","version":"1","consumes":["mesh-mail.mail.box.message.added"]}`) + b.assign("a", "mesh-mail") + b.assign("a", "reader") + b.put(grants.ReadResolved, "a.mesh-mail.grants", map[string]any{"node": "a", "module": "mesh-mail", "key": "grants", "value": map[string]any{"box": []string{"reader"}}}) + p := partOn(b) + res := p.Compose() + const key = "reader.mail.a" + var h grants.WithheldHearing + if res.Withheld == nil || !slices.Equal(res.Withheld.Written, []string{key}) || json.Unmarshal(b.own[grants.StateWithheld][key], &h) != nil || + h.Module != "reader" || h.Node != "a" || h.Stream != "MAIL" || h.Emitter != "mesh-mail" || !strings.Contains(h.Said, "runs-as") { + t.Fatalf("withheld = %+v, stored %+v (errors %v), want %s written naming the remedy", res.Withheld, h, res.Errors, key) + } + if again := p.Compose(); again.Withheld == nil || len(again.Withheld.Written) != 0 || again.Withheld.Withheld != 1 { + t.Errorf("a second pass wrote %+v, want nothing (on change only)", again.Withheld) + } + b.manifestOf("reader", `{"module":"reader","version":"1","runs-as":"reader","resources":[{"id":"account","type":"user","name":"reader","shell":"/usr/bin/nologin","home":"/var/lib/reader"}],`+ + `"own-secrets":{"broker":"/var/lib/r/broker"},"secrets-owner":"reader","consumes":["mesh-mail.mail.box.message.added"]}`) + if heard := p.Compose(); heard.Withheld == nil || !slices.Equal(heard.Withheld.Deleted, []string{key}) || b.own[grants.StateWithheld][key] != nil { + t.Errorf("once reader runs as its own account the pass did %+v (errors %v), want %s deleted", heard.Withheld, heard.Errors, key) + } + }) + t.Run("root-free: the controller's word fills the composition, failing closed (issue 1945)", func(t *testing.T) { const router = `{"module":"messenger","version":"1","runs-as":"messenger","resources":[{"id":"account","type":"user","name":"messenger","shell":"/usr/bin/nologin","home":"/var/lib/messenger"}],"own-secrets":{"broker":"/var/lib/m/broker"},"secrets-owner":"messenger",` + `"seats":[{"name":"operator-channel","scope":"mesh","serves":["open"]},{"name":"channel","scope":"mesh","kinded":true,"accepts":["show"]}],` + @@ -578,8 +690,40 @@ func TestGrants(t *testing.T) { if passes() != before { t.Errorf("a condition of another kind woke a pass") } + // 1952/G1 (ADR 0564 §9): the keeper's events are not heard while the take-over key follows the controller + handle("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + time.Sleep(100 * time.Millisecond) + if passes() != before || wf.Follows() != "mesh-controller" { + t.Errorf("the keeper's event woke a pass while the part follows %s", wf.Follows()) + } handle(grants.EventConditionCleared, []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) waitFor(t, func() bool { return passes() > before }) + // and from the take-over only the keeper's are heard, never both + watcher := wf.Watch.(*fakeWatcher) + waitFor(t, func() bool { + watcher.mu.Lock() + defer watcher.mu.Unlock() + return watcher.changes[grants.ReadTakeover] != nil + }) + watcher.mu.Lock() + toKeeper := watcher.changes[grants.ReadTakeover] + watcher.mu.Unlock() + toKeeper(health.TakeoverKey, []byte(`{"v":1,"follow":"keeper","by":"operator","at":"2026-10-29T20:00:00Z"}`)) + if wf.Follows() != health.KeeperModule { + t.Fatalf("the part follows %s after the take-over", wf.Follows()) + } + before = passes() + handle("mesh.seat.mesh-controller.event.condition-raised", []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + time.Sleep(100 * time.Millisecond) + if passes() != before { + t.Errorf("the controller's event woke a pass after the take-over") + } + handle(grants.EventKeeperRaised, []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`)) + waitFor(t, func() bool { return passes() > before }) + toKeeper(health.TakeoverKey, nil) // handed back: the key deleted follows the controller + if wf.Follows() != "mesh-controller" { + t.Errorf("the part follows %s after the hand-back", wf.Follows()) + } wcancel() <-wdone if !grants.IsRootNotFree([]byte(`{"kind":"root-not-free"}`)) || !grants.IsRootNotFree([]byte(`{"key":"machine.novox.root-not-free"}`)) || @@ -966,7 +1110,10 @@ func TestGrants(t *testing.T) { file("node-tools-novox-licence-subjects.json", &live) licence, _ := os.ReadFile(filepath.Join("testdata", "claude-licence-manager.json")) n := newFakeBus() - n.manifestOf("nats", natsManifest) + // 1952/S5 (ADR 0564 Limits 4): the bus module's tools bundle declares its own account and what it may remove. + n.manifestOf("nats", strings.Replace(strings.TrimSuffix(natsManifest, "}"), `"broker":"/var/lib/nats/broker"`, `"broker":"/var/lib/nats/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"`, 1)+ + `,"resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711"}],"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go",`+ + `"system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts"]}}]}}`) n.manifestOf("claude-licence-manager", string(licence)) n.assign("novox", "nats") n.assign("novox", "claude-licence-manager") @@ -976,6 +1123,15 @@ func TestGrants(t *testing.T) { pl, _ = delivery.EncodePlacement(seed) n.states[grants.ReadPlacement] = map[string][]byte{"placement.novox": pl} recs, _ := grants.Records(grants.ReadInputs(n, &grants.Manifests{})) + var tools []buscompose.ProcessPrincipal + for _, d := range recs.Assigned["novox"] { + if d.Module == "nats" { + tools = d.Processes + } + } + if len(tools) != 1 || tools[0].Process != "nats-tools" || !slices.Equal(tools[0].Removes, []string{"DEAD_LETTER_NOTICES.controller", "mesh-host_acts"}) { + t.Errorf("the bus module's principals = %+v, want nats-tools carrying both removals", tools) + } if m := grants.Memberships(recs)["novox.claude-licence-manager"]; len(m.Membership.Seats) == 0 { t.Errorf("the licence manager's membership under the seeded placement carries no seats: %+v", m.Membership) } @@ -1144,7 +1300,7 @@ func TestGrants(t *testing.T) { t.Errorf("membership = %v", ms) } gr, _ := p.Grants("mod-x") - if g, _ := gr.(map[string]any); g["writes"] != grants.WritesNoGrant || g["count"] != 1 { + if g, _ := gr.(map[string]any); g["writes"] != grants.WritesGrants || g["controller_count"] != 1 { t.Errorf("grants = %v", gr) } @@ -1260,9 +1416,14 @@ func TestGrants(t *testing.T) { row.between() } said = nil + calls := len(b.calls) if err := p.Warranted(row.option, warrant(id)); err != nil { t.Fatal(err) } + // 1953/D8 (ADR 0565 §11): a switch made is recorded in the hand-act log's seat; one refused or declined is not. + if recorded := slices.Contains(b.calls[calls:], grants.CallRecordWarrant); recorded != (row.outcome == "recorded") { + t.Errorf("recorded in the hand-act log %v, the act %s", recorded, row.outcome) + } l := last() if l.Said != row.outcome || l.Ask != id || p.SwitchState() != row.state { t.Errorf("the act = %+v, the switch %s; want %s and %s", l, p.SwitchState(), row.outcome, row.state) @@ -1371,7 +1532,12 @@ func TestGrants(t *testing.T) { var fresh []asks.Condition fp := partOn(b) fp.Objects = p.Objects - fp.Asker, fp.Say = fa, func(c asks.Condition) error { fresh = append(fresh, c); return nil } + fp.Asker, fp.Say = fa, func(c asks.Condition) error { + if c.Key != grants.GrantMissingCondition { // its own row below + fresh = append(fresh, c) + } + return nil + } fp.Compose() if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != asks.StateCleared { t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh) @@ -1504,13 +1670,13 @@ func TestGrants(t *testing.T) { } } for _, w := range b.writes { - if !slices.Contains(grants.States, w) { + if !slices.Contains(grants.States, strings.TrimSuffix(w, " delete")) { t.Errorf("wrote %s, which is not the part's own state", w) } } for _, c := range b.calls { - if c != grants.CallUsers && c != grants.CallStreams { - t.Errorf("called %s: the part calls only the bus module's read tools", c) + if c != grants.CallUsers && c != grants.CallStreams && c != grants.CallRecordWarrant { + t.Errorf("called %s: the part calls only the bus module's read tools, and the hand-act log for a switch", c) } } for _, e := range b.events { @@ -1669,6 +1835,9 @@ func TestGrants(t *testing.T) { !strings.HasPrefix(first.Asserted[0], "mod-x at ") || !slices.Contains(b.events, grants.EventAsserted) { t.Fatalf("the first pass created %v, said %v, events %v", b.creates, first.Asserted, b.events) } + if b.createdAge["LOG_mod-x_history"] != 30*24*time.Hour { // a log's max-age-days is its stream's MaxAge (mesh-sdk 2e5cc4f) + t.Errorf("the log was created with max age %v, want 30 days", b.createdAge["LOG_mod-x_history"]) + } a := assertion() if !a.Asserted || a.Says != grants.CreatesMissing { t.Errorf("the assertion kept = %+v", a) @@ -1714,6 +1883,31 @@ func TestGrants(t *testing.T) { } }) + // 774 (ADR 0563 §2, §3): the node-engine's manifest (mesh-host) is placed on every node though no node is assigned it, + // so its declared state acts is created as a placed module's, and composed for the user list's create right. + t.Run("assertions: the node-engine's declared state is created though no node is assigned it", func(t *testing.T) { + b := fixture() + b.manifestOf(buscompose.EngineModule, `{"module":"mesh-host","version":"1","state":[{"name":"acts"}]}`) + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Create = b + p.Compose() + if !slices.Contains(b.creates, "KV_"+buscompose.EngineActsBucket) { + t.Fatalf("the node-engine's acts were not created: %v", b.creates) + } + recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{})) + if recs.Engine == nil || len(recs.Engine.State) != 1 || recs.Engine.State[0].Bucket() != buscompose.EngineActsBucket { + t.Fatalf("the records carry the node-engine as %+v", recs.Engine) + } + for node, mods := range recs.Assigned { + for _, d := range mods { + if d.Module == buscompose.EngineModule { + t.Errorf("the node-engine is a module of %s, which would give it a module's account", node) + } + } + } + }) + t.Run("assertions: a refused create is said, never stops it, and is tried at the next event", func(t *testing.T) { b := fixture() b.loaded = shadowOf(t, b) @@ -1823,6 +2017,532 @@ func TestGrants(t *testing.T) { t.Errorf("the refusal was not said: %v", logged) } }) + + // 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.. in shadow, once each and only for a + // module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no + // account with a hash and every read was made, and the tool says every difference from the controller's. + t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's; after take-over once the loaded list carries them", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + fp := "src2:" + strings.Repeat("ab", 32) + key := func(module string) string { k, _ := delivery.PartGrantKey(module, fp); return k } + hash := func(user string) { + b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1}) + } + held := func() []string { return slices.Sorted(maps.Keys(b.own[grants.StateGrants])) } + // The controller's grant for this fingerprint, beside the stale one the fixture holds for another. + delete(b.states[grants.ReadControllerGrant], "mod-x.0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123") + b.put(grants.ReadControllerGrant, key("mod-x"), delivery.ControllerGrant{V: delivery.Version, Accounts: []string{"a.mod-x"}}) + hash("a.mod-x") + hash("a.nats") + res := p.Compose() + if res.Grants == nil || !slices.Equal(held(), []string{key("mod-x"), key("nats")}) || len(res.Grants.Written) != 2 { + t.Fatalf("grants of the hashed modules: held %v, %+v", held(), res.Grants) + } + if g, err := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key("mod-x")]); err != nil || !slices.Equal(g.Accounts, []string{"a.mod-x"}) { + t.Errorf("the grant of mod-x is %+v, %v", g, err) + } + said := func() int { + n := 0 + for _, e := range b.events { + if e == grants.EventGrantWritten { + n++ + } + } + return n + } + if said() != 2 { + t.Errorf("grant-written said %d times for two grants: %v", said(), b.events) + } + if res := p.Compose(); len(res.Grants.Written) != 0 || said() != 2 { + t.Errorf("a second pass wrote %v and said %d", res.Grants.Written, said()) + } + // The tool: equal on mod-x, and nats is the part's alone (the controller's fixture holds none). + got, err := p.Grants("") + gm, _ := got.(map[string]any) + diff, _ := gm["differences"].([]grants.GrantDifference) + if err != nil || gm["equal"] != false || len(diff) != 1 || diff[0].Key != key("nats") || diff[0].Kind != "only the part's" { + t.Errorf("grants = %+v, %v", got, err) + } + // A user with a hash later: written once. Without it again: deleted. Unread hashes: nothing deleted. + hash("b.mod-y") + if res := p.Compose(); !slices.Equal(res.Grants.Written, []string{key("mod-y")}) { + t.Errorf("mod-y was granted %v when its account got a hash", res.Grants.Written) + } + b.refused[grants.ReadAccounts] = true + if res := p.Compose(); res.Grants == nil || len(res.Grants.Deleted)+len(res.Grants.Written) != 0 || len(res.Grants.Kept) != 1 || len(held()) != 3 { + t.Errorf("a pass that could not read the hashes changed grants: %+v, held %v", res.Grants, held()) + } + delete(b.refused, grants.ReadAccounts) + delete(b.states[grants.ReadAccounts], "b.mod-y") + if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 { + t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held()) + } + + // Issue 774: after take-over a grant is written at once when the list the bus loaded carries its accounts' permissions, + // held list or not, and waits only when it adds one. (1) mod-x moves to a new fingerprint with the same permissions + // while the list is held (users of the loaded list have no hash): written. + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + loads := func(source, digest string) { + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey, + Follows: delivery.FollowGrants}}) + } + loads("a.nats.src2", "sha256:aa") + p.Objects = &fakeObjects{} + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + moves := func(module, manifest, fp string) string { + digest := delivery.ManifestDigest([]byte(manifest)) + b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: module, JSON: manifest}) + b.put(grants.ReadModules, module, delivery.Module{V: delivery.Version, FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + k, _ := delivery.PartGrantKey(module, fp) + return k + } + written := func(k string) bool { _, ok := b.own[grants.StateGrants][k]; return ok } + keyX2 := moves("mod-x", xManifest, "src2:"+strings.Repeat("cd", 32)) + res = p.Compose() + if !written(keyX2) || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") || len(res.Grants.Waits) != 0 { + t.Fatalf("a new fingerprint with the same permissions under a held list: sealed %+v, grants %+v", res.Sealed, res.Grants) + } + // Every user hashed: the list is sealed, and the part keeps its permissions. The bus loads it, and its tools go quiet. + for _, u := range shadowOf(t, b) { + hash(u.User) + } + p.Compose() + sealed, _, _ := p.Sealed() + if sealed.Digest == "" { + t.Fatal("no list sealed once every user has a hash") + } + loads(delivery.GrantsUserListSource, sealed.Digest) + b.refuseUsers = true + // (2) mod-y's trunk adds a state, so its account adds permissions: it waits, judged from the part's own record of the + // loaded list; mod-x at another fingerprint with the same permissions is written. + fp3 := "src2:" + strings.Repeat("ef", 32) + keyY3 := moves("mod-y", strings.Replace(yManifest, `"reads"`, `"state":[{"name":"fresh"}],"reads"`, 1), fp3) + keyX3 := moves("mod-x", xManifest, fp3) + res = p.Compose() + if !written(keyX3) || written(keyY3) || len(res.Grants.Waits) != 1 || + !strings.HasPrefix(res.Grants.Waits[0], keyY3+": waits for the bus to load a list carrying what it adds: b.mod-y") { + t.Fatalf("an added permission against the loaded list: %+v", res.Grants) + } + // A loaded list the part keeps no permissions of: every grant naming an account waits, said. + loads(delivery.GrantsUserListSource, "sha256:bb") + keyX4 := moves("mod-x", xManifest, "src2:"+strings.Repeat("12", 32)) + if res := p.Compose(); written(keyX4) || len(res.Grants.Waits) != 2 || !strings.Contains(res.Grants.Waits[0], "unknown") { + t.Fatalf("an unknown loaded list let a grant through: %+v", res.Grants) + } + // The bus loads a list carrying mod-y's state (its tools answer it): both written, nothing waits. + listRecs, _ := grants.ListRecords(grants.ReadInputs(b, &grants.Manifests{})) + now, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + b.refuseUsers, b.loaded = false, now + if res := p.Compose(); len(res.Grants.Waits) != 0 || !written(keyY3) || !written(keyX4) { + t.Errorf("once the loaded list carries them: %+v", res.Grants) + } + }) + + // Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the + // new form at that commit (link2pay, snake, f1-circuits, txt-game), is granted at the indexer's fingerprint, never + // judged by its commit; and a placed module left without a grant is said, never passed over: in the pass, and as the + // condition mesh-grants.grant-missing once the part's grants are the ones read, cleared when it has one. + t.Run("grants: at the indexer's fingerprint whatever the commit, and a placed module without one is said", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + var said []asks.Condition + p.Say = func(c asks.Condition) error { + if c.Key == grants.GrantMissingCondition { + said = append(said, c) + } + return nil + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c" + b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)), + Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"}) + b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, None2: "its builder's version is not read yet", + Manifest: delivery.ManifestDigest([]byte(yManifest)), Repository: "novox/y", Commit: "c"}) + for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} { + b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1}) + } + res := p.Compose() + key, _ := delivery.PartGrantKey("mod-x", fp) + if _, held := b.own[grants.StateGrants][key]; !held { + t.Errorf("mod-x has no grant at the indexer's fingerprint %s: %+v", fp, res.Grants) + } + if res.Grants == nil || len(res.Grants.Missing) != 1 || !strings.HasPrefix(res.Grants.Missing[0], + "mod-y: no fingerprint of the new form: its builder's version is not read yet") { + t.Fatalf("missing = %+v", res.Grants) + } + if len(said) != 1 || said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "mod-y") { + t.Fatalf("said %+v", said) + } + if p.Compose(); len(said) != 1 { + t.Errorf("the same missing grant was said again: %+v", said) + } + b.put(grants.ReadModules, "mod-y", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(yManifest)), + Repository: "novox/y", Commit: "c"}) + if res := p.Compose(); len(res.Grants.Missing) != 0 || len(said) != 2 || said[1].State != asks.StateCleared { + t.Errorf("once mod-y has a grant: missing %v, said %+v", res.Grants.Missing, said) + } + }) + + // The guard of ADR 0549 §6 for this repository, as mesh-catalog's checks/placement is for the catalogue: no module.json + // here reads the controller's assignments unless ../assignments-readers names it with why, and that list only shrinks. + t.Run("assignments readers: only the named modules read the controller's assignments, and the list only shrinks", func(t *testing.T) { + raw, err := os.ReadFile("../assignments-readers") + if err != nil { + t.Fatal(err) + } + allowed := map[string]bool{} + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 1 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]] = true + } else if len(f) == 1 { + t.Errorf("../assignments-readers names %s with no why", f[0]) + } + } + paths, _ := filepath.Glob("../*/module.json") + reads := map[string]bool{} + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Reads []string `json:"reads"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", path, err) + } + if slices.Contains(m.Reads, grants.ReadAssignments) { + reads[m.Module] = true + if !allowed[m.Module] { + t.Errorf("%s reads %s: read where modules run from the seat mesh-placement's placement (ADR 0549 §3); the lease write "+ + "is refused while a module reads the assignments (ADR 0549 §6)", m.Module, grants.ReadAssignments) + } + } + } + for m := range allowed { + if !reads[m] { + t.Errorf("../assignments-readers names %s, which no longer reads %s: take it off the list", m, grants.ReadAssignments) + } + } + }) + + // 774 (the L3 review, ADR 0549 §3, §6): once the controller's lease is handed away the assignments are frozen and place + // nothing; the seat's placement alone does. A module left only in the stale assignments loses its account and its + // grant names no account of it; before the lease write the two are still unioned. + t.Run("placement alone once the lease is handed away: a module only in the stale assignments has no account or grant of it", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"} + pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-x"}}}) + b.states[grants.ReadPlacement] = map[string][]byte{"placement.a": pl} + for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} { + b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1}) + } + fp := "src2:" + strings.Repeat("ab", 32) + key, _ := delivery.PartGrantKey("mod-y", fp) + grantOf := func() []string { + g, _ := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key]) + return g.Accounts + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-28", "epoch": 60}) + p.Compose() + if b.own[grants.StateAccounts]["b.mod-y"] == nil || !slices.Equal(grantOf(), []string{"b.mod-y"}) { + t.Fatalf("under the controller's lease the assignments no longer count: account %v, grant %v", + b.own[grants.StateAccounts]["b.mod-y"] != nil, grantOf()) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + in := grants.ReadInputs(b, &grants.Manifests{}) + if len(in.Assigned["b"]) != 0 || !slices.Equal(in.Assigned["a"], []string{"mod-x", "nats"}) || in.BusNode != "a" { + t.Errorf("after the lease write the stale assignments still place: %v, bus %q", in.Assigned, in.BusNode) + } + res := p.Compose() + if res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) || slices.Contains(grantOf(), "b.mod-y") { + t.Fatalf("a module only in the stale assignments kept its account or grant: %+v, grant %v", res.Accounts, grantOf()) + } + }) + + // 774/AC4 (novox/hq ADR 0561 §2): every account the vault mints that the records compose is written to the state + // accounts once, with the bus's address and a module's claims, read by the vault alone; a key is deleted when placement + // no longer names its user, and never in a pass whose reads failed. + t.Run("wanted accounts: written once each, read by the vault alone, retired only when placement drops them", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + p := partOn(b) + p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"} + p.Compose() + p.Compose() + wrote := 0 + for _, w := range b.writes { + if w == grants.StateAccounts { + wrote++ + } + } + held := slices.Sorted(maps.Keys(b.own[grants.StateAccounts])) + if !slices.Equal(held, []string{"a.mod-x", "a.nats", "b.mod-y"}) || wrote != 3 { + t.Fatalf("accounts %v written %d times in two passes, want the three modules' once each", held, wrote) + } + nats, err := buscompose.ReadWantedAccount("a.nats", b.own[grants.StateAccounts]["a.nats"]) + if err != nil || nats.URL != "nats://a.mesh:4222" || nats.Fingerprint != "ab" || len(nats.Claims) != 1 || + nats.Claims[0].Seat != catalogue.BrokerSeat || nats.At.IsZero() { + t.Errorf("a.nats is wanted as %+v (%v)", nats, err) + } + var readers []string + for _, s := range grants.Announced() { + if s.Name == grants.StateAccounts { + if s.Readable { + t.Error("the wanted accounts are announced readable by anyone") + } + readers = s.Readers + } + } + if !slices.Equal(readers, []string{"mesh-vault"}) || buscompose.ReaderMay("mesh-delivery-view", buscompose.GrantsAccounts) { + t.Errorf("the wanted accounts are read by %v", readers) + } + // A read that fails retires nothing; once read, the user placement dropped is retired. + delete(b.states[grants.ReadAssignments], "b.mod-y") + b.refused[grants.ReadResolved] = true + if res := p.Compose(); res.Accounts == nil || len(res.Accounts.Deleted) != 0 || b.own[grants.StateAccounts]["b.mod-y"] == nil { + t.Fatalf("a pass whose read failed deleted %+v", res.Accounts) + } + delete(b.refused, grants.ReadResolved) + if res := p.Compose(); res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) { + t.Fatalf("the user placement dropped is not retired: %+v", res.Accounts) + } + // The tool reads them, one and all, each with when it was written. + one, err := p.Accounts("a.nats") + all, err2 := p.Accounts("") + if err != nil || err2 != nil || one.(map[string]any)["count"] != 1 || all.(map[string]any)["count"] != 2 || + one.(map[string]any)["accounts"].([]buscompose.WantedAccount)[0].At.IsZero() { + t.Errorf("the tool reads %v and %v (%v %v)", one, all, err, err2) + } + if _, err := p.Accounts("b.mod-y"); err == nil { + t.Error("the tool reads an account no longer wanted") + } + p.Pass() + if all, _ := p.Accounts(""); all.(map[string]any)["consumers"] == nil { + t.Errorf("the tool does not say the account consumers in shadow: %v", all) + } + // With no bus address no account is written, and the pass says why. + bare := partOn(fixture()) + if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") { + t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors) + } + }) + + // 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash + // within one pass; an event of another kind does not. + t.Run("the vault's account-issued composes the list again", func(t *testing.T) { + b := fixture() + var handle func(string, []byte) + p := partOn(b) + f := &grants.Follower{Part: p, Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {}, + Hear: func(h func(string, []byte)) error { handle = h; return nil }} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { f.Run(ctx); close(done) }() + defer func() { cancel(); <-done }() + waitFor(t, func() bool { return b.count(grants.CallUsers) > 0 && handle != nil }) + held := func() bool { + for _, c := range p.Last().UserList.Credentials.Users { + if c.User == "a.mod-x" { + return c.HashHeld + } + } + return false + } + if held() { + t.Fatal("a.mod-x has a hash before the vault made one") + } + b.mu.Lock() + b.states[grants.ReadAccounts] = map[string][]byte{"a.mod-x": []byte(`{"user":"a.mod-x","hash":"$2a$10$x","generation":1}`)} + b.mu.Unlock() + handle(buscompose.AccountIssuedSubject, []byte(`{"user":"a.mod-x","kind":"module","node":"a","module":"mod-x","generation":1}`)) + waitFor(t, held) + if !slices.Contains(grants.Consumes, grants.EventAccountIssued) || !grants.IsAccountIssued(grants.EventAccountIssued) || + grants.IsAccountIssued("mesh.mod.mesh-vault.event.secret-changed") { + t.Errorf("the part hears %v", grants.Consumes) + } + }) + + // 774/AC6 (novox/hq ADR 0561 §6): the durable consumers an account needs, with the controller's configuration, are + // compared in shadow while the controller holds the lease, and created, only where the bus lacks them, from the write + // that hands the lease away; one the bus holds is never changed. + t.Run("account consumers: compared while the controller holds the lease, created once it is handed away", func(t *testing.T) { + const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done"]}` + b := fixture() + b.manifestOf("mod-c", cManifest) + b.assign("b", "mod-c") + b.loaded = shadowOf(t, b) + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57}) + p := partOn(b) + p.Consumers = b + res := p.Compose() + if res.Consumers == nil || res.Consumers.On || !slices.Contains(res.Consumers.Missing, "EVENTS.b_mod-c") || len(b.consumerCreates) != 0 { + t.Fatalf("while the controller holds the lease: %+v, created %v", res.Consumers, b.consumerCreates) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}} + res = p.Compose() + if !res.Consumers.On || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) { + t.Fatalf("once handed away: %+v", res.Consumers) + } + var made map[string]any + for _, c := range b.consumers["EVENTS"] { + if c["name"] == "b_mod-c" { + made = c + } + } + if made == nil || made["max_deliver"] != 5 || !slices.Contains(made["filters"].([]string), "mesh.mod.mod-y.event.done") { + t.Errorf("b_mod-c was made as %v, not as the controller makes it", made) + } + res = p.Compose() + if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 { + t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates) + } + // A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to + // the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease. + var said []asks.Condition + p.Say = func(c asks.Condition) error { + if c.Key != grants.GrantMissingCondition { // its own row + said = append(said, c) + } + return nil + } + made["max_deliver"] = 9 + res = p.Compose() + if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") || + len(said) != 1 || said[0].Key != grants.ConsumerDiffersCondition || said[0].State != asks.StateRaised || + !strings.Contains(said[0].Explanation, "EVENTS.b_mod-c") { + t.Fatalf("a differing consumer: %+v, said %+v", res.Consumers, said) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/2/2026-10-27", "epoch": 58}) + if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") { + t.Errorf("matched by hand under the controller's lease: %v", err) + } + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + if _, err := p.MatchConsumer("EVENTS", "b_nothing"); err == nil { + t.Error("a consumer the records do not compose was asked to match") + } + if _, err := p.MatchConsumer("EVENTS", "b_mod-c"); err != nil { + t.Fatal(err) + } + res = p.Compose() + if !slices.Contains(b.consumerCreates, "match EVENTS.b_mod-c") || made["max_deliver"] != 5 || len(res.Consumers.Matched) != 1 { + t.Errorf("the ask to match: %+v, %v", res.Consumers, b.consumerCreates) + } + res = p.Compose() + if len(res.Consumers.Differ) != 0 || said[len(said)-1].State != asks.StateCleared { + t.Errorf("once matched: %+v, said %+v", res.Consumers, said) + } + }) + + // 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no + // rotation by the vault naming each, is held: the list in force stays, user-list-held is raised naming those users. One + // user's change, or changes the vault's rotation names, are written. + t.Run("a list changing several loaded users' hashes is held unless the vault's rotation names each", func(t *testing.T) { + b := fixture() + b.loaded = shadowOf(t, b) + hash := func(user, h string, generation int, rotating bool) { + v := map[string]any{"user": user, "hash": h, "generation": generation, "fingerprint": fmt.Sprintf("sha256:%d", generation)} + if rotating { + v["rotating"] = map[string]any{"login": user + "--b", "generation": generation, "revision": 7} + } + b.put(grants.ReadAccounts, user, v) + } + for _, u := range shadowOf(t, b) { + hash(u.User, "$2a$10$first", 1, false) + } + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}}) + p := partOn(b) + p.Objects = &fakeObjects{} + var said []asks.Condition + p.Say = func(c asks.Condition) error { said = append(said, c); return nil } + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + p.Compose() + inForce, _, _ := p.Sealed() + if inForce.Digest == "" { + t.Fatal("no list written after take-over") + } + // One user's hash changes: written. + hash("a.mod-x", "$2a$10$second", 2, false) + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest { + t.Fatal("a list changing one loaded user's hash was not written") + } + inForce, _, _ = p.Sealed() + // Two change at once, no rotation names them: held, said naming both. + hash("a.mod-x", "$2a$10$third", 3, false) + hash("a.nats", "$2a$10$third", 2, false) + said = nil + res := p.Compose() + if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "a.mod-x, a.nats") { + t.Fatalf("a list changing two loaded users' hashes: %+v", res.Sealed) + } + if len(said) != 1 || said[0].Key != grants.HeldCondition || said[0].State != asks.StateRaised || !strings.Contains(said[0].Summary, "a.mod-x, a.nats") { + t.Errorf("the hold was said as %+v", said) + } + // The hold is its own guard, beside take-over's refusals (ADR 0562): the switch stays taken over. + if p.SwitchState() != grants.TookOver { + t.Errorf("the hold moved the switch to %s", p.SwitchState()) + } + // A person accepts exactly the named users' changes of the held list, with why: refused for another set, with no + // why, or once the list changed since; accepted, the next pass writes it once, said as a hand act. + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x"}, "rotated by hand"); err == nil || !strings.Contains(err.Error(), "a.mod-x, a.nats") { + t.Errorf("an acceptance of another set than the held one: %v", err) + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, ""); err == nil { + t.Error("an acceptance with no why") + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.nats", "a.mod-x"}, "the vault's store was restored"); err != nil { + t.Fatal(err) + } + // The list changed since the acceptance (a.mod-x's hash again): the acceptance is bound to the held list, refused. + hash("a.mod-x", "$2a$10$fourth", 4, false) + res = p.Compose() + if now, _, _ := p.Sealed(); now.Digest != inForce.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") { + t.Fatalf("an acceptance of a list that changed since released it: %+v", res.Sealed) + } + if _, err := p.AcceptHashes("person.jochen", []string{"a.mod-x", "a.nats"}, "the vault's store was restored"); err != nil { + t.Fatal(err) + } + b.events = nil + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared || + !slices.Contains(b.events, grants.EventHashesAccepted) { + t.Fatalf("an accepted hold was not released once and said: %+v, events %v", said, b.events) + } + inForce, _, _ = p.Sealed() + // The vault's rotation names each: written, the warning cleared. + hash("a.mod-x", "$2a$10$fifth", 5, true) + hash("a.nats", "$2a$10$fifth", 3, true) + p.Compose() + if now, _, _ := p.Sealed(); now.Digest == inForce.Digest || said[len(said)-1].State != asks.StateCleared { + t.Errorf("changes the vault's rotation names were held: %+v", said) + } + }) } type fakeAsker struct { diff --git a/modules/mesh-grants/internal/grants/accounts.go b/modules/mesh-grants/internal/grants/accounts.go new file mode 100644 index 0000000..bf806a8 --- /dev/null +++ b/modules/mesh-grants/internal/grants/accounts.go @@ -0,0 +1,217 @@ +package grants + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// **The accounts the mesh wants, said to the vault** (novox/hq ADR 0561 §2, issue 774). The bus composer derives every +// user from placement, the manifests and the wants; for each account the vault mints (a module's, a process's, a +// machine's provisioner client's), the part writes one key of its state `accounts`, keyed by the bus user, with the +// fields the controller's credential carries beside the password. A key is written only when it differs, and deleted +// only when placement no longer names its user and every input of the pass was read: a refused read never retires an +// account. The vault alone reads the state (buscompose.NamedReaders); it mints each user it holds no value for. + +// StateAccounts is the part's state of wanted accounts. +const StateAccounts = buscompose.AccountsState + +// AccountsReaders are the only modules that may read StateAccounts. +var AccountsReaders = buscompose.NamedReaders[buscompose.GrantsAccounts] + +// Announced is what the part says it keeps: every state readable by any reader, but the wanted accounts, by the vault +// alone (ADR 0495 §2-§3, ADR 0561 §2). +func Announced() []announce.State { + var out []announce.State + for _, s := range States { + if s == StateAccounts { + out = append(out, announce.State{Name: s, Readers: slices.Clone(AccountsReaders)}) + continue + } + out = append(out, announce.State{Name: s, Readable: true}) + } + return out +} + +// BusAddress is where every node on the private network reaches the bus, and its certificate's fingerprint: the part's +// own credential says both, since the controller gave every node's account the bus holder's overlay name (its +// brokerReachableAt). +type BusAddress struct { + URL string + Fingerprint string +} + +// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address. A +// node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure. +func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) { + principals, err := buscompose.Users(recs) + if err != nil { + return nil, err + } + out := map[string]buscompose.WantedAccount{} + for _, p := range principals { + w := buscompose.WantedAccount{User: p.Username(), Kind: p.Kind, Node: p.Node, Module: p.Module, Process: p.Process, + URL: bus.URL, Fingerprint: bus.Fingerprint} + switch p.Kind { + case buscompose.KindModule, buscompose.KindNodeTools: + if m, ok := manifestOn(in, p.Node, p.Module); ok { + w.Claims = ClaimsOf(m) + } + case buscompose.KindProcess, buscompose.KindProvisioner: + default: + continue + } + out[w.User] = w + } + return out, nil +} + +// manifestOn is the manifest a node runs of module: the build its want names, else the trunk's. +func manifestOn(in Inputs, node, module string) (catalogue.Manifest, bool) { + if m, ok := in.Running[node+"."+module]; ok { + return m, true + } + m, ok := in.Manifests[module] + return m, ok +} + +// ClaimsOf is a module's claims as its credential carries them: the controller's claimsFor, over the mesh's own seats +// (its seat table is seeded from them): each claim's seat and scope, and the verbs the runtime serves for it, the claim's +// own or every verb the seat promises; a seat the mesh does not define keeps the claim's scope and no verbs. +func ClaimsOf(m catalogue.Manifest) []buscompose.ClaimedSeat { + if len(m.Claims) == 0 { + return nil + } + byName := map[string]catalogue.Seat{} + for _, s := range catalogue.DefaultSeats() { + byName[s.Name] = s + } + var out []buscompose.ClaimedSeat + for _, c := range m.Claims { + claimed := buscompose.ClaimedSeat{Seat: c.Name, Scope: c.At()} + if s, known := byName[c.Name]; known { + claimed.Scope = s.Scope + claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)}) + } + out = append(out, claimed) + } + return out +} + +// Accounts answers the tool accounts (ADR 0561 §9): the wanted accounts as the part last wrote them, one user's or all, +// each with when it was written; and what the last pass did to them. Never a password: the state holds none. +func (p *Part) Accounts(user string) (any, error) { + users := []string{user} + if user == "" { + keys, err := p.Store.Keys(StateAccounts) + if err != nil { + return nil, fmt.Errorf("its own %s cannot be listed: %w", StateAccounts, err) + } + users = keys + } + out := []buscompose.WantedAccount{} + for _, u := range users { + raw, _, found, err := p.Store.Get(StateAccounts, u) + if err != nil { + return nil, fmt.Errorf("its own %s %s cannot be read: %w", StateAccounts, u, err) + } + if !found { + if user != "" { + return nil, fmt.Errorf("no account %s is wanted: placement composes no module, process or provisioner client of that user", user) + } + continue + } + w, err := buscompose.ReadWantedAccount(u, raw) + if err != nil { + return nil, err + } + out = append(out, w) + } + answer := map[string]any{"accounts": out, "count": len(out), "read_by": AccountsReaders} + if raw, _, found, err := p.Store.Get(StateUserList, KeyPass); err == nil && found { + var res Result + if json.Unmarshal(raw, &res) == nil && res.Accounts != nil { + answer["last_pass"] = res.Accounts + } + // And the durable consumers the accounts need, as the last pass found them: missing and differing ones are read + // here in shadow too, where no condition is raised (ADR 0561 §6, §9). + if res.Consumers != nil { + answer["consumers"] = res.Consumers + } + } + return answer, nil +} + +// AccountsWritten is what one pass did to the state of wanted accounts. +type AccountsWritten struct { + Wanted int `json:"wanted"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept are keys placement no longer names that this pass did not delete, with why. + Kept []string `json:"kept,omitempty"` +} + +// writeAccounts writes each wanted account whose key differs, and deletes the key of a user no longer placed, but only +// when every input of the pass was read. +func (p *Part) writeAccounts(res *Result, in Inputs, recs buscompose.Records) { + if p.Bus.URL == "" { + res.Errors = append(res.Errors, "the wanted accounts are not written: the part knows no bus address to give them (its own credential names none)") + return + } + wanted, err := WantedAccounts(in, recs, p.Bus) + if err != nil { + res.Errors = append(res.Errors, "the wanted accounts could not be composed: "+err.Error()) + return + } + done := &AccountsWritten{Wanted: len(wanted)} + res.Accounts = done + for _, user := range sortedKeys(wanted) { + w := wanted[user] + if raw, _, found, err := p.Store.Get(StateAccounts, user); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateAccounts, user, err)) + continue + } else if found { + if was, err := buscompose.ReadWantedAccount(user, raw); err == nil && was.Same(w) { + continue + } + } + w.At = p.now() + data, _ := json.Marshal(w) + if _, err := p.Store.Put(StateAccounts, user, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateAccounts, user, err)) + continue + } + done.Written = append(done.Written, user) + } + held, err := p.Store.Keys(StateAccounts) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no key of a user gone is deleted: %v", StateAccounts, err)) + return + } + failed := in.ReadFailures() + placed := Placed(in) + for _, user := range held { + if _, still := wanted[user]; still { + continue + } + switch { + case placed(user): + done.Kept = append(done.Kept, user+": placement still names it (composed with no account this pass)") + continue + case len(failed) > 0: + done.Kept = append(done.Kept, user+": a read of this pass failed, so no account is retired") + continue + } + if err := p.Store.Delete(StateAccounts, user); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateAccounts, user, err)) + continue + } + done.Deleted = append(done.Deleted, user) + } + sort.Strings(done.Kept) +} diff --git a/modules/mesh-grants/internal/grants/consumers.go b/modules/mesh-grants/internal/grants/consumers.go new file mode 100644 index 0000000..90a5f62 --- /dev/null +++ b/modules/mesh-grants/internal/grants/consumers.go @@ -0,0 +1,242 @@ +package grants + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// **The durable consumers an account needs** (novox/hq ADR 0561 §6, issue 774). The controller creates a module's +// consumer on EVENTS and each process's on its work queue when it issues the account. From the write that hands its +// lease away the grants part creates them, with the configuration the bus composer derives (buscompose.AccountConsumers), +// only where the bus lacks one; one the bus holds is never changed or deleted, a difference is only said. Before that +// write they are compared in shadow alone, so two parts never create one consumer. + +// ReadLease is the controller's lease as the part reads it (buscompose.NamedReaders names the vault and the part), and +// LeaseKey its one key. +const ( + ReadLease = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey +) + +// ConsumerCreator creates one durable consumer the bus does not hold: the part's process alone, whose account is granted +// the create of each account's consumer by stream and name. One the bus holds answers ErrHeld. +type ConsumerCreator interface { + CreateConsumer(stream string, cfg jetstream.ConsumerConfig) error + // MatchConsumer brings a held consumer to cfg, keeping where it delivers from: only on a person's ask + // (grants_consumer_match), never by a pass. + MatchConsumer(stream string, cfg jetstream.ConsumerConfig) error +} + +// ConsumersState is what one pass found and did of the account consumers. +type ConsumersState struct { + // On says the lease is handed away, so the part creates what the bus lacks; Why says why it is on or off. + On bool `json:"on"` + Why string `json:"why"` + // Wanted is how many the records compose; Missing those the bus lacks (`.`), Differ those it holds + // otherwise, each with how. + Wanted int `json:"wanted"` + Missing []string `json:"missing,omitempty"` + Differ []string `json:"differ,omitempty"` + Created []string `json:"created,omitempty"` + // Matched is a consumer brought to the composed configuration on a person's ask, or why not. + Matched []string `json:"matched,omitempty"` + NotCreated []string `json:"not_created,omitempty"` + // Unread is a stream whose consumers could not be read: nothing on it is compared or created. + Unread []string `json:"unread,omitempty"` +} + +// heldConsumer is one consumer as the bus module's nats_streams reads it for one stream. +type heldConsumer struct { + Name string `json:"name"` + Filters []string `json:"filters"` + MaxDeliver int `json:"max_deliver"` +} + +// heldConsumers reads the consumers the bus holds on stream, by name. +func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) { + raw, err := c.Call(CallStreams, map[string]any{"stream": stream}) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", CallStreams, stream, err) + } + var got struct { + Detail []struct { + Name string `json:"name"` + Consumers []heldConsumer `json:"consumer_detail"` + } `json:"detail"` + } + if err := json.Unmarshal(unwrap(raw), &got); err != nil { + return nil, fmt.Errorf("%s %s answered no consumers: %w", CallStreams, stream, err) + } + out := map[string]heldConsumer{} + for _, d := range got.Detail { + if d.Name != stream { + continue + } + for _, c := range d.Consumers { + out[c.Name] = c + } + } + return out, nil +} + +// leaseHandedAway reads the lease as the part's reads give it. +func leaseHandedAway(r Reader) (bool, string) { + raw, found, err := r.Get(ReadLease, LeaseKey) + return buscompose.LeaseHandedAway(raw, found, err) +} + +// consumers compares the account consumers the records compose with those the bus holds, and creates the missing ones +// once the lease is handed away and the part holds a creator. +func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { + users, err := buscompose.Users(recs) + if err != nil { + res.Errors = append(res.Errors, "the account consumers could not be composed: "+err.Error()) + return + } + st := &ConsumersState{} + st.On, st.Why = leaseHandedAway(p.Read) + res.Consumers = st + wanted := buscompose.AccountConsumers(users) + st.Wanted = len(wanted) + byStream := map[string][]buscompose.Consumer{} + for _, mc := range wanted { + byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer) + } + for _, stream := range sortedKeys(byStream) { + held, err := heldConsumers(p.Call, stream) + if err != nil { + st.Unread = append(st.Unread, stream+": "+err.Error()) + continue + } + for _, c := range byStream[stream] { + key := stream + "." + c.Name + h, ok := held[c.Name] + if ok { + why := consumerDiffers(c, h) + if why != "" && key == match && st.On && p.Consumers != nil { + if err := p.Consumers.MatchConsumer(stream, buscompose.ConsumerConfig(c)); err != nil { + st.Matched = append(st.Matched, key+": not matched: "+err.Error()) + } else { + st.Matched = append(st.Matched, key+": matched, asked by hand") + continue + } + } + if why != "" { + st.Differ = append(st.Differ, key+": "+why) + } + continue + } + if !st.On || p.Consumers == nil { + st.Missing = append(st.Missing, key) + continue + } + switch err := p.Consumers.CreateConsumer(stream, buscompose.ConsumerConfig(c)); { + case err == nil: + st.Created = append(st.Created, key) + p.say(res, EventAsserted, map[string]any{"consumer": c.Name, "stream": stream}) + case errors.Is(err, ErrHeld): + default: + st.Missing = append(st.Missing, key) + st.NotCreated = append(st.NotCreated, key+": "+err.Error()) + } + } + } + p.sayDiffer(st) +} + +// sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the +// lease write) differs from its composed configuration, naming each with the fields that differ, and clears it when they +// match. Said once per set of differences, again when it changes. Before the lease write the consumers are the +// controller's, and a difference is only said in the pass. +func (p *Part) sayDiffer(st *ConsumersState) { + if p.Say == nil { + return + } + why := "" + if st.On && len(st.Differ) > 0 { + why = strings.Join(st.Differ, "; ") + } + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.differKnown && why == p.differSaid { + return + } + c := asks.Condition{Key: ConsumerDiffersCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = asks.StateCleared + } else { + c.Severity, c.State = asks.SeverityWarning, asks.StateRaised + c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ)) + c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each" + c.Explanation = asks.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " + + "and the grants part never changes a held one by itself (its delivery position would be lost): " + why + c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise" + } + if p.Say(c) == nil { + p.differSaid, p.differKnown = why, true + } +} + +// ConsumerDiffersCondition is the key of the warning raised while an account consumer differs (ADR 0561 §6). +const ConsumerDiffersCondition = "mesh-grants.consumer-differs" + +// MatchConsumer is the tool consumer-match (ADR 0561 §6, §9): it asks the part's process to bring one account consumer +// the records compose to its composed configuration, keeping where it delivers from. Refused while the controller holds +// the lease (the consumers are the controller's then), and for a consumer the records do not compose. +func (p *Part) MatchConsumer(stream, name string) (any, error) { + if stream == "" || name == "" { + return nil, errors.New("consumer-match names a stream and a consumer") + } + if on, why := leaseHandedAway(p.Read); !on { + return nil, fmt.Errorf("not asked: %s; the controller makes and matches the consumers while it holds the lease", why) + } + in := ReadInputs(p.Read, &p.cache) + recs, _ := Records(in) + users, err := buscompose.Users(recs) + if err != nil { + return nil, err + } + found := false + for _, mc := range buscompose.AccountConsumers(users) { + if mc.Consumer.Stream == stream && mc.Consumer.Name == name { + found = true + } + } + if !found { + return nil, fmt.Errorf("the records compose no account consumer %s on %s: the part brings only those to their configuration", name, stream) + } + ask, _ := json.Marshal(Ask{By: "grants_consumer_match", At: p.now(), Consumer: stream + "." + name}) + rev, err := p.Store.Put(StateUserList, KeyAsked, ask) + if err != nil { + return nil, fmt.Errorf("the ask could not be written: %w", err) + } + return map[string]any{"asked": rev, "says": "the part's process brings " + stream + "." + name + " to its composed " + + "configuration at this ask, keeping where it delivers from; the last pass's consumers.matched says what became of it"}, nil +} + +// consumerDiffers says how a held consumer differs from the composed one in what nats_streams reads back, or "". +func consumerDiffers(c buscompose.Consumer, h heldConsumer) string { + var why []string + want, have := slices.Sorted(slices.Values(c.Filters)), slices.Sorted(slices.Values(h.Filters)) + if !slices.Equal(want, have) { + why = append(why, fmt.Sprintf("filters %v, composed %v", have, want)) + } + maxDeliver := c.MaxDeliver + if maxDeliver == 0 { + maxDeliver = -1 + } + if h.MaxDeliver != maxDeliver { + why = append(why, fmt.Sprintf("max deliver %d, composed %d", h.MaxDeliver, maxDeliver)) + } + sort.Strings(why) + return strings.Join(why, "; ") +} diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 3d19217..86073f5 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -6,7 +6,9 @@ import ( "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // Watcher watches another part's state: changed hears each change after what stands, ended hears the watch end. A watch @@ -18,7 +20,7 @@ type Watcher interface { // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. -var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement} +var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect). @@ -28,9 +30,9 @@ type Follower struct { Logf func(string, ...any) // Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none. Asked func(wake func()) error - // Hear hears the controller seat's condition events (Consumes) and hands each, by event name, with its body, to - // handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: a machine's - // root-free word changed. + // Hear hears the controller seat's condition events and the vault's account-issued (Consumes) and hands each, by event + // name, with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: + // a machine's root-free word changed; and so does an account the vault minted (ADR 0561 §1), whose hash the list carries. Hear func(handle func(event string, body []byte)) error wake chan struct{} @@ -46,6 +48,21 @@ type Follower struct { held string notAsserted string refusedMans string + // leaseOn is whether the lease was handed away at its last change heard, once leaseSeen. + leaseOn, leaseSeen bool + // follows is the seat whose condition events the part hears, as the take-over key names it: the controller's until + // the operator's take-over (an absent key, or one not readable, follows the controller). + follows string +} + +// Follows is the seat whose condition events the part hears now. +func (f *Follower) Follows() string { + f.mu.Lock() + defer f.mu.Unlock() + if f.follows == "" { + return "mesh-controller" + } + return f.follows } func (f *Follower) init() { @@ -85,6 +102,34 @@ func (f *Follower) logf(format string, args ...any) { // heard says whether a change composes again: every change but a have's, and of the haves a machine's first and the // bus module's loaded list moving. func (f *Follower) heard(state, key string, value []byte) bool { + if state == ReadTakeover { + // Which seat's condition events are heard: no pass, since nothing composed reads it. + if key != health.TakeoverKey { + return false + } + t, err := health.DecodeTakeover(value, len(value) > 0) + if err != nil { + f.logf("the take-over key cannot be read (%v): the controller's condition events are heard", err) + t = health.Takeover{Follow: health.FollowController} + } + f.mu.Lock() + was := f.follows + f.follows = t.Seat() + f.mu.Unlock() + if was != t.Seat() { + f.logf("root-free words are heard from %s's condition events (the take-over key follows %s)", t.Seat(), t.Follow) + } + return false + } + if state == ReadLease { + // Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6). + on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil) + f.mu.Lock() + defer f.mu.Unlock() + was, seen := f.leaseOn, f.leaseSeen + f.leaseOn, f.leaseSeen = on, true + return !seen || was != on + } if state != ReadHave { return true } @@ -124,6 +169,14 @@ func (f *Follower) open() { } if f.Hear != nil && !f.heard_ { err := f.Hear(func(event string, body []byte) { + if IsAccountIssued(event) { + f.Wake() + return + } + // A condition event of the seat the take-over key does not name is not heard (ADR 0564 §9). + if from := HeardFrom(event); from != "" && from != f.Follows() { + return + } if IsRootNotFree(body) { f.Wake() } diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go new file mode 100644 index 0000000..92df498 --- /dev/null +++ b/modules/mesh-grants/internal/grants/grants.go @@ -0,0 +1,399 @@ +package grants + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record +// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `.` in +// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the +// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key. +// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's +// (delivery.GrantSource), and the tool grants says every difference, so the switch is made on a measured equality. +// +// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has +// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the +// assigner want it, and a grant naming no account names every node). Nothing else is ever deleted, and nothing is written +// or deleted in a pass that could not read the vault's hashes, the indexer's modules or a manifest of the module. + +// GrantsWritten is what one pass did to the state grants. +type GrantsWritten struct { + Wanted int `json:"wanted"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept says why a pass wrote and deleted nothing, or left a module out. + Kept []string `json:"kept,omitempty"` + // Missing is each module the placement or an assignment puts on a machine that has no grant after the pass, + // `: `: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once + // the part's grants are the ones read, it is the condition mesh-grants.grant-missing. + Missing []string `json:"missing,omitempty"` + // Waits is each grant composed and not written, `: `: after take-over a grant is new or changed only once the + // list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass). + Waits []string `json:"waits,omitempty"` +} + +// GrantWaits says why a grant composed now, new or changed, is not written yet, or "" when it may be (issue 774). The +// assigner moves a module's want to the fingerprint once its grant is there, so the grant is the word that the list the bus +// has loaded already carries the module's accounts as composed now (composed, from ListRecords: what runs and what the trunk +// names): every publish and subscribe allowed, the allowance of responses, and no deny the composed account lacks. A grant +// that adds no permission is written at once, whatever list is held or pending; one that adds any waits until a list that +// carries it is loaded. loaded is the loaded list by user, nil when its contents are unknown (unknown says why): then only +// a grant naming no account is written. In shadow nobody reads the part's grants and nothing waits. Pure. +func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]User, unknown string) string { + if !tookOver || len(accounts) == 0 { + return "" + } + if loaded == nil { + return "waits: the contents of the list the bus loaded are unknown (" + unknown + "), so no grant adding an account is written" + } + var adds []string + for _, user := range accounts { + c, ok := composed[user] + if !ok { + continue + } + l, ok := loaded[user] + if !ok { + adds = append(adds, user+" (not in the loaded list)") + continue + } + n := len(minus(c.Publish, l.Publish)) + len(minus(c.Subscribe, l.Subscribe)) + if c.AllowResponses && !l.AllowResponses { + n++ + } + if l.PublishDeny != nil { + n += len(minus(l.PublishDeny, c.PublishDeny)) + } + if n > 0 { + adds = append(adds, fmt.Sprintf("%s (%d permissions)", user, n)) + } + } + if len(adds) == 0 { + return "" + } + return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ") +} + +// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each +// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus +// module's tools (issue 774). +const KeySealedGrants = "sealed-grants" + +// SealedGrants is the value of KeySealedGrants. +type SealedGrants struct { + Lists []SealedList `json:"lists"` +} + +// SealedList is one sealed list's users and their permissions. +type SealedList struct { + Digest string `json:"digest"` + Users []User `json:"users"` +} + +// keepSealedGrants records the permissions of the list sealed now, beside the one before. +func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) { + in := listUsers(list) + var users []User + for _, u := range composed { + if slices.Contains(in, u.User) { + u.Hash = "" + users = append(users, u) + } + } + var was SealedGrants + if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found { + _ = json.Unmarshal(raw, &was) + } + next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}} + for _, l := range was.Lists { + if l.Digest != digest && len(next.Lists) < 2 { + next.Lists = append(next.Lists, l) + } + } + data, _ := json.Marshal(next) + if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err)) + } +} + +// loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the +// list of the digest the bus node says it loaded from the part's key; nil and why when neither says. +func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string]User, string) { + byUser := func(us []User) map[string]User { + out := map[string]User{} + for _, u := range us { + out[u.User] = u + } + return out + } + if readErr == nil { + return byUser(read), "" + } + l := in.Loaded + switch { + case l == nil: + return nil, "the bus node says no loaded list, and nats_users: " + readErr.Error() + case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "": + return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error() + } + raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants) + if err == nil && found { + var sg SealedGrants + if json.Unmarshal(raw, &sg) == nil { + for _, sl := range sg.Lists { + if sl.Digest == l.Digest { + return byUser(sl.Users), "" + } + } + } + } + return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error() +} + +// GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126). +const GrantMissingCondition = "mesh-grants.grant-missing" + +// MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `: `, +// sorted: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash +// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits +// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it, +// and that is the fingerprint granted). Pure. +func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []string, skip map[string]bool) []string { + placed := map[string]bool{} + for _, ms := range in.Assigned { + for _, m := range ms { + placed[m] = true + } + } + var out []string + for _, module := range sortedKeys(placed) { + fp, has := in.Fingerprints[module] + key, err := delivery.PartGrantKey(module, fp) + if has && err == nil { + if _, granted := want[key]; granted { + continue + } + } + why := "" + switch { + case skip[module]: + why = "its manifest at the indexer's trunk is refused" + case !has: + why = "no fingerprint of the new form: " + orSaid(in.NoFingerprint[module], "the indexer holds no value of it") + case err != nil: + why = fmt.Sprintf("no grant key for %s: %v", fp, err) + case slices.Contains(drop, key): + why = "none of its accounts at " + fp + " has a hash in the vault" + default: + why = "no grant composed at " + fp + } + out = append(out, module+": "+why) + } + return out +} + +func orSaid(s, none string) string { + if s == "" { + return none + } + return s +} + +// sayMissing raises mesh-grants.grant-missing while a placed module has no grant, once the controller's lease is handed +// away (the part's grants are then the ones the assigner reads), and clears it when every one has one; in shadow the +// pass and the tool say them. Said once per set, again when it changes, and always at the first pass after a start. +func (p *Part) sayMissing(missing []string) { + if p.Say == nil { + return + } + why := "" + if on, _ := leaseHandedAway(p.Read); on && len(missing) > 0 { + why = strings.Join(missing, "; ") + } + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.missingKnown && why == p.missingSaid { + return + } + c := asks.Condition{Key: GrantMissingCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = asks.StateCleared + } else { + c.Severity, c.State = asks.SeverityWarning, asks.StateRaised + c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing)) + c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none" + c.Explanation = asks.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " + + "credential rule gives them no want: " + why + c.Resolved = "every placed module has a grant at the fingerprint the indexer names" + } + if p.Say(c) == nil { + p.missingSaid, p.missingKnown = why, true + } +} + +// GrantsOf is the grant of every module at the fingerprint the indexer's trunk names for it, by key, and the keys that +// have none (drop): see the comment above. A module in skip (its manifest was refused) is neither. Pure. +func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.ControllerGrant, []string, error) { + principals, err := buscompose.Users(recs) + if err != nil { + return nil, nil, err + } + accounts, missing := map[string][]string{}, map[string]bool{} + for _, p := range principals { + if p.Module == "" || (p.Kind != buscompose.KindModule && p.Kind != buscompose.KindProcess) { + continue + } + if in.Hashes[p.Username()].Hash == "" { + missing[p.Module] = true + continue + } + accounts[p.Module] = append(accounts[p.Module], p.Username()) + } + out := map[string]delivery.ControllerGrant{} + var drop []string + for module, fp := range in.Fingerprints { + if skip[module] || !strings.HasPrefix(fp, delivery.SourcePrefix2) { + continue + } + key, err := delivery.PartGrantKey(module, fp) + if err != nil { + continue + } + names := accounts[module] + if missing[module] && len(names) == 0 { + drop = append(drop, key) + continue + } + sort.Strings(names) + out[key] = delivery.ControllerGrant{Accounts: names} + } + sort.Strings(drop) + return out, drop, nil +} + +// grantsReadFailed names a read of this pass without which no grant is written: the hashes and the indexer's modules and +// manifests. A manifest refused for one module only leaves that module out (writeGrants). +func grantsReadFailed(in Inputs) string { + for _, g := range in.Gaps { + if !g.Failed { + continue + } + for _, input := range []string{ReadAccounts, ReadModules, ReadManifests} { + if strings.HasPrefix(g.Input, input) { + return g.Input + ": " + g.Why + } + } + } + return "" +} + +// writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a +// module that has none now. +func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed []User, loaded map[string]User, unknown string) { + if why := grantsReadFailed(in); why != "" { + res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}} + return + } + skip := map[string]bool{} + for _, g := range in.Refused { + skip[strings.TrimPrefix(g.Input, "manifest of ")] = true + } + want, drop, err := GrantsOf(in, recs, skip) + if err != nil { + res.Errors = append(res.Errors, "the grants could not be composed: "+err.Error()) + return + } + done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)} + res.Grants = done + p.sayMissing(done.Missing) + tookOver := p.SwitchState() == TookOver + byUser := map[string]User{} + for _, u := range composed { + byUser[u.User] = u + } + for _, key := range sortedKeys(want) { + g := want[key] + data, err := delivery.EncodeControllerGrant(g) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("the grant %s cannot be encoded: %v", key, err)) + continue + } + if raw, _, found, err := p.Store.Get(StateGrants, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateGrants, key, err)) + continue + } else if found { + if was, err := delivery.DecodeControllerGrant(raw); err == nil && delivery.SameControllerGrant(was, withVersion(g)) { + continue + } + } + if waits := GrantWaits(tookOver, g.Accounts, byUser, loaded, unknown); waits != "" { + done.Waits = append(done.Waits, key+": "+waits) + continue + } + if _, err := p.Store.Put(StateGrants, key, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err)) + continue + } + done.Written = append(done.Written, key) + module, fp, _ := delivery.ParsePartGrantKey(key) + p.say(res, EventGrantWritten, map[string]any{"module": module, "fingerprint": fp, "accounts": g.Accounts}) + } + for _, key := range drop { + if _, _, found, err := p.Store.Get(StateGrants, key); err != nil || !found { + continue + } + if err := p.Store.Delete(StateGrants, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateGrants, key, err)) + continue + } + done.Deleted = append(done.Deleted, key) + } +} + +func withVersion(g delivery.ControllerGrant) delivery.ControllerGrant { + g.V = delivery.Version + return g +} + +// GrantDifference is one key on which the part's grants and the controller's differ. +type GrantDifference struct { + Key string `json:"key"` + // Kind is `only the controller's`, `only the part's` or `accounts differ`. + Kind string `json:"kind"` + Controller []string `json:"controller_accounts,omitempty"` + Part []string `json:"part_accounts,omitempty"` +} + +// CompareGrants lists every key the two sets differ on, in key order: a key held by one alone, or held by both with other +// accounts. Pure. +func CompareGrants(controller, part map[string]delivery.ControllerGrant) []GrantDifference { + keys := map[string]bool{} + for k := range controller { + keys[k] = true + } + for k := range part { + keys[k] = true + } + out := []GrantDifference{} + for _, k := range sortedKeys(keys) { + c, inC := controller[k] + g, inP := part[k] + switch { + case inC && !inP: + out = append(out, GrantDifference{Key: k, Kind: "only the controller's", Controller: c.Accounts}) + case inP && !inC: + out = append(out, GrantDifference{Key: k, Kind: "only the part's", Part: g.Accounts}) + case !delivery.SameControllerGrant(withVersion(c), withVersion(g)): + out = append(out, GrantDifference{Key: k, Kind: "accounts differ", Controller: c.Accounts, Part: g.Accounts}) + } + } + return out +} diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 68d256e..556ff70 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -38,9 +38,11 @@ const ( ReadPlacement = delivery.PlacementRead ) -// Reads is every state the part reads, in the manifest's order. +// Reads is every state the part reads, in the manifest's order. ReadAssignments stays (and in Followed) only while the +// controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write +// they place nothing, and the first change after it takes them out (../assignments-readers names the part until then). var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement} + ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -50,7 +52,7 @@ const ( ) // Invokes is every tool the part calls. -var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree} +var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant, CallRecordWarrantController} // Reader reads another part's state, named `.`. A refusal (a read not granted yet, a bucket not there) is // an error the part names as a gap and composes past: it never stops for one. @@ -105,7 +107,7 @@ type resolved struct { type Inputs struct { // Nodes is every machine seen: in a node agent's have or a standing assignment. Nodes []string - // Assigned is each machine's standing assignments and what the catalogue's placement puts there, sorted: while both + // Assigned is each machine's standing assignments and what the seat's placement puts there, sorted: while both // stand (ADR 0549 §5) a module either names keeps its membership. Assigned map[string][]string // Placed is each node's placement as the seat mesh-placement holds it (ADR 0567 §1); empty before any is read. @@ -115,6 +117,12 @@ type Inputs struct { Manifests map[string]catalogue.Manifest // Digests is each module's manifest digest, by module. Digests map[string]string + // Fingerprints is the fingerprint of the new form the indexer's trunk value names for each module, by module: what a + // grant is written for (grants.go). + Fingerprints map[string]string + // NoFingerprint is why the indexer's value of a module names no fingerprint of the new form, by module (its none2, or + // that it names none): a module here gets no grant, and that is said (grants.go MissingGrants). + NoFingerprint map[string]string // Running is the manifest a machine is wanted to run, by `.`: the one the builder built at the // fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest. Running map[string]catalogue.Manifest @@ -138,6 +146,22 @@ type HeldHash struct { Generation int `json:"generation,omitempty"` At time.Time `json:"at"` Adopted bool `json:"adopted,omitempty"` + // Fingerprint is the vault's fingerprint of the value the hash is of: with the generation, what says the password + // changed, never the hash itself (a hash of the same password differs by its salt). + Fingerprint string `json:"fingerprint,omitempty"` + // Rotating is the vault's rotation of the account under way: a change of its hash it announces (ADR 0561 §4, §7). + Rotating *struct { + Login string `json:"login"` + } `json:"rotating,omitempty"` +} + +// credentialOf is what says which password a held hash is of, never the hash: adopted, or the vault's generation and the +// fingerprint of its value. +func credentialOf(h HeldHash) string { + if h.Adopted { + return "adopted" + } + return fmt.Sprintf("generation %d %s", h.Generation, h.Fingerprint) } // readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held. @@ -221,19 +245,23 @@ func each(r Reader, state string, gaps *[]Gap) map[string][]byte { // ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap. func ReadInputs(r Reader, cache *Manifests) Inputs { - in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, - Running: map[string]catalogue.Manifest{}, - Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)} + in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, Fingerprints: map[string]string{}, + NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{}, + Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)} nodes := map[string]bool{} + // **After the lease write the assignments place nothing** (novox/hq ADR 0549 §3, §6): they are frozen then, so a + // module the catalogue takes off a node would keep its account, grant, membership and user-list entry. Once the lease + // is handed away and the placement is read, the placement alone places modules, as the assigner's Placed does with + // the same lease reading (buscompose.LeaseHandedAway); before, or with no placement on the bus, the two are unioned. + byAssignment, busByAssignment := map[string][]string{}, "" for _, raw := range each(r, ReadAssignments, &in.Gaps) { var a assignment if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" { continue } - nodes[a.Node] = true - in.Assigned[a.Node] = append(in.Assigned[a.Node], a.Module) + byAssignment[a.Node] = append(byAssignment[a.Node], a.Module) if a.Module == BusModule { - in.BusNode = a.Node + busByAssignment = a.Node } } // The seat's placement (ADR 0567 §9): what it places is composed beside the assignments, and the seats held once @@ -257,11 +285,27 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Placed = map[string]delivery.Placement{} } in.Placed[node] = p + } + } + away, _ := leaseHandedAway(r) + if away && len(in.Placed) > 0 { + in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "not used: the controller's lease is handed away, " + + "so the seat's placement alone places modules (ADR 0549 §3, §6)"}) + } else { + for node, ms := range byAssignment { nodes[node] = true - for _, m := range p.Assigned() { - if !slices.Contains(in.Assigned[node], m) { - in.Assigned[node] = append(in.Assigned[node], m) - } + in.Assigned[node] = append(in.Assigned[node], ms...) + } + in.BusNode = busByAssignment + } + for node, p := range in.Placed { + nodes[node] = true + for _, m := range p.Assigned() { + if !slices.Contains(in.Assigned[node], m) { + in.Assigned[node] = append(in.Assigned[node], m) + } + if m == BusModule && in.BusNode == "" { + in.BusNode = node } } } @@ -322,6 +366,15 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { if err == nil && v.Manifest == "" { err = fmt.Errorf("the indexer's value names no manifest digest") } + if err == nil && strings.HasPrefix(v.FP2, delivery.SourcePrefix2) { + in.Fingerprints[module] = v.FP2 + } else if err == nil { + why := v.None2 + if why == "" { + why = "the indexer's value names no fingerprint of the new form" + } + in.NoFingerprint[module] = why + } var m catalogue.Manifest if err == nil { m, err = cache.get(r, v.Manifest) diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 9692077..38da1f4 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/membership" ) @@ -23,31 +24,38 @@ const ( StateMemberships = "memberships" StateAssertions = "assertions" StateTakeover = "takeover" - KeyList = "list" - KeySwitch = "switch" + // StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go). + StateGrants = delivery.PartGrantsState + KeyList = "list" + KeySwitch = "switch" // KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList. KeyPass = "pass" KeyAsked = "asked" ) // States is every state the part keeps. -var States = []string{StateAssertions, StateMemberships, StateTakeover, StateUserList} +var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList, StateWithheld} // The events the part says. const ( EventUserList = "user-list-changed" EventMembership = "membership-changed" + // EventGrantWritten is said when the part wrote a grant of its state grants (ADR 0547 §5, ADR 0561 §2). + EventGrantWritten = "grant-written" // EventAsserted is said when the part created a bucket, log or stream a manifest declares (issue 1790). EventAsserted = "asserted" ) // Emits is every event the part says. -var Emits = []string{EventAsserted, EventMembership, EventUserList} +var Emits = []string{EventAsserted, EventGrantWritten, EventHashesAccepted, EventMembership, EventUserList} // Store is the part's own state, by local name: the only thing the part writes. type Store interface { Get(state, key string) ([]byte, uint64, bool, error) Put(state, key string, value []byte) (uint64, error) + // Keys and Delete are for the wanted accounts (accounts.go) and the grants (grants.go): every other state is only put. + Keys(state string) ([]string, error) + Delete(state, key string) error } // Emitter says one of the part's events. @@ -111,9 +119,17 @@ type Result struct { // free, `: a verified sender is composed as from no root-free machine there (issue 1945). RootNotFree []string `json:"root_not_free,omitempty"` // Sealed is the list sealed to the bus node as the key `user-list.sealed` names it (ADR 0547 clause 7). - Sealed *SealedState `json:"sealed,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + Sealed *SealedState `json:"sealed,omitempty"` + // Consumers is what the pass found and created of the durable consumers an account needs (ADR 0561 §6). + Consumers *ConsumersState `json:"consumers,omitempty"` + // Accounts is what the pass wrote of the wanted accounts the vault mints from (ADR 0561 §2). + Accounts *AccountsWritten `json:"accounts,omitempty"` + // Grants is what the pass wrote of its grants (grants.go). + Grants *GrantsWritten `json:"grants,omitempty"` + // Withheld is what the pass wrote of the hearings it withholds (withheld.go). + Withheld *WithheldWritten `json:"withheld,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -136,6 +152,12 @@ type Part struct { Logf func(string, ...any) // Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools. Objects Objects + // Consumers creates an account's durable consumer from the lease write (consumers.go): the process's connection; + // nil in the tools, which compare only. + Consumers ConsumerCreator + // Bus is where the nodes reach the bus, which each wanted account carries (accounts.go): the process's own credential's; + // empty in the tools, which write no account. + Bus BusAddress // Say says a condition to the operator: the process's; nil in the tools. Say func(asks.Condition) error // Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix. @@ -157,6 +179,14 @@ type Part struct { holdMu sync.Mutex heldSaid string heldKnown bool + // differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go). + differSaid string + differKnown bool + // missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go). + missingSaid string + missingKnown bool + // hashHold are the users whose hash changes held the pass's list, or nil (sealed.go). + hashHold []string } // Ask is the value of KeyAsked: a tool's ask to compose now, and, from grants_assert, to create one module's missing @@ -165,18 +195,21 @@ type Ask struct { By string `json:"by"` At time.Time `json:"at"` Assert string `json:"assert,omitempty"` + // Consumer is, from grants_consumer_match, one account consumer `.` to bring to the composed + // configuration by hand (ADR 0561 §6). + Consumer string `json:"consumer,omitempty"` } -// askedAssert is the module a new ask names to assert by hand, read once per ask; empty when none. -func (p *Part) askedAssert() string { +// asked is a new ask, read once per ask; the zero Ask when none. +func (p *Part) asked() Ask { raw, rev, found, err := p.Store.Get(StateUserList, KeyAsked) if err != nil || !found || rev == p.askSeen { - return "" + return Ask{} } p.askSeen = rev var a Ask _ = json.Unmarshal(raw, &a) - return a.Assert + return a } func (p *Part) now() time.Time { @@ -202,24 +235,28 @@ func (p *Part) Compose() Result { for _, g := range in.Refused { res.Refused = append(res.Refused, g.Input+": "+g.Why) } - recs, left := Records(in) + // recs say what runs (memberships, consumers); listRecs what the user list and the grants carry: what runs and what + // the trunk names, so the list grants a build's subjects before the want moves to it (issue 774). + recs, _ := Records(in) + listRecs, left := ListRecords(in) holdings := HoldingsOf(in) - res.RootNotFree = WithRootFree(&recs, p.Call) + res.RootNotFree = WithRootFree(&listRecs, p.Call) + recs.RootFree = listRecs.RootFree st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode, Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()} if st.LeftOut == nil { st.LeftOut = []Gap{} } - users, err := ComposeUsers(recs) + users, err := ComposeUsers(listRecs) if err != nil { res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error()) } users = WithHashes(users, in.Hashes) st.Digest, st.Users = Digest(users), len(users) - loaded, err := LoadedUsers(p.Call) - st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(recs))) - if err != nil { - st.Compare.Unread = err.Error() + loaded, loadErr := LoadedUsers(p.Call) + st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) + if loadErr != nil { + st.Compare.Unread = loadErr.Error() } st.Credentials = CredentialsOf(users, loaded, in.Hashes) st.Inputs = st.inputs() @@ -240,8 +277,23 @@ func (p *Part) Compose() Result { } } } + // The accounts the vault mints from, said in the state only it reads (ADR 0561 §2). + if p.Store != nil { + p.writeAccounts(&res, in, listRecs) + } + // The hearings withheld of what runs, for the self-check's probe D-heard (withheld.go). + if p.Store != nil { + p.writeWithheld(&res, in, recs) + } + list, n, leftOut, err := ComposeList(listRecs, in.Hashes) + // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over + // one that adds a permission only once the bus loaded a list that carries it (issue 774). + if p.Store != nil { + inForce, unknown := p.loadedContents(in, loaded, loadErr) + p.writeGrants(&res, in, listRecs, users, inForce, unknown) + } // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). - if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { + if err != nil { res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) } else { // The users the bus loaded: what a first list after take-over must keep where the part has not yet written one. @@ -250,15 +302,15 @@ func (p *Part) Compose() Result { bus = names(loaded) lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users)) } - p.writeSealed(&res, list, n, leftOut, in, bus, lost) + p.writeSealed(&res, list, n, leftOut, in, bus, lost, users) } ms := Memberships(recs) res.Memberships = len(ms) - composed := make([]membership.Membership, 0, len(ms)) + members := make([]membership.Membership, 0, len(ms)) for _, m := range ms { - composed = append(composed, m.Membership) + members = append(members, m.Membership) } - seated := holdings.ByMemberships(composed...) + seated := holdings.ByMemberships(members...) for _, key := range sortedKeys(ms) { m, now := publish(p.Publish, ms[key], holdings, seated) switch { @@ -274,10 +326,11 @@ func (p *Part) Compose() Result { } } held, heldErr := HeldStreams(p.Call) - forced := "" - if p.Create != nil { - forced = p.askedAssert() + var ask Ask + if p.Create != nil || p.Consumers != nil { + ask = p.asked() } + forced := ask.Assert for _, module := range sortedKeys(in.Manifests) { if !assigned(in, module) { continue @@ -299,6 +352,7 @@ func (p *Part) Compose() Result { res.AssertionsWritten++ } } + p.consumers(&res, recs, ask.Consumer) if _, _, found, err := p.Store.Get(StateTakeover, KeySwitch); err == nil && !found { data, _ := json.Marshal(Switch{V: 1, State: Shadow, Acts: []Act{}}) if _, err := p.Store.Put(StateTakeover, KeySwitch, data); err != nil { @@ -324,6 +378,11 @@ func (p *Part) Last() Result { } func assigned(in Inputs, module string) bool { + // The node-engine runs on every node though no node is assigned it (novox/hq ADR 0563 §2, §3): its manifest is + // placed everywhere, so its declared state is asserted and created as a placed module's. + if module == buscompose.EngineModule { + return true + } for _, ms := range in.Assigned { for _, m := range ms { if m == module { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index bf479b2..cf2556e 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -2,6 +2,7 @@ package grants import ( "fmt" + "reflect" "slices" "strings" @@ -14,7 +15,15 @@ import ( // BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest // the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow // composes what it can and says what it could not. -func Records(in Inputs) (buscompose.Records, []Gap) { +func Records(in Inputs) (buscompose.Records, []Gap) { return records(in, false) } + +// ListRecords is Records for the user list and the grants (issue 774): a module whose wanted build's manifest is not the +// trunk's is declared with what both grant (unionDeclared), so the list carries the trunk manifest's subjects before the +// assigner moves the want to it, and a grant at the trunk's fingerprint waits for nothing the want's move would bring. +// Memberships and consumers stay with Records: they say what runs. Pure. +func ListRecords(in Inputs) (buscompose.Records, []Gap) { return records(in, true) } + +func records(in Inputs, union bool) (buscompose.Records, []Gap) { var gaps []Gap seats := map[string]catalogue.SeatDeclaration{} declarers := map[string]string{} @@ -59,25 +68,38 @@ func Records(in Inputs) (buscompose.Records, []Gap) { gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"}) continue } - d := declaredFor(m, seats, declarers) + declare := func(m catalogue.Manifest) buscompose.Declared { + d := declaredFor(m, seats, declarers) + if len(in.Placed) > 0 { + d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + } + for _, sr := range catalogue.ReadsGranted(m, here, node) { + d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) + } + d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) + return d + } + d := declare(m) + trunk, onTrunk := in.Manifests[module] // What a node composing a consumer reads of it is the trunk manifest's served keys (frombus reads the indexer's // manifest of another node's module), and what runs may still be another build: both are granted (issue 1126). - if trunk, ok := in.Manifests[module]; ok { + if onTrunk { d.Served = slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(d.Served), catalogue.ServedSettings(trunk)...)))) } - if len(in.Placed) > 0 { - d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + if union && onTrunk && !reflect.DeepEqual(m, trunk) { + d = unionDeclared(d, declare(trunk)) } - for _, sr := range catalogue.ReadsGranted(m, here, node) { - d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) - } - d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) out.Assigned[node] = append(out.Assigned[node], d) - if m.Instances == catalogue.InstancesInterchangeable { + if m.Instances == catalogue.InstancesInterchangeable || union && onTrunk && trunk.Instances == catalogue.InstancesInterchangeable { out.Interchangeable[m.Module] = true } } } + // The node-engine's own manifest, placed on every node (ADR 0563): its state, logs and streams alone, created by this + // part; it adds no principal, since each node's engine speaks as the node's account. + if m, ok := in.Manifests[buscompose.EngineModule]; ok { + out.Engine = &buscompose.Declared{Module: m.Module, State: bucketsOf(m), Logs: logsOf(m), Streams: streamsOf(m)} + } return out, gaps } @@ -211,7 +233,7 @@ func bucketsOf(m catalogue.Manifest) []buscompose.Bucket { func logsOf(m catalogue.Manifest) []buscompose.Log { var out []buscompose.Log for _, l := range m.Logs { - out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap()}) + out = append(out, buscompose.Log{Module: m.Module, Name: l.Name, MaxMiB: l.Cap(), MaxAgeDays: l.MaxAgeDays}) } return out } @@ -272,21 +294,19 @@ func queueOwnersOf(m catalogue.Manifest) []buscompose.QueueOwner { } func processesOf(m catalogue.Manifest) []buscompose.ProcessPrincipal { - buses := m.ProcessBuses() - if len(buses) == 0 { + accounts := m.BusAccounts() + if len(accounts) == 0 { return nil } queues := queueOwnersOf(m) var out []buscompose.ProcessPrincipal - for _, r := range m.Resources { - id := fmt.Sprint(r["id"]) - b, declared := buses[id] - if !declared { - continue - } - pr := buscompose.ProcessPrincipal{Process: id, Emits: b.Emits, State: b.State, Asks: b.Asks, Invokes: b.Invokes} + // Every own bus account, processes first and then tools bundles (catalogue.BusAccounts): a tools bundle's account carries + // what it may remove from the bus (ADR 0564 Limits 4, issue 1952 S5). + for _, a := range accounts { + pr := buscompose.ProcessPrincipal{Process: a.ID, Emits: a.Bus.Emits, State: a.Bus.State, Asks: a.Bus.Asks, + Invokes: a.Bus.Invokes, Removes: a.Bus.Removes} for _, q := range queues { - if q.Process == id { + if q.Process == a.ID { pr.Queues = append(pr.Queues, q) } } @@ -351,3 +371,48 @@ func sameSeat(a, b string) bool { sb, okb := catalogue.SeatNamed(b) return oka && okb && sa.Name == sb.Name } + +// unionDeclared is what either declaration grants (issue 774): every list of b's entries a lacks appended to a's, a +// process of both merged by its name, a flag that adds a grant set where either sets it, and an account kept where +// either has one (NoAccount only where both say none). a's names and RunsAs stand. Pure. +func unionDeclared(a, b buscompose.Declared) buscompose.Declared { + out := a + unionFields(reflect.ValueOf(&out).Elem(), reflect.ValueOf(b)) + out.NoAccount = a.NoAccount && b.NoAccount + procs := slices.Clone(a.Processes) + for _, bp := range b.Processes { + i := slices.IndexFunc(procs, func(ap buscompose.ProcessPrincipal) bool { return ap.Process == bp.Process }) + if i < 0 { + procs = append(procs, bp) + continue + } + merged := procs[i] + unionFields(reflect.ValueOf(&merged).Elem(), reflect.ValueOf(bp)) + procs[i] = merged + } + out.Processes = procs + return out +} + +// unionFields appends to each slice field of dst the elements of src's it lacks, and sets each bool field src sets. +func unionFields(dst, src reflect.Value) { + for i := 0; i < dst.NumField(); i++ { + f, g := dst.Field(i), src.Field(i) + switch f.Kind() { + case reflect.Slice: + merged := reflect.AppendSlice(reflect.MakeSlice(f.Type(), 0, f.Len()+g.Len()), f) + for j := 0; j < g.Len(); j++ { + held := false + for k := 0; k < merged.Len() && !held; k++ { + held = reflect.DeepEqual(merged.Index(k).Interface(), g.Index(j).Interface()) + } + if !held { + merged = reflect.Append(merged, g.Index(j)) + } + } + f.Set(merged) + case reflect.Bool: + f.SetBool(f.Bool() || g.Bool()) + } + } +} diff --git a/modules/mesh-grants/internal/grants/rootfree.go b/modules/mesh-grants/internal/grants/rootfree.go index b4e1223..568b516 100644 --- a/modules/mesh-grants/internal/grants/rootfree.go +++ b/modules/mesh-grants/internal/grants/rootfree.go @@ -8,23 +8,51 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" ) // CallRootFree is the controller seat's verb that judges machines root-free (novox/hq ADR 0259 §8): the one judgement // the composer needs and the part cannot make from the bus (issue 1945). Only the controller answers it, on purpose. const CallRootFree = "seat:mesh-controller.root-free" -// The controller seat's condition events the part hears (never a timer): a machine's root-free word changes with the -// raising and the clearing of its `root-not-free` condition. +// The condition events the part hears (never a timer): a machine's root-free word changes with the raising and the +// clearing of its `root-not-free` condition. They are the controller seat's until the operator's take-over, and the +// condition keeper's (mesh-conditions) from it: the part hears only the seat the take-over key names, never both (novox/hq +// ADR 0564 §9, issue 1952 G1). const ( EventConditionRaised = "mesh-controller.condition-raised" EventConditionCleared = "mesh-controller.condition-cleared" + EventKeeperRaised = health.KeeperModule + ".condition-raised" + EventKeeperCleared = health.KeeperModule + ".condition-cleared" // KindRootNotFree is the condition kind whose events wake a pass. KindRootNotFree = "root-not-free" ) +// ReadTakeover is the condition keeper's take-over key's state (`switch`): which seat's condition events the part hears. +const ReadTakeover = health.KeeperModule + "." + health.TakeoverState + +// HeardFrom is the seat a condition event heard comes from, by its name (`.`) or its bus subject +// (`mesh.seat..event.`); empty for any other. +func HeardFrom(event string) string { + for _, seat := range []string{health.KeeperModule, "mesh-controller"} { + if strings.HasPrefix(event, seat+".") || strings.HasPrefix(event, "mesh.seat."+seat+".event.") { + return seat + } + } + return "" +} + +// EventAccountIssued is the vault's word that it minted an account (novox/hq ADR 0561 §1): its hash is in the vault's +// bus-accounts, so a pass composes the list with it. +const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent + // Consumes is every event the part hears, as its manifest's consumes name them. -var Consumes = []string{EventConditionRaised, EventConditionCleared} +var Consumes = []string{EventConditionRaised, EventConditionCleared, EventKeeperRaised, EventKeeperCleared, EventAccountIssued} + +// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject. +func IsAccountIssued(event string) bool { + return event == EventAccountIssued || event == buscompose.AccountIssuedSubject +} // EventSubject is the bus subject of a controller seat event named `mesh-controller.`. func EventSubject(event string) string { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index eb76a44..097e09d 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -2,10 +2,12 @@ package grants import ( "encoding/json" + "errors" "fmt" "regexp" "slices" "strings" + "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -72,7 +74,7 @@ func followOf(state string) string { // writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in // res what it wrote, or why it wrote nothing. -func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string) { +func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) { cur, _, had := p.Sealed() next := cur next.Follow = followOf(p.SwitchState()) @@ -91,11 +93,12 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string why = "no node runs the bus's module, so the list is sealed to nobody" case key == "": why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" - case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost) != "": + case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "": // After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user // of it for want of a hash: that user's module would be locked out of the bus (review of the switch). - why = p.holdsBack(in, leftOut, bus, lost) + why = p.holdsBack(in, leftOut, bus, lost, digest) held = why + p.keepHashHold(res, digest) case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: object, err := delivery.SealUserList(list, key) if err == nil { @@ -133,7 +136,9 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string return } if next.Digest == digest && cur.Digest != digest { - p.keepWrittenUsers(res, list) + p.keepWrittenUsers(res, list, in.Hashes) + p.keepSealedGrants(res, digest, list, composed) + p.acceptedWritten(res, digest) } // The object the key named before is taken away once the key names the new one: the bucket keeps the list in force. if had && cur.Object != "" && cur.Object != next.Object { @@ -288,11 +293,50 @@ func (p *Part) followReasons(res Result, in Inputs) []string { // composed after take-over must keep. const KeyWrittenUsers = "sealed-users" -func (p *Part) keepWrittenUsers(res *Result, list []byte) { - data, _ := json.Marshal(listUsers(list)) +// KeyWrittenCredentials is the key of the state user-list naming, per user of the list in force, which password its hash +// is of (credentialOf: adopted, or the vault's generation and fingerprint), never the hash: what tells a list that changes +// several loaded users' hashes at once (ADR 0561 §7). +const KeyWrittenCredentials = "sealed-credentials" + +func (p *Part) keepWrittenUsers(res *Result, list []byte, hashes map[string]HeldHash) { + users := listUsers(list) + data, _ := json.Marshal(users) if _, err := p.Store.Put(StateUserList, KeyWrittenUsers, data); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenUsers, err)) } + creds := map[string]string{} + for _, u := range users { + if h, ok := hashes[u]; ok { + creds[u] = credentialOf(h) + } + } + data, _ = json.Marshal(creds) + if _, err := p.Store.Put(StateUserList, KeyWrittenCredentials, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyWrittenCredentials, err)) + } +} + +// changedHashes are the users of the list in force whose hash the list composed now would change with no rotation by the +// vault naming them, sorted; none when the part kept no credentials of the list in force (a list written before). +func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { + raw, _, found, err := p.Store.Get(StateUserList, KeyWrittenCredentials) + if err != nil || !found { + return nil, err + } + var had map[string]string + if err := json.Unmarshal(raw, &had); err != nil { + return nil, err + } + var changed []string + for user, was := range had { + h, ok := hashes[user] + if !ok || credentialOf(h) == was || h.Rotating != nil { + continue + } + changed = append(changed, user) + } + slices.Sort(changed) + return changed, nil } var listUser = regexp.MustCompile(`\{\s*user:\s*"([^"]+)"`) @@ -312,7 +356,8 @@ func listUsers(list []byte) []string { // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when // they could not be read). Empty when it may. -func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string { +func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string { + p.hashHold = nil if f := in.ReadFailures(); len(f) > 0 { return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")" } @@ -342,5 +387,127 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string) string { if len(unhashed) > 0 { return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault" } + // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new + // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. + changed, err := p.changedHashes(in.Hashes) + if err != nil { + return "held: the credentials of the list in force cannot be read: " + err.Error() + } + if len(changed) > 1 && !p.accepted(digest, changed) { + p.hashHold = changed + return "held: the list in force is kept, since the hashes of " + fmt.Sprint(len(changed)) + " loaded users would change at " + + "once with no rotation by the vault naming them: " + strings.Join(changed, ", ") + } return "" } + +// **A person accepts a held list's hash changes** (ADR 0561 §7, §9; tools-are-how-a-module-is-spoken-to): the hold of +// several loaded users' hashes changing at once is released only by grants_accept_hashes, naming exactly the users held, +// with why, bound to the held list's digest. The acceptance releases this guard alone, once: the other holds (a failed +// read, a user dropped for want of a hash, a lost right) stand whatever it says; and a list composed otherwise since is +// held again, for a new acceptance. + +// KeyHashHold names the list held for its hash changes (its digest and the users), and KeyHashAccept a person's acceptance. +const ( + KeyHashHold = "hash-hold" + KeyHashAccept = "hash-accept" +) + +// EventHashesAccepted is said when a list held for its hash changes is written on a person's acceptance: a hand act. +const EventHashesAccepted = "hashes-accepted" + +// HashHold is the value of KeyHashHold, and HashAccept of KeyHashAccept. +type HashHold struct { + Digest string `json:"digest"` + Users []string `json:"users"` + At time.Time `json:"at"` +} + +type HashAccept struct { + Digest string `json:"digest"` + Users []string `json:"users"` + By string `json:"by"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// keepHashHold writes which list is held for its hash changes, when that is the hold. +func (p *Part) keepHashHold(res *Result, digest string) { + if p.hashHold == nil { + return + } + data, _ := json.Marshal(HashHold{Digest: digest, Users: p.hashHold, At: p.now()}) + if raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold); err == nil && found { + var was HashHold + if json.Unmarshal(raw, &was) == nil && was.Digest == digest && slices.Equal(was.Users, p.hashHold) { + return + } + } + if _, err := p.Store.Put(StateUserList, KeyHashHold, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeyHashHold, err)) + } +} + +// accepted says a person accepted exactly these users' hash changes of the list of this digest. +func (p *Part) accepted(digest string, users []string) bool { + raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept) + if err != nil || !found { + return false + } + var a HashAccept + return json.Unmarshal(raw, &a) == nil && a.Digest == digest && slices.Equal(a.Users, users) +} + +// acceptedWritten says, once, that the list a person accepted is written, and forgets the acceptance and the hold. +func (p *Part) acceptedWritten(res *Result, digest string) { + raw, _, found, err := p.Store.Get(StateUserList, KeyHashAccept) + if err != nil || !found { + return + } + var a HashAccept + if json.Unmarshal(raw, &a) != nil || a.Digest != digest { + return + } + p.say(res, EventHashesAccepted, map[string]any{"digest": digest, "users": a.Users, "by": a.By, "why": a.Why}) + for _, k := range []string{KeyHashAccept, KeyHashHold} { + if err := p.Store.Delete(StateUserList, k); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, k, err)) + } + } +} + +// AcceptHashes is the tool accept-hashes: a person accepts the hash changes of the list held for them, naming exactly its +// users, with why. Refused when no list is held for its hashes, or for another set of users. Bound to the held list's +// digest: a list composed otherwise since is held again. +func (p *Part) AcceptHashes(caller string, users []string, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("accept-hashes says why (`why`)") + } + raw, _, found, err := p.Store.Get(StateUserList, KeyHashHold) + if err != nil { + return nil, fmt.Errorf("the held list cannot be read: %w", err) + } + var h HashHold + if !found || json.Unmarshal(raw, &h) != nil || h.Digest == "" { + return nil, errors.New("no list is held for its users' hashes changing: there is nothing to accept") + } + named := slices.Sorted(slices.Values(users)) + if !slices.Equal(named, h.Users) { + return nil, fmt.Errorf("the list held (%s) changes the hashes of %s: an acceptance names exactly those, and %s is not that", + short(h.Digest), strings.Join(h.Users, ", "), strings.Join(named, ", ")) + } + data, _ := json.Marshal(HashAccept{Digest: h.Digest, Users: h.Users, By: someoneOr(caller), Why: strings.TrimSpace(why), At: p.now()}) + rev, err := p.Store.Put(StateUserList, KeyHashAccept, data) + if err != nil { + return nil, fmt.Errorf("the acceptance could not be written: %w", err) + } + return map[string]any{"accepted": rev, "digest": h.Digest, "users": h.Users, "says": "the part's process writes the held " + + "list at its next pass, once, unless another hold stands or the list composed then is another; it says hashes-accepted"}, nil +} + +func someoneOr(caller string) string { + if caller == "" { + return "a caller the runtime did not prove" + } + return caller +} diff --git a/modules/mesh-grants/internal/grants/takeover.go b/modules/mesh-grants/internal/grants/takeover.go index bd5e678..d5c94f7 100644 --- a/modules/mesh-grants/internal/grants/takeover.go +++ b/modules/mesh-grants/internal/grants/takeover.go @@ -1,6 +1,8 @@ package grants import ( + "git.novox.be/novox/mesh-sdk/go/handacts" + "crypto/rand" "encoding/hex" "encoding/json" @@ -378,6 +380,7 @@ func (p *Part) Warranted(o asks.Option, w asks.Warrant) error { if err := p.finish(s, a); err != nil { return err } + p.recordWarrant(w.Ask) // The bus node turns on the key: written now, so the switch reaches it on the operator's yes (ADR 0547 clause 8). if err := p.syncFollow(s.State); err != nil { return fmt.Errorf("the switch is recorded and the key the bus node follows could not be written (the next pass writes it): %w", err) @@ -512,3 +515,37 @@ func (p *Part) TakeOverState() (any, error) { } return out, nil } + +// Module is the grants part's module, the asker of its take-over and hand-back. +const Module = "mesh-grants" + +// The hand-act log's verbs the part records its switches by (novox/hq ADR 0565 §11, ADR 0547 §6): the seat's, and the +// controller's while the seat has no holder and the controller holds its lease (handacts.Recorder). +const ( + CallRecordWarrant = "seat:" + handacts.Seat + ".warranted" + CallRecordWarrantController = "seat:mesh-controller.warranted" +) + +// recordWarrant records, in the hand-act log, that a take-over or a hand-back was made on the operator's warrant for ask. +// Said, never fatal: the switch stands either way. +func (p *Part) recordWarrant(ask string) { + if p.Call == nil { + return + } + rec := handacts.Recorder{Ask: p.Call.Call, LeaseAway: func() bool { + if p.Read == nil { + return false + } + away, _ := leaseHandedAway(p.Read) + return away + }} + taken, err := rec.RecordWarrant(Module, ask) + if p.Logf == nil { + return + } + if err != nil { + p.Logf("the switch on the warrant %s is not recorded in the hand-act log: %v", ask, err) + return + } + p.Logf("the switch on the warrant %s is recorded in the log of %s", ask, taken.Log) +} diff --git a/modules/mesh-grants/internal/grants/tools.go b/modules/mesh-grants/internal/grants/tools.go index 09700dc..defb2f5 100644 --- a/modules/mesh-grants/internal/grants/tools.go +++ b/modules/mesh-grants/internal/grants/tools.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -21,43 +20,92 @@ const ( ToolTakeOver = "grants_take_over" ToolTakeOverState = "grants_take_over_state" ToolHandBack = "grants_hand_back" + ToolAccounts = "grants_accounts" + ToolConsumerMatch = "grants_consumer_match" + ToolAcceptHashes = "grants_accept_hashes" ) // Tools is every tool, in the manifest's order. -var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState} +var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState, + ToolAccounts, ToolConsumerMatch, ToolAcceptHashes} -// WritesNoGrant is what grants says of the part's own grants in this milestone. -const WritesNoGrant = "none: the grants part writes no grants.. before ADR 0547's third milestone (clause 5); the controller's are beside" +// WritesGrants is what grants says of the part's own grants in this milestone: written, and read by nobody until the +// controller's lease is handed away (delivery.GrantSource). +const WritesGrants = "in shadow: the grants part writes grants.. into its state grants, which the assigner reads only once the controller's lease is handed away (ADR 0547 clause 5); the controller's are beside, with every difference" -// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all. +// Grants answers the tool grants: the grants the part wrote, the controller's, and every difference between them, one +// module's or all. func (p *Part) Grants(module string) (any, error) { - keys, err := p.Read.Keys(ReadControllerGrant) - if err != nil { - return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil - } - type grant struct { - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Accounts []string `json:"accounts,omitempty"` - } - out := []grant{} - for _, k := range keys { - m, fp, err := delivery.ParseControllerGrantKey(k) - if err != nil || (module != "" && m != module) { - continue + out := map[string]any{"writes": WritesGrants} + read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.ControllerGrant, error) { + keys, one, err := get() + if err != nil { + return nil, err } - g := grant{Module: m, Fingerprint: fp} - if module != "" { - if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found { - if v, err := delivery.DecodeControllerGrant(raw); err == nil { - g.Accounts = v.Accounts - } + m := map[string]delivery.ControllerGrant{} + for _, k := range keys { + mod, _, err := delivery.ParseControllerGrantKey(k) + if err != nil || (module != "" && mod != module) { + continue } + raw, found, err := one(k) + if err != nil { + return nil, err + } + if !found { + continue + } + g, err := delivery.DecodeControllerGrant(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + m[k] = g } - out = append(out, g) + return m, nil } - sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint }) - return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil + controller, cerr := read(func() ([]string, func(string) ([]byte, bool, error), error) { + keys, err := p.Read.Keys(ReadControllerGrant) + return keys, func(k string) ([]byte, bool, error) { return p.Read.Get(ReadControllerGrant, k) }, err + }) + mine, perr := read(func() ([]string, func(string) ([]byte, bool, error), error) { + keys, err := p.Store.Keys(StateGrants) + return keys, func(k string) ([]byte, bool, error) { + raw, _, found, err := p.Store.Get(StateGrants, k) + return raw, found, err + }, err + }) + type grant struct { + Key string `json:"key"` + Module string `json:"module"` + Accounts []string `json:"accounts,omitempty"` + } + list := func(m map[string]delivery.ControllerGrant) []grant { + out := []grant{} + for _, k := range sortedKeys(m) { + mod, _, _ := delivery.ParseControllerGrantKey(k) + out = append(out, grant{Key: k, Module: mod, Accounts: m[k].Accounts}) + } + return out + } + if cerr != nil { + out["controller_unread"] = cerr.Error() + } else { + out["controller"], out["controller_count"] = list(controller), len(controller) + } + if perr != nil { + out["written_unread"] = perr.Error() + } else { + out["written"], out["count"] = list(mine), len(mine) + } + if cerr == nil && perr == nil { + diff := CompareGrants(controller, mine) + out["differences"], out["differing"] = diff, len(diff) + out["equal"] = len(diff) == 0 + } + if last := p.Last(); last.Grants != nil { + out["last_pass"] = last.Grants + } + return out, nil } // MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the diff --git a/modules/mesh-grants/internal/grants/withheld.go b/modules/mesh-grants/internal/grants/withheld.go new file mode 100644 index 0000000..1c130cb --- /dev/null +++ b/modules/mesh-grants/internal/grants/withheld.go @@ -0,0 +1,90 @@ +package grants + +import ( + "encoding/json" + "fmt" + "sort" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The hearings the part withholds (novox/hq ADR 0376 §2; issue 1955 L7, the lead 2026-10-29): a module granted to hear a +// restricted stream that runs in its machine's tool runner (no runs-as) hears none of it, since nothing is composed for +// the account every session there shares. The part writes each such hearing to its state `withheld`, on change only, so the +// self-check's probe D-heard reads it from the bus in place of the controller's store. +// +// Layout (mesh-sdk buscompose): key `..` (the controller's condition id of it); value +// buscompose.WithheldHearing; readers name buscompose.GrantsWithheldRead. +const StateWithheld = buscompose.GrantsWithheldState + +// WithheldHearing is one withheld hearing, as mesh-sdk lays it (buscompose.WithheldHearing). +type WithheldHearing = buscompose.WithheldHearing + +// WithheldKey is a withheld hearing's key in StateWithheld (buscompose.WithheldKey). +func WithheldKey(w buscompose.Withheld) string { return buscompose.WithheldKey(w) } + +// WithheldWritten is what one pass did to StateWithheld. +type WithheldWritten struct { + Withheld int `json:"withheld"` + Written []string `json:"written,omitempty"` + Deleted []string `json:"deleted,omitempty"` + // Kept is set when a read of the pass failed: no key of a hearing no longer withheld is deleted then. + Kept string `json:"kept,omitempty"` +} + +// writeWithheld writes each withheld hearing of what runs whose key differs (its At aside), and deletes the key of one no +// longer withheld when every input of the pass was read. +func (p *Part) writeWithheld(res *Result, in Inputs, recs buscompose.Records) { + now := buscompose.WithheldOf(recs) + done := &WithheldWritten{Withheld: len(now)} + res.Withheld = done + want := map[string]WithheldHearing{} + for _, w := range now { + want[WithheldKey(w)] = WithheldHearing{V: buscompose.WithheldVersion, Node: w.Node, Module: w.Module, Stream: w.Stream, Emitter: w.Emitter, Said: w.Said()} + } + for _, key := range sortedKeys(want) { + h := want[key] + raw, _, found, err := p.Store.Get(StateWithheld, key) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateWithheld, key, err)) + continue + } + if found { + var was WithheldHearing + if json.Unmarshal(raw, &was) == nil { + was.At = time.Time{} + if was == h { + continue + } + } + } + h.At = p.now() + data, _ := json.Marshal(h) + if _, err := p.Store.Put(StateWithheld, key, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateWithheld, key, err)) + continue + } + done.Written = append(done.Written, key) + } + held, err := p.Store.Keys(StateWithheld) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no hearing no longer withheld is deleted: %v", StateWithheld, err)) + return + } + if len(in.ReadFailures()) > 0 { + done.Kept = "a read of this pass failed, so no hearing no longer withheld is deleted" + return + } + sort.Strings(held) + for _, key := range held { + if _, still := want[key]; still { + continue + } + if err := p.Store.Delete(StateWithheld, key); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateWithheld, key, err)) + continue + } + done.Deleted = append(done.Deleted, key) + } +} diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 785008e..822b9e3 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -12,11 +12,16 @@ "mesh-delivery-indexer.modules", "mesh-delivery-assigner.placement", "node-engine.have", - "mesh-vault.bus-accounts" + "mesh-vault.bus-accounts", + "mesh-controller.lease", + "mesh-conditions.takeover" ], "consumes": [ "mesh-controller.condition-raised", - "mesh-controller.condition-cleared" + "mesh-controller.condition-cleared", + "mesh-conditions.condition-raised", + "mesh-conditions.condition-cleared", + "mesh-vault.account-issued" ], "uses": [ "operator-channel" @@ -25,10 +30,14 @@ "nats.nats_streams", "nats.nats_user_can", "nats.nats_users", - "seat:mesh-controller.root-free" + "seat:mesh-controller.root-free", + "seat:mesh-hand-acts.warranted", + "seat:mesh-controller.warranted" ], "emits": [ "asserted", + "grant-written", + "hashes-accepted", "membership-changed", "user-list-changed" ], @@ -40,13 +49,24 @@ "grants_assert", "grants_take_over", "grants_take_over_state", - "grants_hand_back" + "grants_hand_back", + "grants_accounts", + "grants_consumer_match", + "grants_accept_hashes" ], "state": [ + { + "name": "accounts", + "history": 5 + }, { "name": "assertions", "history": 5 }, + { + "name": "grants", + "history": 5 + }, { "name": "memberships", "history": 5 @@ -58,6 +78,10 @@ { "name": "user-list", "history": 20 + }, + { + "name": "withheld", + "history": 5 } ], "own-secrets": {