diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go index 501800a..c5922f8 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-watchman/internal/deadletters" @@ -38,6 +39,13 @@ const ( routeDead = "dead-letter" routeBusFault = "bus-fault" routeProv = "provisioned" + routeLease = "lease" +) + +// The lease write's subjects (ADR 0582 clauses 4, 6): the seat mesh-grants' events, and ASSIGNMENTS deleted. +var ( + leaseEventPrefix = "mesh.seat." + catalogue.GrantsSeat + ".event." + assignmentsGone = "$JS.EVENT.ADVISORY.STREAM.DELETED." + buscompose.AssignmentsStream ) // The bus module's events of the connections the bus closed (S9). @@ -61,6 +69,8 @@ func routeOf(subject string) string { return routeTools case subject == CheckedSubject: return routeChecked + case strings.HasPrefix(subject, leaseEventPrefix) || subject == assignmentsGone: + return routeLease case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."): return routeAdvisory case strings.HasPrefix(subject, "mesh.events.dead."): @@ -96,6 +106,7 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, . } if dead.isTheControllersNotices(a.ID) { dead.check() // the lease write's step removed it: the watchman may take the notices now + w.RemovalGone(watch.RemovalConsumer) return } w.Advised(a) @@ -110,6 +121,22 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, . kind = watch.KindConnectionClosed } w.Closed(kind, c) + case routeLease: + var body struct { + Altered string `json:"altered"` + Failed string `json:"failed"` + } + _ = json.Unmarshal(m.Data, &body) + switch m.Subject { + case assignmentsGone: + w.RemovalGone(watch.RemovalStream) + case leaseEventPrefix + catalogue.GrantsEventLeaseAltered: + w.LeaseAltered(body.Altered) + case leaseEventPrefix + catalogue.GrantsEventLeaseHanded: + w.LeaseHanded() + case leaseEventPrefix + catalogue.GrantsEventRemovalFailed: + w.RemovalFailed(body.Failed) + } case routeProv: var r watch.ProvReport if err := json.Unmarshal(m.Data, &r); err != nil { diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index b067f4a..d25eec9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -59,6 +59,23 @@ func TestTheWatchmansFindings(t *testing.T) { } w.ModuleVerdict("node-tools", value(""), Put) }, map[string]string{}}, + {"774/S22/AC1, AC2: lease-altered and removal-failed are findings; lease-handed and the removal read gone clear them", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.LeaseAltered("the key holder names controller@a pid 9, not mesh-grants by lease-write-1") + if f := o.findings[FindingKey(ScopeBus, "mesh-controller_lease", KindLeaseAltered)]; !strings.Contains(f.Summary, "controller@a pid 9") { + t.Fatalf("findings = %v", o.findings) + } + w.LeaseHanded() + w.RemovalFailed("the stream ASSIGNMENTS: ASSIGNMENTS has 1 consumers") + k := FindingKey(ScopeBus, "lease-removal", KindRemovalFailed) + if f := o.findings[k]; !strings.Contains(f.Summary, "ASSIGNMENTS has 1 consumers") { + t.Fatalf("findings = %v", o.findings) + } + w.RemovalGone(RemovalConsumer) // not what it waits on + if _, ok := o.findings[k]; !ok { + t.Fatal("cleared by the consumer, while the stream it names stands") + } + w.RemovalGone(RemovalStream) + }, map[string]string{}}, {"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Step("go", "n1.ask1", runningStep("ask1", t0), Put) expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second) diff --git a/modules/mesh-delivery-watchman/internal/watch/lease.go b/modules/mesh-delivery-watchman/internal/watch/lease.go new file mode 100644 index 0000000..5142c9d --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/lease.go @@ -0,0 +1,80 @@ +package watch + +import ( + "strings" +) + +// **The lease write's findings** (novox/hq ADR 0582 clauses 4 and 6, issue 774): mesh-grants says lease-altered when the +// lease key or its bucket's expiry differs from what it wrote, and removal-failed when the day-later removal of the +// controller's consumer of DEAD_LETTER_NOTICES or the stream ASSIGNMENTS failed; the watchman raises each as a finding. +// lease-altered clears when lease-handed is said again (the operator asked the write again); removal-failed clears when +// what it names reads gone: the consumer's deletion advisory and ASSIGNMENTS' deletion advisory, each heard. + +// The two kinds. +const ( + KindLeaseAltered = "lease-altered" + KindRemovalFailed = "removal-failed" +) + +// The finding's IDs: the lease's bucket, and the removal. +const ( + leaseID = "mesh-controller_lease" + removalID = "lease-removal" +) + +// What a failed removal still waits on. +const ( + RemovalConsumer = "DEAD_LETTER_NOTICES.controller" + RemovalStream = "ASSIGNMENTS" +) + +// LeaseAltered raises the finding that the lease differs from what mesh-grants wrote, naming what changed. +func (w *Watchman) LeaseAltered(what string) { + w.mu.Lock() + defer w.mu.Unlock() + w.find(Finding{Scope: ScopeBus, ID: leaseID, Kind: KindLeaseAltered, Severity: Warning, Source: "mesh-grants", + Summary: "the controller's lease differs from what mesh-grants wrote: " + what, + Said: what, + Needs: "a look at who changed the lease; mesh-grants writes nothing until lease-write is asked again.", + Resolved: "lease-write is asked and warranted again, and the lease reads as written"}) +} + +// LeaseHanded clears the lease-altered finding: the lease reads as written again. +func (w *Watchman) LeaseHanded() { + w.mu.Lock() + defer w.mu.Unlock() + w.unfind(FindingKey(ScopeBus, leaseID, KindLeaseAltered)) +} + +// RemovalFailed raises the finding that the day-later removal failed, naming what failed; it waits on each of the +// consumer and the stream the failure names. +func (w *Watchman) RemovalFailed(failed string) { + w.mu.Lock() + defer w.mu.Unlock() + if w.removalWaits == nil { + w.removalWaits = map[string]bool{} + } + for _, what := range []string{RemovalConsumer, RemovalStream} { + if strings.Contains(failed, strings.SplitN(what, ".", 2)[0]) { + w.removalWaits[what] = true + } + } + if len(w.removalWaits) == 0 { // names neither: waits on both + w.removalWaits[RemovalConsumer], w.removalWaits[RemovalStream] = true, true + } + w.find(Finding{Scope: ScopeBus, ID: removalID, Kind: KindRemovalFailed, Severity: Warning, Source: "mesh-grants", + Summary: "the day-later removal of the controller's consumer and stream failed: " + failed, + Said: failed, + Needs: "a look at why; mesh-grants tries again in a day, and grants_removal_due starts it sooner.", + Resolved: "the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS read gone"}) +} + +// RemovalGone hears one of the two removed (RemovalConsumer or RemovalStream), and clears the finding once neither waits. +func (w *Watchman) RemovalGone(what string) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.removalWaits, what) + if len(w.removalWaits) == 0 { + w.unfind(FindingKey(ScopeBus, removalID, KindRemovalFailed)) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index fa6d5b7..a8d355e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -86,6 +86,8 @@ type Watchman struct { recount func() // waiting is, by `.`, each of its provisions in waits, with its words. waiting map[string]map[string]string + // removalWaits is what a failed removal of the lease write still waits on to read gone (lease.go). + removalWaits map[string]bool // health is each node's own health report, and tunnels each node's tunnel key from its record. health map[string]delivery.NodeHealth tunnels map[string]string