novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The watchman raises lease-altered and removal-failed as findings, and clears them when the lease and the removal read right

Browse Source 
ADR 0582 clauses 4 and 6: mesh-grants says lease-altered and removal-failed "as a finding the watchman raises".
The watchman hears the seat mesh-grants' three events on core subjects (mesh-sdk WatchmanHears, its grant and
its subscriptions from one list, rather than a consume and a durable consumer), raises lease-altered naming
what changed and clears it at the next lease-handed, and raises removal-failed naming what failed and clears it
once what it names reads gone: the controller's consumer of DEAD_LETTER_NOTICES by its deletion advisory
(heard already), the stream ASSIGNMENTS by its own. The finding DEAD_LETTER_NOTICES.controller stays as it is.

Test: a row in TestTheWatchmansFindings; no new function.

(issue 774, 774/S22)

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 00:37:54 +01:00
parent a78fa3db9d
commit e18be5cabe
4 changed files with 126 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/signals.go
+27
View File
@@ -13,6 +13,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-watchman/internal/deadletters"
@@ -38,6 +39,13 @@ const (
routeDead = "dead-letter"
routeBusFault = "bus-fault"
routeProv = "provisioned"
routeLease = "lease"
)
// The lease write's subjects (ADR 0582 clauses 4, 6): the seat mesh-grants' events, and ASSIGNMENTS deleted.
var (
leaseEventPrefix = "mesh.seat." + catalogue.GrantsSeat + ".event."
assignmentsGone = "$JS.EVENT.ADVISORY.STREAM.DELETED." + buscompose.AssignmentsStream
)
// The bus module's events of the connections the bus closed (S9).
@@ -61,6 +69,8 @@ func routeOf(subject string) string {
return routeTools
case subject == CheckedSubject:
return routeChecked
case strings.HasPrefix(subject, leaseEventPrefix) || subject == assignmentsGone:
return routeLease
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
return routeAdvisory
case strings.HasPrefix(subject, "mesh.events.dead."):
@@ -96,6 +106,7 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
}
if dead.isTheControllersNotices(a.ID) {
dead.check() // the lease write's step removed it: the watchman may take the notices now
w.RemovalGone(watch.RemovalConsumer)
return
}
w.Advised(a)
@@ -110,6 +121,22 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
kind = watch.KindConnectionClosed
}
w.Closed(kind, c)
case routeLease:
var body struct {
Altered string `json:"altered"`
Failed string `json:"failed"`
}
_ = json.Unmarshal(m.Data, &body)
switch m.Subject {
case assignmentsGone:
w.RemovalGone(watch.RemovalStream)
case leaseEventPrefix + catalogue.GrantsEventLeaseAltered:
w.LeaseAltered(body.Altered)
case leaseEventPrefix + catalogue.GrantsEventLeaseHanded:
w.LeaseHanded()
case leaseEventPrefix + catalogue.GrantsEventRemovalFailed:
w.RemovalFailed(body.Failed)
}
case routeProv:
var r watch.ProvReport
if err := json.Unmarshal(m.Data, &r); err != nil {
modules/mesh-delivery-watchman/internal/watch/findings_test.go
+17
View File
@@ -59,6 +59,23 @@ func TestTheWatchmansFindings(t *testing.T) {
}
w.ModuleVerdict("node-tools", value(""), Put)
}, map[string]string{}},
{"774/S22/AC1, AC2: lease-altered and removal-failed are findings; lease-handed and the removal read gone clear them", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.LeaseAltered("the key holder names controller@a pid 9, not mesh-grants by lease-write-1")
if f := o.findings[FindingKey(ScopeBus, "mesh-controller_lease", KindLeaseAltered)]; !strings.Contains(f.Summary, "controller@a pid 9") {
t.Fatalf("findings = %v", o.findings)
}
w.LeaseHanded()
w.RemovalFailed("the stream ASSIGNMENTS: ASSIGNMENTS has 1 consumers")
k := FindingKey(ScopeBus, "lease-removal", KindRemovalFailed)
if f := o.findings[k]; !strings.Contains(f.Summary, "ASSIGNMENTS has 1 consumers") {
t.Fatalf("findings = %v", o.findings)
}
w.RemovalGone(RemovalConsumer) // not what it waits on
if _, ok := o.findings[k]; !ok {
t.Fatal("cleared by the consumer, while the stream it names stands")
}
w.RemovalGone(RemovalStream)
}, map[string]string{}},
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Step("go", "n1.ask1", runningStep("ask1", t0), Put)
expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second)
modules/mesh-delivery-watchman/internal/watch/lease.go
+80
View File
@@ -0,0 +1,80 @@
package watch
import (
"strings"
)
// **The lease write's findings** (novox/hq ADR 0582 clauses 4 and 6, issue 774): mesh-grants says lease-altered when the
// lease key or its bucket's expiry differs from what it wrote, and removal-failed when the day-later removal of the
// controller's consumer of DEAD_LETTER_NOTICES or the stream ASSIGNMENTS failed; the watchman raises each as a finding.
// lease-altered clears when lease-handed is said again (the operator asked the write again); removal-failed clears when
// what it names reads gone: the consumer's deletion advisory and ASSIGNMENTS' deletion advisory, each heard.
// The two kinds.
const (
KindLeaseAltered = "lease-altered"
KindRemovalFailed = "removal-failed"
)
// The finding's IDs: the lease's bucket, and the removal.
const (
leaseID = "mesh-controller_lease"
removalID = "lease-removal"
)
// What a failed removal still waits on.
const (
RemovalConsumer = "DEAD_LETTER_NOTICES.controller"
RemovalStream = "ASSIGNMENTS"
)
// LeaseAltered raises the finding that the lease differs from what mesh-grants wrote, naming what changed.
func (w *Watchman) LeaseAltered(what string) {
w.mu.Lock()
defer w.mu.Unlock()
w.find(Finding{Scope: ScopeBus, ID: leaseID, Kind: KindLeaseAltered, Severity: Warning, Source: "mesh-grants",
Summary: "the controller's lease differs from what mesh-grants wrote: " + what,
Said: what,
Needs: "a look at who changed the lease; mesh-grants writes nothing until lease-write is asked again.",
Resolved: "lease-write is asked and warranted again, and the lease reads as written"})
}
// LeaseHanded clears the lease-altered finding: the lease reads as written again.
func (w *Watchman) LeaseHanded() {
w.mu.Lock()
defer w.mu.Unlock()
w.unfind(FindingKey(ScopeBus, leaseID, KindLeaseAltered))
}
// RemovalFailed raises the finding that the day-later removal failed, naming what failed; it waits on each of the
// consumer and the stream the failure names.
func (w *Watchman) RemovalFailed(failed string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.removalWaits == nil {
w.removalWaits = map[string]bool{}
}
for _, what := range []string{RemovalConsumer, RemovalStream} {
if strings.Contains(failed, strings.SplitN(what, ".", 2)[0]) {
w.removalWaits[what] = true
}
}
if len(w.removalWaits) == 0 { // names neither: waits on both
w.removalWaits[RemovalConsumer], w.removalWaits[RemovalStream] = true, true
}
w.find(Finding{Scope: ScopeBus, ID: removalID, Kind: KindRemovalFailed, Severity: Warning, Source: "mesh-grants",
Summary: "the day-later removal of the controller's consumer and stream failed: " + failed,
Said: failed,
Needs: "a look at why; mesh-grants tries again in a day, and grants_removal_due starts it sooner.",
Resolved: "the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS read gone"})
}
// RemovalGone hears one of the two removed (RemovalConsumer or RemovalStream), and clears the finding once neither waits.
func (w *Watchman) RemovalGone(what string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.removalWaits, what)
if len(w.removalWaits) == 0 {
w.unfind(FindingKey(ScopeBus, removalID, KindRemovalFailed))
}
}
modules/mesh-delivery-watchman/internal/watch/watch.go
+2
View File
@@ -86,6 +86,8 @@ type Watchman struct {
recount func()
// waiting is, by `<consumer>.<node>`, each of its provisions in waits, with its words.
waiting map[string]map[string]string
// removalWaits is what a failed removal of the lease write still waits on to read gone (lease.go).
removalWaits map[string]bool
// health is each node's own health report, and tunnels each node's tunnel key from its record.
health map[string]delivery.NodeHealth
tunnels map[string]string