diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go index a1c25a10..86072e9f 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/homes.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -28,6 +28,9 @@ type Home struct { // Dropped is the reason of the probes the operator dropped from this release (D2, D3, D4, D12, D-heard, D-root, D11, // and D13's data held twice): their kinds end with the controller and are carried until the lease write. +// EndedD6 is the reason of the bus-consumer probe's kinds: the probe ended with the controller (issue 1126). +const EndedD6 = "ended with the controller on 2026-11-02; the grants part says its account consumers' differences and delays (issue 1126)" + const Dropped = "dropped 2026-10-30 by the operator; the owning module declares its own check later (research 074)" // The kinds' homes, by kind: ADR 0564's rows, and the 63 kinds the controller's code raises that the record left without @@ -61,9 +64,6 @@ var Homes = map[string]Home{ "directory-used-as-found": {Source: "health", Controller: true, Observer: watchman}, // The self-check's probes (clause 6, the Context's table). "lease-split": {Source: "D5", Controller: true, Observer: selfCheck}, // a seat held once answers from two holders - "consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck}, - "consumer-lost": {Source: "D6", Controller: true, Observer: selfCheck}, // and the watchman's S9 - "consumer-behind": {Source: "D6", Controller: true, Observer: selfCheck}, "stream-wrong": {Source: "D7", Controller: true, Observer: selfCheck}, "own-address-banned": {Source: "D8", Controller: true, Observer: selfCheck}, "data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck}, @@ -118,6 +118,12 @@ var Homes = map[string]Home{ // is carried until then, approved by the operator on 2026-10-31 (issue 1952); ADR 0564's table had homed it with the // assigner, which writes no such finding, and the lead records the change of the record. "bus-upgrade-failed": {Source: "DB", Controller: true, Carried: true}, + // The self-check's bus-consumer probe (D6) judged the consumers the controller defined, and ended with it (issue + // 1126): the account consumers are the grants part's, which raises mesh-grants.consumer-differs and + // mesh-grants.consumer-behind of its own. Its kinds are carried, as the others that end with the controller. + "consumer-wrong": {Source: "D6", Controller: true, Carried: true, Why: EndedD6}, + "consumer-lost": {Source: "D6", Controller: true, Carried: true, Why: EndedD6}, + "consumer-behind": {Source: "D6", Controller: true, Carried: true, Why: EndedD6}, // The probes the operator dropped from this release (D2 resolvers, D3 holders, D4 archives, D12 bindings, D-heard // withheld hearings, D-root, D11 retired data, and D13's data held twice): their kinds are carried until the lease // write. diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 01367614..c2d6b3bb 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -287,8 +287,14 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { t.Errorf("the dropped probe's kind %s is %+v", kind, h) } } - // And the controller's own bus upgrade, carried (the operator, 2026-10-31): 34. - if len(Homes) != 84 || carried != 34 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled + // And the controller's own bus upgrade, carried (the operator, 2026-10-31): 34; and the bus-consumer probe's three + // kinds, ended with the controller (issue 1126): 37. + for _, kind := range []string{"consumer-wrong", "consumer-lost", "consumer-behind"} { + if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != EndedD6 { + t.Errorf("the ended probe's kind %s is %+v", kind, h) + } + } + if len(Homes) != 84 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried) } } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/observation.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go index ac535f44..782d6bd0 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/observation.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -107,16 +107,6 @@ type plain struct{ Headline, Explanation, Resolved, Needs string } // words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the // controller's name for itself replaced by the self-check's where the words spoke of it. var words = map[string]plain{ - "consumer-lost": {Headline: "A listener on the bus is gone", - Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", - Resolved: "The listener is back"}, - "consumer-wrong": {Headline: "A listener on the bus is set up wrong", - Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.", - Resolved: "The listener is set up right again"}, - "consumer-behind": {Headline: "Messages pile up for a listener", - Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " + - "restarts it if it stays.", - Resolved: "The listener caught up"}, "stream-wrong": {Headline: "Part of the bus's storage is wrong", Needs: "check the machine the bus runs on; the details say what is missing.", Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 5e0d3db3..f78e8f5a 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -4,8 +4,9 @@ package main // each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home // on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the // table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the -// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and -// the bus step (its home is issue 1953's). +// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding), the +// bus step (its home is issue 1953's), and D6: the consumers it judged were the controller's, and the account consumers +// are the grants part's, which raises mesh-grants.consumer-differs and mesh-grants.consumer-behind (issue 1126). import ( "context" @@ -26,7 +27,6 @@ const ( SignalsVerb = "seat:mesh-watchdogs.signals" notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; " storeOnly = "held only in the controller's store today" - behindBound = 1000 reconnectMax = 12 ) @@ -50,9 +50,6 @@ func Registry() []Probe { "records are " + storeOnly + ", and the watchman's presence has no read granted to this module"}, {ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld", Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly}, - {ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " + - "stream's head", From: "issues 248, 266", Kind: "consumer-wrong", - Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers}, {ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208", Kind: "stream-wrong", Run: probeStreams}, {ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list", @@ -150,80 +147,6 @@ func expectedConsumers() map[string][]buscompose.Consumer { return out } -// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. -func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { - var out []Observation - byStream := expectedConsumers() - streams := make([]string, 0, len(byStream)) - for s := range byStream { - streams = append(streams, s) - } - sort.Strings(streams) - for _, stream := range streams { - if ctx.Err() != nil { - return nil, ctx.Err() - } - rows, err := c.streams(stream) - held := map[string]consumerRow{} - switch { - case errors.Is(err, errNoStream): - case err != nil: - return nil, err - default: - for _, r := range rows { - for _, cr := range r.Consumers { - held[cr.Name] = cr - } - } - } - for _, want := range byStream[stream] { - who := want.Stream + "." + want.Name - have, ok := held[want.Name] - if !ok { - out = append(out, Observation{Kind: "consumer-lost", Ref: "missing", Severity: Warning, - Subject: Subject{Scope: ScopeBus, ID: who}, - Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), - Said: "no consumer " + want.Name + " on " + want.Stream}) - continue - } - if differs := consumerDiffers(want, have); differs != "" { - out = append(out, Observation{Ref: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, - Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), - Said: differs}) - } - if want.Stream != "NODES" && have.Pending > behindBound { - // One look at a consumer a burst put behind is not one that stays behind: held for the next run, - // so healer H4 resets nothing on a single sample (novox/hq issue 277). - out = append(out, Observation{Kind: "consumer-behind", Ref: "behind", Severity: Warning, Confirm: true, - Subject: Subject{Scope: ScopeBus, ID: who}, - Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), - Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) - } - } - } - return sorted(out), nil -} - -// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its -// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is. -func consumerDiffers(want buscompose.Consumer, have consumerRow) string { - var differs []string - haveFilters := slices.Clone(have.Filters) - wantFilters := slices.Clone(want.Filters) - sort.Strings(haveFilters) - sort.Strings(wantFilters) - if !slices.Equal(haveFilters, wantFilters) { - differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters)) - } - if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) { - differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver)) - } - if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush { - differs = append(differs, "delivers by the other shape (push or pull) than defined") - } - return strings.Join(differs, "; ") -} - // probeStreams is D7: every stream the mesh defines exists as defined. func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { rows, err := c.streams("") diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index e1789f93..d980724c 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -129,19 +129,6 @@ func TestTheBusProbes(t *testing.T) { err string mesh *memStore // the bus states the machine probes read; nil reads none }{ - {"D6 all as defined", probeConsumers, bus{}, nil, "", nil}, - {"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}}, - []string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, "", nil}, - {"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}}, - []string{"bus.CONTROL.controller.missing"}, "", nil}, - {"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) { - for i, c := range rows["EVENTS"].Consumers { - if c.Name == "controller" { - rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000 - } - } - }}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, "", nil}, - {"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, {"D7 all as defined", probeStreams, bus{}, nil, "", nil}, {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { rows["EVENTS"].Retention = "workqueue" @@ -361,7 +348,7 @@ func agents(verdicts map[string]string) *memStore { // The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what // it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). func TestTheRegistryIsADR0564sTable(t *testing.T) { - moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", + moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", "DW", "H-bus"} var ids []string for _, p := range Registry() { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index 09fa8f5c..ec294bb0 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -120,7 +120,7 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} missing := Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "the stream EVENTS is not on the bus"} - behind := Observation{Kind: "consumer-behind", Ref: "behind", Severity: Warning, Confirm: true, + behind := Observation{Kind: "bus-reconnects", Ref: "behind", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} store := newStore() c, said := checker(store, steady, deferred) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go index 80e256d1..446ee344 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/tools.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -39,7 +39,7 @@ func Tools(c *Checker) []stdio.Tool { "every probe now and answers that run; probe= gives one probe's verdict whole, every finding.", Input: obj(map[string]any{ "run": map[string]any{"type": "boolean", "description": "run every probe now"}, - "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}), + "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D7"}}), Run: func(a map[string]any) (any, error) { return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a)) }},