Count a bus refusal by what the bus said, not by how often the controller looked (issue 402)
mesh/merge-gate pass: builds mesh-controller → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

S9 reads the one remembered advisory again on every look for an hour, and
each look was kept as an observation and a line of evidence, so a single
refusal read as one repeated every 30 seconds. An observation may now say
when what it reads happened and how often; the keeper counts that, and a
look with nothing newer adds nothing. Sources that look at the present
keep counting their looks.
This commit is contained in:
2026-10-11 02:10:48 +02:00
parent f008fab9d8
commit c11222026a
4 changed files with 131 additions and 9 deletions
@@ -0,0 +1,95 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
// never how often the controller looked.
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
now := refused.Add(10 * time.Second)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
t.Cleanup(func() { k.Close(t.Context()) })
// The words the live condition bus.controller.refused carried on 2026-10-10.
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
`Publish to "mesh.seat.mesh-delivery.tool.times"`
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
First: refused, Last: refused, Count: 1}
look := func() conditions.Condition {
t.Helper()
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
t.Fatal(err)
}
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
if err != nil || !found {
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
}
return c
}
var c conditions.Condition
for range 15 {
c = look()
now = now.Add(30 * time.Second)
}
if c.Observations != 1 || len(c.Evidence) != 1 {
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
c.Observations, len(c.Evidence), c.Evidence)
}
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
}
// The bus refuses it three times more between two looks: the condition counts four refusals, says
// the newest, and adds one line of evidence for the look that saw them.
again := now.Add(-5 * time.Second)
advisory.Last, advisory.Count = again, 4
c = look()
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
}
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
t.Fatalf("summary %q", c.Summary)
}
now = now.Add(30 * time.Second)
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
}
}
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
// what reads a record): a machine silent for three looks was observed three times, and says so.
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
t.Cleanup(func() { k.Close(t.Context()) })
var c conditions.Condition
for range 3 {
var err error
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
Source: "S1"}); err != nil {
t.Fatal(err)
}
now = now.Add(30 * time.Second)
}
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
}
}
+4 -1
View File
@@ -533,7 +533,10 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
machine = f.host
}
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said,
// The advisory is remembered and read again on every look for an hour: the condition counts
// what the bus said and when, not the looks (novox/hq issue 402).
Happened: a.Last, Times: a.Count}
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
o.Machine = consumerMachine(a.Stream, a.Consumer)
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
+12 -1
View File
@@ -152,7 +152,9 @@ type Condition struct {
// reopened after a send was there at the send unless the send fell in one of its gaps.
First time.Time `json:"first,omitzero"`
Gaps []Gap `json:"gaps,omitempty"`
// Observations is how many times it was observed since raised.
// Observations is how many times it was observed since raised: a look that sees it, for a source
// that looks at the present; for one that reads a record of what happened (Observation.Happened),
// how many times it happened, and LastObserved when it last did (novox/hq issue 402).
Observations int `json:"observations"`
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
Count int `json:"count"`
@@ -208,6 +210,15 @@ type Observation struct {
Said string
Source string
Resolver string
// Happened is set by a source that reads a record of what happened rather than looking at the
// present — a bus advisory, remembered for an hour and read again on every look — to when it last
// happened; Times is how many times the record counts it. The condition then counts what happened,
// not the looks (novox/hq issue 402): one refusal looked at every half minute read as "observed 15
// time(s)", with a line of evidence every 30 seconds, and was taken for a refusal going on. A look
// that brings nothing newer than the condition's last observation adds no observation and no
// evidence. Zero for a source that looks at the present, whose every look is an observation.
Happened time.Time
Times int
// Confirm says a single look can be wrong about this finding — a question over the network that
// went unanswered, a time measured once on a loaded machine. The keeper does not read it: the
// source that looks again does, and raises it only when the next look sees it too, or while it is
+20 -7
View File
@@ -191,11 +191,16 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if err != nil {
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
}
// What was observed and when: the look, or what the record says happened (novox/hq issue 402).
at, observations := now, 1
if !o.Happened.IsZero() {
at, observations = o.Happened.UTC(), max(1, o.Times)
}
if !found {
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: at, Said: said}},
Source: o.Source, Raised: now, LastObserved: at, Observations: observations, Count: 1,
Resolver: orSelf(o.Resolver)}
change := ChangeRaised
k.mu.Lock()
@@ -244,7 +249,10 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
}
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
// A record read again with nothing newer in it is not observed again: its words are refreshed,
// its count, evidence and last observation stay (novox/hq issue 402).
fresh := o.Happened.IsZero() || at.After(c.LastObserved)
c.Kind, c.Summary, c.Source = o.Kind, o.Summary, o.Source
wasNeeds, wasActions := c.Needs, c.Actions
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c)
@@ -257,10 +265,15 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if len(o.Also) > 0 {
c.Subject.Also = o.Also
}
c.Observations++
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
if len(c.Evidence) > KeptEvidence {
c.Evidence = c.Evidence[:KeptEvidence]
if fresh {
// A record counts what happened since it began; the condition never counts down, so a
// record begun again (a controller restarted) still adds the one it shows.
c.Observations = max(c.Observations+1, o.Times)
c.LastObserved = at
c.Evidence = append([]Evidence{{At: at, Said: said}}, c.Evidence...)
if len(c.Evidence) > KeptEvidence {
c.Evidence = c.Evidence[:KeptEvidence]
}
}
if err := k.stamp(&c); err != nil {
return Condition{}, err