novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Look twice before saying a probe failed, and say conditions in machine names (hq issue 277)

Browse Source 
D2 raised a resolver urgent on one query that timed out while its machine was
loaded, and its summary carried the resolver's address and socket text, so the
operator channel withheld the whole alert.

- D2 asks every question up to three times, all at once; a resolver that
  answers nothing is held for the next run and raised urgent when two runs
  in a row find it silent. A wrong answer is still raised at once.
- Findings a single look can be wrong about carry Confirm: raised on the
  second look in a row, kept while open, never cleared-and-reraised. Used by
  D2 silence, D3 (also asks discovery twice), D6 behind, D9, D13 unmeasured,
  probe-failed of the doctor, and blind watchdog rows.
- Probe seat asks (D8, D13) are asked again when the bus brought no answer.
- Summaries name machines and say things in words; addresses, paths,
  domains and raw errors move to the evidence (D2, D5, D8, D9, D13, S12).
- internal/outward mirrors the messenger's content rule, allowing the mesh's
  machine names; the keeper rewords a summary that would be withheld and keeps
  it whole in the evidence; a TestMain lint fails the suite on any raised or
  linted finding that would be withheld.
This commit is contained in:
jochen
2026-10-06 18:40:33 +02:00
parent 7aa98e64ce
commit 8e8712e352
16 changed files with 1215 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/conditions/condition.go
+10 -1
View File
@@ -170,10 +170,19 @@ type Observation struct {
Also []string
Severity Severity
Summary string
// Said is this observation's evidence, in the mesh's words; Summary when empty.
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
// operator's channel, whose content rule withholds a message that carries any of them (ADR 0234
// §6), and names machines in words.
Said string
Source string
Resolver string
// Confirm says a single look can be wrong about this finding — a question over the network that
// went unanswered, a time measured once on a loaded machine. The keeper does not read it: the
// source that looks again does, and raises it only when the next look sees it too, or while it is
// already open (novox/hq issue 277).
Confirm bool
}
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
internal/conditions/sayable_test.go
+46
View File
@@ -0,0 +1,46 @@
package conditions
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/outward"
)
// **A summary the operator's channel would withhold is said in words, and kept whole in the evidence**
// (novox/hq issue 277, ADR 0234 §6): the condition that carried a resolver's address reached the
// operator as "its words are withheld" instead of the alert.
func TestASummaryCarryingAnAddressIsSaidInWordsAndKeptInTheEvidence(t *testing.T) {
k, _, _, _ := keeper(t)
before := Unsayable
var told []string
Unsayable = func(o Observation, field string, r outward.Refusal) { told = append(told, field+": "+r.What) }
t.Cleanup(func() { Unsayable = before })
raw := "AAAA for anchor.internal: no answer from 10.77.0.1: read udp 10.77.0.3:41234->10.77.0.1:53: i/o timeout"
c, err := k.Observe(t.Context(), Observation{Scope: ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong",
Kind: "resolver-wrong", Machine: "anchor", Severity: Urgent, Source: "D2",
Summary: "the mesh's resolver on anchor does not answer: " + raw})
if err != nil {
t.Fatal(err)
}
if r, ok := outward.Check(c.Summary, "anchor"); !ok {
t.Fatalf("the summary kept is still withheld (%s): %q", r, c.Summary)
}
if !strings.HasPrefix(c.Summary, "the mesh's resolver on anchor does not answer: ") {
t.Errorf("the summary's words were not kept: %q", c.Summary)
}
if !strings.Contains(c.Evidence[0].Said, raw) {
t.Errorf("the evidence lost what the summary carried: %q", c.Evidence[0].Said)
}
if len(told) != 1 || !strings.HasPrefix(told[0], "summary") {
t.Errorf("the producer was not named to the test: %v", told)
}
// A summary that may leave is kept as it is said, machine names and all.
ok, err := k.Observe(t.Context(), Observation{Scope: ScopeMachine, ID: "anchor", Kind: "silent", Machine: "anchor",
Severity: Warning, Source: "S1", Summary: "anchor has not been heard from since 12:00 UTC (bound 3m0s)"})
if err != nil || ok.Summary != "anchor has not been heard from since 12:00 UTC (bound 3m0s)" {
t.Fatalf("%q %v", ok.Summary, err)
}
}
internal/conditions/store.go
+52 -1
View File
@@ -8,6 +8,8 @@ import (
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/outward"
)
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
@@ -70,6 +72,8 @@ type Keeper struct {
closing sync.Once
// Unsaid counts the transitions given up on, for the self-check to say.
unsaid int
// reworded are the keys whose summary was said in words for the operator's channel, said once each.
reworded map[string]bool
}
type clearing struct {
@@ -105,7 +109,7 @@ var TellFor = 10 * time.Minute
func NewKeeper(ctx context.Context, o Options) *Keeper {
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
epoch: o.Epoch,
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
cleared: map[string]clearing{}, reworded: map[string]bool{}, out: make(chan Event, 1024), drained: make(chan struct{})}
if k.now == nil {
k.now = time.Now
}
@@ -170,6 +174,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if err := o.check(); err != nil {
return Condition{}, err
}
o = k.sayable(o)
key := o.Key()
for i := 0; i < tries; i++ {
now := k.now().UTC()
@@ -627,3 +632,49 @@ func orSelf(resolver string) string {
}
return resolver
}
// Unsayable is told of every observation whose words the operator's channel would withhold (novox/hq
// ADR 0234 §6): its summary or its key carries an address, a domain, a path or a secret's shape. The
// keeper says such a summary in words itself, and keeps what it said whole in the evidence; a test
// suite sets this to fail the producer, which is where the summary should have been said rightly.
var Unsayable func(o Observation, field string, r outward.Refusal)
// sayable is an observation whose summary may leave the mesh: **a summary says things in machine names
// and the mesh's words** (novox/hq issue 277). One that carries what the channel withholds — a raw error
// with an address in it, a path — is said in words here, as the last stand before the operator would
// read "its words are withheld" instead of the alert; what it carried goes to the evidence, which
// stays inside the mesh.
func (k *Keeper) sayable(o Observation) Observation {
machines := append([]string{o.Machine}, o.Also...)
if r, ok := outward.Check(o.Key(), machines...); !ok && Unsayable != nil {
Unsayable(o, "key", r)
}
r, ok := outward.Check(o.Summary, machines...)
if ok {
return o
}
if Unsayable != nil {
Unsayable(o, "summary", r)
}
whole := o.Summary
o.Summary = outward.Scrub(whole, fmt.Sprintf("a %s condition about %s %s: what it says is kept in its evidence "+
"(`conditions show`)", o.Kind, o.Scope, o.ID), machines...)
switch {
case o.Said == "":
o.Said = whole
case !strings.Contains(o.Said, whole):
o.Said += " — as raised: " + whole
}
k.mu.Lock()
if k.reworded == nil {
k.reworded = map[string]bool{}
}
first := !k.reworded[o.Key()]
k.reworded[o.Key()] = true
k.mu.Unlock()
if first {
k.say("the condition %s's summary carried %s, which the operator's channel withholds: said in words, "+
"and kept whole in its evidence — its source should say it so", o.Key(), r.What)
}
return o
}