package main import ( "context" "sync" "github.com/novox/mesh-controller/internal/conditions" ) // **A finding one look can be wrong about is raised on the second look in a row** (novox/hq issue 277). // // D2 raised its resolver urgent on one unanswered question, asked once, while the resolver's machine // was loaded by a push and a build starting; thirty questions right after were answered at once. A // single sample of something that is answered over the network, or timed on a machine under load, // is not the invariant failing. So a source marks such a finding `Confirm`, and this holds it back: // // - raised when the source's previous look saw it too — two runs of the self-check in a row (five // minutes apart), or two ticks of the watchdogs (half a minute) — at the severity the source says; // - kept while it is already open, however it is seen, so a condition the next look still sees is // never cleared and raised again (flapping is not news; a reopening within ReopenWithin still is); // - cleared, as everything is, by the look that no longer sees it. // // A finding held back is not a pass: the self-check's verdict names it as unconfirmed. A finding that is // a definite answer — a resolver that answered wrongly, a stream that is not there — is not marked, and // is raised at once. type confirming struct { mu sync.Mutex // last is, by source, the keys of the Confirm findings its previous look saw. last map[string]map[string]bool } // pass splits one source's findings into what is raised now and what is held for the next look, and // remembers what it saw. An open condition that cannot be read is kept: unknown is not a reason to // hold a finding back. func (c *confirming) pass(ctx context.Context, keeper *conditions.Keeper, source string, found []conditions.Observation) (raise, held []conditions.Observation) { c.mu.Lock() defer c.mu.Unlock() if c.last == nil { c.last = map[string]map[string]bool{} } before, seen := c.last[source], map[string]bool{} for _, o := range found { if !o.Confirm { raise = append(raise, o) continue } key := o.Key() seen[key] = true if before[key] || isOpen(ctx, keeper, key) { raise = append(raise, o) continue } held = append(held, o) } c.last[source] = seen return raise, held } // isOpen says whether a condition is open; one that cannot be read is taken as open. func isOpen(ctx context.Context, keeper *conditions.Keeper, key string) bool { if keeper == nil { return false } _, open, err := keeper.Get(ctx, key) return open || err != nil }