jschoubben
cfbbad8209
The give-up time is not newer for every letter kept: a notice that could not be kept is offered again a minute later, so a later give-up can be kept first and the one after it read as not fresh. The stored time rises with the stream's sequence. A consumer whose letters vanish between the two reads is left out of the look instead of counted as a look, and the skip of a token-less max-deliveries advisory now has a test of its own.
181 lines
8.5 KiB
Go
181 lines
8.5 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// **A refusal the bus said once is counted once** (novox/hq issue 402). The bus refused the controller
|
|
// one publish at 18:52:59 UTC on 2026-10-10 and never again, yet S9 looked at the one remembered
|
|
// advisory every half minute for the hour it is kept, and the condition said "observed 15 time(s), last
|
|
// 13s ago" with a refusal in its evidence every 30 seconds: two of us read it as a refusal going on and
|
|
// went looking for a missing grant. The condition counts what the bus said, with when it last said it,
|
|
// never how often the controller looked.
|
|
func TestARefusalSaidOnceIsCountedOnceHoweverOftenItIsLookedAt(t *testing.T) {
|
|
refused := time.Date(2026, 10, 10, 18, 52, 59, 0, time.UTC)
|
|
now := refused.Add(10 * time.Second)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
t.Cleanup(func() { k.Close(t.Context()) })
|
|
|
|
// The words the live condition bus.controller.refused carried on 2026-10-10.
|
|
said := "the bus refused the controller: nats: permissions violation: Permissions Violation for " +
|
|
`Publish to "mesh.seat.mesh-delivery.tool.times"`
|
|
advisory := link.Advisory{Kind: link.AdvisoryRefused, ID: "controller", Said: said,
|
|
First: refused, Last: refused, Count: 1}
|
|
look := func() conditions.Condition {
|
|
t.Helper()
|
|
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory}}
|
|
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, found, err := conditions.ReadOne(t.Context(), store, "bus.controller.refused")
|
|
if err != nil || !found {
|
|
t.Fatalf("bus.controller.refused: found %v, %v", found, err)
|
|
}
|
|
return c
|
|
}
|
|
|
|
var c conditions.Condition
|
|
for range 15 {
|
|
c = look()
|
|
now = now.Add(30 * time.Second)
|
|
}
|
|
if c.Observations != 1 || len(c.Evidence) != 1 {
|
|
t.Fatalf("one refusal, looked at 15 times, reads as observed %d time(s) with %d evidence lines: %+v",
|
|
c.Observations, len(c.Evidence), c.Evidence)
|
|
}
|
|
if !c.LastObserved.Equal(refused) || !c.Evidence[0].At.Equal(refused) {
|
|
t.Fatalf("last observed %s, evidence at %s: the refusal was at %s", c.LastObserved, c.Evidence[0].At, refused)
|
|
}
|
|
|
|
// The bus refuses it three times more between two looks: the condition counts four refusals, says
|
|
// the newest, and adds one line of evidence for the look that saw them.
|
|
again := now.Add(-5 * time.Second)
|
|
advisory.Last, advisory.Count = again, 4
|
|
c = look()
|
|
if c.Observations != 4 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) || !c.Evidence[0].At.Equal(again) {
|
|
t.Fatalf("four refusals read as observed %d time(s), last %s, evidence %+v", c.Observations, c.LastObserved, c.Evidence)
|
|
}
|
|
if !strings.Contains(c.Summary, "(4 times since 18:52 UTC)") {
|
|
t.Fatalf("summary %q", c.Summary)
|
|
}
|
|
now = now.Add(30 * time.Second)
|
|
if c = look(); c.Observations != 4 || len(c.Evidence) != 2 {
|
|
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
}
|
|
}
|
|
|
|
// **A condition that looks at the present still counts its looks** (novox/hq issue 402 changes only
|
|
// what reads a record): a machine silent for three looks was observed three times, and says so.
|
|
func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
|
|
now := time.Date(2026, 10, 10, 19, 0, 0, 0, time.UTC)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
t.Cleanup(func() { k.Close(t.Context()) })
|
|
var c conditions.Condition
|
|
for range 3 {
|
|
var err error
|
|
if c, err = k.Observe(t.Context(), conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace",
|
|
Kind: "silent", Machine: "ace", Severity: conditions.Warning, Summary: "ace has not been heard from",
|
|
Source: "S1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(30 * time.Second)
|
|
}
|
|
if c.Observations != 3 || len(c.Evidence) != 3 || !c.LastObserved.Equal(now.Add(-30*time.Second)) {
|
|
t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
|
|
}
|
|
}
|
|
|
|
// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
|
|
// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
|
|
// without a token names the same key; read beside it, each look added an observation and a line of
|
|
// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
|
|
// dead-letter row alone says that key, and counts the messages given up on, never the looks.
|
|
func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
|
|
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
now := gaveUp.Add(20 * time.Second)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
t.Cleanup(func() { k.Close(t.Context()) })
|
|
|
|
const key = "bus.EVENTS.media_sonarr.max-deliveries"
|
|
held := map[string]int{"EVENTS.media_sonarr": 1}
|
|
newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
|
|
advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
|
|
Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
|
|
First: gaveUp, Last: gaveUp, Count: 1}
|
|
look := func() conditions.Condition {
|
|
t.Helper()
|
|
f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
|
|
deadLetters: held, deadLettersNewest: newest}
|
|
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
|
if err != nil || !found {
|
|
t.Fatalf("%s: found %v, %v", key, found, err)
|
|
}
|
|
if !strings.Contains(c.Summary, "dead-letters verb") {
|
|
t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
|
|
}
|
|
return c
|
|
}
|
|
|
|
var c conditions.Condition
|
|
for range 5 {
|
|
c = look()
|
|
now = now.Add(30 * time.Second)
|
|
}
|
|
if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
|
|
t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
|
|
"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
|
|
}
|
|
|
|
// The consumer gives up on a second message between two looks: two given up on, one more line.
|
|
again := now.Add(-10 * time.Second)
|
|
held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
|
|
c = look()
|
|
if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
|
|
t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
|
|
c.LastObserved, c.Evidence)
|
|
}
|
|
now = now.Add(30 * time.Second)
|
|
if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
|
|
t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
|
|
}
|
|
}
|
|
|
|
// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
|
|
// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
|
|
func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
|
|
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
|
|
Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
|
|
said := watchAdvisories(f)
|
|
if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
|
|
t.Fatalf("%+v", said)
|
|
}
|
|
}
|
|
|
|
// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
|
|
// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
|
|
func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
|
|
at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
|
|
f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
|
|
ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
|
|
Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
|
|
if said := watchAdvisories(f); len(said) != 0 {
|
|
t.Fatalf("said %+v", said)
|
|
}
|
|
}
|