package main import ( "encoding/json" "strings" "sync" "testing" "time" stdio "git.novox.be/novox/mesh-sdk/go" ) type notified struct { mu sync.Mutex said []string clear []string resolved []string err error } // fixedAtFixing names a fix while the issue stays at fixing, as an agent does when the fix merges (ADR 0281 ยง3, // ADR 0284): a fix stuck is said only for an issue not closed (issue 547). func fixedAtFixing(t *testing.T, h *harness, n uint64, fixedBy ...string) { t.Helper() h.fixing(t, n) if _, err := h.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", LocatedIn: []string{"mesh-catalog modules/records"}, FixedBy: fixedBy}); err != nil { t.Fatal(err) } } func (n *notified) notify(key, severity, summary, headline, explanation, resolved string, clear bool) error { n.mu.Lock() defer n.mu.Unlock() if n.err != nil { return n.err } if clear { n.clear = append(n.clear, key) n.resolved = append(n.resolved, resolved) } else { n.said = append(n.said, key+" "+severity+" "+summary) } return nil } func transition(t *testing.T, h *harness, number int, to, why, mergedAs string, at time.Time) { t.Helper() raw, _ := json.Marshal(map[string]any{"repository": "novox/mesh-catalog", "number": number, "to": to, "why": why, "merged_as": mergedAs, "at": at}) if err := h.OnTransition(stdio.Envelope{Key: "mesh-delivery.transition", Body: raw}); err != nil { t.Fatal(err) } } func countEvents(h *harness, event string) int { n := 0 for _, e := range h.said() { if e.Event == event { n++ } } return n } // The delivery line follows what the delivery says, as a measured fact: not merged, merged, held and why, // delivered; an issue not yet resolved is said ready to verify once, and stays unresolved. func TestAnIssueKnowsWhetherItsFixIsDelivered(t *testing.T) { h := newHarness(t) n := h.open(t, "the counter skips a number") if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"}, FixedBy: []string{"mesh-catalog#9"}}); err != nil { t.Fatal(err) } line := func() string { v, err := h.View(n) if err != nil { t.Fatal(err) } return v.Delivery } if got := line(); got != "nothing heard from the delivery of mesh-catalog#9" { t.Fatal(got) } at := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) transition(t, h, 9, "ready", "", "", at) if got := line(); got != "fix not merged: mesh-catalog#9" { t.Fatal(got) } transition(t, h, 9, "published", "", "abc", at) if got := line(); got != "fix merged, not delivered: mesh-catalog#9" { t.Fatal(got) } transition(t, h, 9, "held", "dnsmasq waits for a person's push on two machines", "abc", at) if got := line(); got != "held: mesh-catalog#9: dnsmasq waits for a person's push on two machines" { t.Fatal(got) } rows, _ := h.List(ListAsk{}) if rows.Rows[0].Delivery != line() { t.Fatalf("%+v", rows.Rows[0]) } for i := 0; i < 2; i++ { transition(t, h, 9, "delivered", "", "abc", at) } if got := line(); got != "delivered to every machine it should reach" { t.Fatal(got) } is := get(t, h, n) if is.Status != StatusLocated || countEvents(h, EventReadyToVerify) != 1 { t.Fatalf("%s, %d ready-to-verify", is.Status, countEvents(h, EventReadyToVerify)) } var verify []string for _, nt := range is.Notes { if strings.HasPrefix(nt.Text, "Ready to verify:") { verify = append(verify, nt.Text) } } if len(verify) != 1 || !strings.Contains(verify[0], "Delivered is not resolved") { t.Fatalf("%q", verify) } // Covered by a later merge counts as delivered, and says so; a resolved issue is not said ready to verify. h2 := newHarness(t) m := h2.open(t, "x") resolve(t, h2, m, "mesh-catalog#9") transition(t, h2, 9, "covered", "", "abc", at) if v, _ := h2.View(m); !strings.HasPrefix(v.Delivery, "delivered to every machine it should reach (by a later merge") || countEvents(h2, EventReadyToVerify) != 0 { t.Fatalf("%q", v.Delivery) } } // A fix merged and not delivered past the bound is said once on the issue and the operator's channel, and // cleared there when it is delivered; one that fails to be said is said at the next sweep. func TestAStuckFixIsSaidOnceAndClearedWhenDelivered(t *testing.T) { h := newHarness(t) n := h.open(t, "the counter skips a number") fixedAtFixing(t, h, n, "mesh-catalog#9") note := ¬ified{} h.Notify = note.notify merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) // three hours before the harness's clock transition(t, h, 9, "held", "waits for a person", "abc", merged) note.err = errStuck if said, err := h.Sweep(); err != nil || said != 0 { t.Fatalf("%d %v", said, err) } note.err = nil for i := 0; i < 2; i++ { if _, err := h.Sweep(); err != nil { t.Fatal(err) } } stuck := 0 for _, nt := range get(t, h, n).Notes { if strings.HasPrefix(nt.Text, "Stuck: the fix mesh-catalog#9") { stuck++ } } if stuck != 1 || len(note.said) != 1 || !strings.HasPrefix(note.said[0], "issue-tracker.fix-stuck.365 warning") { t.Fatalf("%d notes, said %q", stuck, note.said) } transition(t, h, 9, "delivered", "", "abc", merged) if len(note.clear) != 1 { t.Fatalf("not cleared: %q", note.clear) } // Under the bound nothing is said. h2 := newHarness(t) m := h2.open(t, "y") fixedAtFixing(t, h2, m, "mesh-catalog#9") n2 := ¬ified{} h2.Notify = n2.notify transition(t, h2, 9, "delivering", "", "abc", time.Date(2026, 10, 10, 11, 30, 0, 0, time.UTC)) if said, _ := h2.Sweep(); said != 0 || len(n2.said) != 0 { t.Fatal("said stuck under the bound") } h2.settings.StuckAfter = "10m" if said, _ := h2.Sweep(); said != 1 { t.Fatal("the setting's bound was not read") } } var errStuck = stdioErr("the operator's channel did not answer") type stdioErr string func (e stdioErr) Error() string { return string(e) } // A fix is named while the issue stays located: a move to the same status changes only what it holds. func TestAFixIsNamedWhileTheIssueStaysLocated(t *testing.T) { h := newHarness(t) n := h.open(t, "x") if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"}}); err != nil { t.Fatal(err) } if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated}); err == nil { t.Fatal("a move that changes nothing was taken") } if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, FixedBy: []string{"mesh-catalog#9"}}); err != nil { t.Fatal(err) } if is := get(t, h, n); is.Status != StatusLocated || is.Closed != nil || len(is.FixedBy) != 1 { t.Fatalf("%+v", is) } } // Transitions out of order never take a delivered fix back, and a delivered fix is never said stuck; a clear // that fails is tried again; a fix no longer named is not swept; an empty answer from the delivery is asked again. func TestDeliveredStaysDeliveredAndClearsAreRetried(t *testing.T) { h := newHarness(t) n := h.open(t, "x") fixedAtFixing(t, h, n, "mesh-catalog#9") h.settings.DeliveredNotResolvedAfter = "24h" // only the stuck warning is under test here note := ¬ified{} h.Notify = note.notify merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) transition(t, h, 9, "held", "waits", "abc", merged) if said, _ := h.Sweep(); said != 1 { t.Fatal("not said stuck") } note.err = errStuck transition(t, h, 9, "delivered", "", "abc", merged.Add(time.Hour)) transition(t, h, 9, "held", "an older word", "abc", merged.Add(30*time.Minute)) // out of order if v, _ := h.View(n); v.Delivery != "delivered to every machine it should reach" { t.Fatal(v.Delivery) } if len(note.clear) != 0 || !get(t, h, n).StuckOpen { t.Fatal("cleared while the channel failed") } note.err = nil if said, _ := h.Sweep(); said != 0 || len(note.clear) != 1 || get(t, h, n).StuckOpen { t.Fatalf("said %d, cleared %q", said, note.clear) } // A fix no longer named is not swept, and its warning is cleared. h2 := newHarness(t) m := h2.open(t, "y") if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"}, FixedBy: []string{"mesh-catalog#9"}}); err != nil { t.Fatal(err) } n2 := ¬ified{} h2.Notify = n2.notify transition(t, h2, 9, "held", "waits", "abc", merged) _, _ = h2.Sweep() if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, FixedBy: []string{"mesh-catalog#10"}}); err != nil { t.Fatal(err) } if said, _ := h2.Sweep(); said != 0 || len(n2.said) != 1 || len(n2.clear) != 1 { t.Fatalf("said %d %q, cleared %q", said, n2.said, n2.clear) } if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, FixedBy: []string{"mesh-catalog#10"}}); err == nil { t.Fatal("a move that changes nothing was taken") } // The delivery knows nothing yet: not kept, asked again at the next move. asks := 0 h2.Deliveries = func(string, string) (Delivery, error) { asks++; return Delivery{}, nil } for i := 0; i < 2; i++ { _ = h2.OnMoved(stdio.Envelope{Body: json.RawMessage(`{"number":365}`)}) } if v, _ := h2.View(m); asks != 2 || v.Delivery != "nothing heard from the delivery of mesh-catalog#10" { t.Fatalf("%d asks, %q", asks, v.Delivery) } } // A resolved issue is the end of its stuck warning, as wontfix is: resolving clears it on the operator's channel, // saying so, and the sweep never says a fix stuck for an issue resolved or wontfix, even one whose delivery failed // and never reads delivered (issue 547). func TestAResolvedIssueEndsItsStuckWarningAndIsNeverSaidStuck(t *testing.T) { merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) // three hours before the harness's clock moved := func(t *testing.T, h *harness, n uint64) { t.Helper() raw, _ := json.Marshal(map[string]any{"number": n}) if err := h.OnMoved(stdio.Envelope{Body: raw}); err != nil { t.Fatal(err) } } for _, to := range []string{StatusResolved, StatusWontfix} { t.Run(to, func(t *testing.T) { h := newHarness(t) n := h.open(t, "the counter skips a number") fixedAtFixing(t, h, n, "mesh-catalog#9") note := ¬ified{} h.Notify = note.notify transition(t, h, 9, "failed", "the gate failed on one machine", "abc", merged) if said, err := h.Sweep(); err != nil || said != 1 || len(note.said) != 1 { t.Fatalf("not said stuck: %d %v %q", said, err, note.said) } ask := MoveAsk{Number: n, Status: to, FixedBy: []string{"mesh-catalog#9"}, ReplayNone: "proven live"} if to == StatusWontfix { ask = MoveAsk{Number: n, Status: to, Reason: "not worth it"} } if _, err := h.Move(ask); err != nil { t.Fatal(err) } moved(t, h, n) if len(note.clear) != 1 || note.clear[0] != stuckKey(n) || get(t, h, n).StuckOpen { t.Fatalf("not cleared at the move: %q", note.clear) } if to == StatusResolved && !strings.HasPrefix(note.resolved[0], "resolved: ") { t.Fatalf("%q", note.resolved) } if said, _ := h.Sweep(); said != 0 || len(note.said) != 1 || len(note.clear) != 1 { t.Fatalf("said again after %s: %d %q, cleared %q", to, said, note.said, note.clear) } }) } // A warning left open by a tracker that kept it for a resolved issue is cleared at the first sweep, and an // issue resolved before its fix was ever said stuck is never said stuck. h := newHarness(t) n := h.open(t, "x") fixedAtFixing(t, h, n, "mesh-catalog#9") note := ¬ified{err: errStuck} h.Notify = note.notify transition(t, h, 9, "failed", "the gate failed", "abc", merged) if _, err := h.change(n, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.Fixes[0].StuckSaid, i.StuckOpen = true, true return nil }); err != nil { t.Fatal(err) } if _, err := h.Move(MoveAsk{Number: n, Status: StatusResolved, FixedBy: []string{"mesh-catalog#9"}}); err != nil { t.Fatal(err) } moved(t, h, n) // the channel does not answer: kept open, tried at the sweep note.err = nil if said, _ := h.Sweep(); said != 0 || len(note.clear) != 1 || get(t, h, n).StuckOpen { t.Fatalf("said %d, cleared %q", said, note.clear) } m := h.open(t, "y") fixedAtFixing(t, h, m, "mesh-catalog#10") transition(t, h, 10, "failed", "the gate failed", "abc", merged) if _, err := h.Move(MoveAsk{Number: m, Status: StatusResolved, FixedBy: []string{"mesh-catalog#10"}}); err != nil { t.Fatal(err) } if said, _ := h.Sweep(); said != 0 || len(note.said) != 0 { t.Fatalf("a resolved issue said stuck: %q", note.said) } }