mesh-catalog/modules/mesh-issues/cmd/mesh-issues/delivery_test.go

package main

import (
	"encoding/json"
	"strings"
	"sync"
	"testing"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

type notified struct {
	mu       sync.Mutex
	said     []string
	clear    []string
	resolved []string
	err      error
}

// fixedAtFixing names a fix while the issue stays at fixing, as an agent does when the fix merges (ADR 0281 §3,
// ADR 0284): a fix stuck is said only for an issue not closed (issue 547).
func fixedAtFixing(t *testing.T, h *harness, n uint64, fixedBy ...string) {
	t.Helper()
	h.fixing(t, n)
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", LocatedIn: []string{"mesh-catalog modules/records"},
		FixedBy: fixedBy}); err != nil {
		t.Fatal(err)
	}
}

func (n *notified) notify(key, severity, summary, headline, explanation, resolved string, clear bool) error {
	n.mu.Lock()
	defer n.mu.Unlock()
	if n.err != nil {
		return n.err
	}
	if clear {
		n.clear = append(n.clear, key)
		n.resolved = append(n.resolved, resolved)
	} else {
		n.said = append(n.said, key+" "+severity+" "+summary)
	}
	return nil
}

func transition(t *testing.T, h *harness, number int, to, why, mergedAs string, at time.Time) {
	t.Helper()
	raw, _ := json.Marshal(map[string]any{"repository": "novox/mesh-catalog", "number": number, "to": to, "why": why,
		"merged_as": mergedAs, "at": at})
	if err := h.OnTransition(stdio.Envelope{Key: "mesh-delivery.transition", Body: raw}); err != nil {
		t.Fatal(err)
	}
}

func countEvents(h *harness, event string) int {
	n := 0
	for _, e := range h.said() {
		if e.Event == event {
			n++
		}
	}
	return n
}

// The delivery line follows what the delivery says, as a measured fact: not merged, merged, held and why,
// delivered; an issue not yet resolved is said ready to verify once, and stays unresolved.
func TestAnIssueKnowsWhetherItsFixIsDelivered(t *testing.T) {
	h := newHarness(t)
	n := h.open(t, "the counter skips a number")
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"},
		FixedBy: []string{"mesh-catalog#9"}}); err != nil {
		t.Fatal(err)
	}
	line := func() string {
		v, err := h.View(n)
		if err != nil {
			t.Fatal(err)
		}
		return v.Delivery
	}
	if got := line(); got != "nothing heard from the delivery of mesh-catalog#9" {
		t.Fatal(got)
	}
	at := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
	transition(t, h, 9, "ready", "", "", at)
	if got := line(); got != "fix not merged: mesh-catalog#9" {
		t.Fatal(got)
	}
	transition(t, h, 9, "published", "", "abc", at)
	if got := line(); got != "fix merged, not delivered: mesh-catalog#9" {
		t.Fatal(got)
	}
	transition(t, h, 9, "held", "dnsmasq waits for a person's push on two machines", "abc", at)
	if got := line(); got != "held: mesh-catalog#9: dnsmasq waits for a person's push on two machines" {
		t.Fatal(got)
	}
	rows, _ := h.List(ListAsk{})
	if rows.Rows[0].Delivery != line() {
		t.Fatalf("%+v", rows.Rows[0])
	}
	for i := 0; i < 2; i++ {
		transition(t, h, 9, "delivered", "", "abc", at)
	}
	if got := line(); got != "delivered to every machine it should reach" {
		t.Fatal(got)
	}
	is := get(t, h, n)
	if is.Status != StatusLocated || countEvents(h, EventReadyToVerify) != 1 {
		t.Fatalf("%s, %d ready-to-verify", is.Status, countEvents(h, EventReadyToVerify))
	}
	var verify []string
	for _, nt := range is.Notes {
		if strings.HasPrefix(nt.Text, "Ready to verify:") {
			verify = append(verify, nt.Text)
		}
	}
	if len(verify) != 1 || !strings.Contains(verify[0], "Delivered is not resolved") {
		t.Fatalf("%q", verify)
	}
	// Covered by a later merge counts as delivered, and says so; a resolved issue is not said ready to verify.
	h2 := newHarness(t)
	m := h2.open(t, "x")
	resolve(t, h2, m, "mesh-catalog#9")
	transition(t, h2, 9, "covered", "", "abc", at)
	if v, _ := h2.View(m); !strings.HasPrefix(v.Delivery, "delivered to every machine it should reach (by a later merge") ||
		countEvents(h2, EventReadyToVerify) != 0 {
		t.Fatalf("%q", v.Delivery)
	}
}

// A fix merged and not delivered past the bound is said once on the issue and the operator's channel, and
// cleared there when it is delivered; one that fails to be said is said at the next sweep.
func TestAStuckFixIsSaidOnceAndClearedWhenDelivered(t *testing.T) {
	h := newHarness(t)
	n := h.open(t, "the counter skips a number")
	fixedAtFixing(t, h, n, "mesh-catalog#9")
	note := &notified{}
	h.Notify = note.notify
	merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) // three hours before the harness's clock
	transition(t, h, 9, "held", "waits for a person", "abc", merged)
	note.err = errStuck
	if said, err := h.Sweep(); err != nil || said != 0 {
		t.Fatalf("%d %v", said, err)
	}
	note.err = nil
	for i := 0; i < 2; i++ {
		if _, err := h.Sweep(); err != nil {
			t.Fatal(err)
		}
	}
	stuck := 0
	for _, nt := range get(t, h, n).Notes {
		if strings.HasPrefix(nt.Text, "Stuck: the fix mesh-catalog#9") {
			stuck++
		}
	}
	if stuck != 1 || len(note.said) != 1 || !strings.HasPrefix(note.said[0], "issue-tracker.fix-stuck.365 warning") {
		t.Fatalf("%d notes, said %q", stuck, note.said)
	}
	transition(t, h, 9, "delivered", "", "abc", merged)
	if len(note.clear) != 1 {
		t.Fatalf("not cleared: %q", note.clear)
	}
	// Under the bound nothing is said.
	h2 := newHarness(t)
	m := h2.open(t, "y")
	fixedAtFixing(t, h2, m, "mesh-catalog#9")
	n2 := &notified{}
	h2.Notify = n2.notify
	transition(t, h2, 9, "delivering", "", "abc", time.Date(2026, 10, 10, 11, 30, 0, 0, time.UTC))
	if said, _ := h2.Sweep(); said != 0 || len(n2.said) != 0 {
		t.Fatal("said stuck under the bound")
	}
	h2.settings.StuckAfter = "10m"
	if said, _ := h2.Sweep(); said != 1 {
		t.Fatal("the setting's bound was not read")
	}
}

var errStuck = stdioErr("the operator's channel did not answer")

type stdioErr string

func (e stdioErr) Error() string { return string(e) }

// A fix is named while the issue stays located: a move to the same status changes only what it holds.
func TestAFixIsNamedWhileTheIssueStaysLocated(t *testing.T) {
	h := newHarness(t)
	n := h.open(t, "x")
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"}}); err != nil {
		t.Fatal(err)
	}
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated}); err == nil {
		t.Fatal("a move that changes nothing was taken")
	}
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusLocated, FixedBy: []string{"mesh-catalog#9"}}); err != nil {
		t.Fatal(err)
	}
	if is := get(t, h, n); is.Status != StatusLocated || is.Closed != nil || len(is.FixedBy) != 1 {
		t.Fatalf("%+v", is)
	}
}

// Transitions out of order never take a delivered fix back, and a delivered fix is never said stuck; a clear
// that fails is tried again; a fix no longer named is not swept; an empty answer from the delivery is asked again.
func TestDeliveredStaysDeliveredAndClearsAreRetried(t *testing.T) {
	h := newHarness(t)
	n := h.open(t, "x")
	fixedAtFixing(t, h, n, "mesh-catalog#9")
	h.settings.DeliveredNotResolvedAfter = "24h" // only the stuck warning is under test here
	note := &notified{}
	h.Notify = note.notify
	merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC)
	transition(t, h, 9, "held", "waits", "abc", merged)
	if said, _ := h.Sweep(); said != 1 {
		t.Fatal("not said stuck")
	}
	note.err = errStuck
	transition(t, h, 9, "delivered", "", "abc", merged.Add(time.Hour))
	transition(t, h, 9, "held", "an older word", "abc", merged.Add(30*time.Minute)) // out of order
	if v, _ := h.View(n); v.Delivery != "delivered to every machine it should reach" {
		t.Fatal(v.Delivery)
	}
	if len(note.clear) != 0 || !get(t, h, n).StuckOpen {
		t.Fatal("cleared while the channel failed")
	}
	note.err = nil
	if said, _ := h.Sweep(); said != 0 || len(note.clear) != 1 || get(t, h, n).StuckOpen {
		t.Fatalf("said %d, cleared %q", said, note.clear)
	}
	// A fix no longer named is not swept, and its warning is cleared.
	h2 := newHarness(t)
	m := h2.open(t, "y")
	if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, LocatedIn: []string{"mesh-catalog modules/records"}, FixedBy: []string{"mesh-catalog#9"}}); err != nil {
		t.Fatal(err)
	}
	n2 := &notified{}
	h2.Notify = n2.notify
	transition(t, h2, 9, "held", "waits", "abc", merged)
	_, _ = h2.Sweep()
	if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, FixedBy: []string{"mesh-catalog#10"}}); err != nil {
		t.Fatal(err)
	}
	if said, _ := h2.Sweep(); said != 0 || len(n2.said) != 1 || len(n2.clear) != 1 {
		t.Fatalf("said %d %q, cleared %q", said, n2.said, n2.clear)
	}
	if _, err := h2.Move(MoveAsk{Number: m, Status: StatusLocated, FixedBy: []string{"mesh-catalog#10"}}); err == nil {
		t.Fatal("a move that changes nothing was taken")
	}
	// The delivery knows nothing yet: not kept, asked again at the next move.
	asks := 0
	h2.Deliveries = func(string, string) (Delivery, error) { asks++; return Delivery{}, nil }
	for i := 0; i < 2; i++ {
		_ = h2.OnMoved(stdio.Envelope{Body: json.RawMessage(`{"number":365}`)})
	}
	if v, _ := h2.View(m); asks != 2 || v.Delivery != "nothing heard from the delivery of mesh-catalog#10" {
		t.Fatalf("%d asks, %q", asks, v.Delivery)
	}
}

// A resolved issue is the end of its stuck warning, as wontfix is: resolving clears it on the operator's channel,
// saying so, and the sweep never says a fix stuck for an issue resolved or wontfix, even one whose delivery failed
// and never reads delivered (issue 547).
func TestAResolvedIssueEndsItsStuckWarningAndIsNeverSaidStuck(t *testing.T) {
	merged := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) // three hours before the harness's clock
	moved := func(t *testing.T, h *harness, n uint64) {
		t.Helper()
		raw, _ := json.Marshal(map[string]any{"number": n})
		if err := h.OnMoved(stdio.Envelope{Body: raw}); err != nil {
			t.Fatal(err)
		}
	}
	for _, to := range []string{StatusResolved, StatusWontfix} {
		t.Run(to, func(t *testing.T) {
			h := newHarness(t)
			n := h.open(t, "the counter skips a number")
			fixedAtFixing(t, h, n, "mesh-catalog#9")
			note := &notified{}
			h.Notify = note.notify
			transition(t, h, 9, "failed", "the gate failed on one machine", "abc", merged)
			if said, err := h.Sweep(); err != nil || said != 1 || len(note.said) != 1 {
				t.Fatalf("not said stuck: %d %v %q", said, err, note.said)
			}
			ask := MoveAsk{Number: n, Status: to, FixedBy: []string{"mesh-catalog#9"}, ReplayNone: "proven live"}
			if to == StatusWontfix {
				ask = MoveAsk{Number: n, Status: to, Reason: "not worth it"}
			}
			if _, err := h.Move(ask); err != nil {
				t.Fatal(err)
			}
			moved(t, h, n)
			if len(note.clear) != 1 || note.clear[0] != stuckKey(n) || get(t, h, n).StuckOpen {
				t.Fatalf("not cleared at the move: %q", note.clear)
			}
			if to == StatusResolved && !strings.HasPrefix(note.resolved[0], "resolved: ") {
				t.Fatalf("%q", note.resolved)
			}
			if said, _ := h.Sweep(); said != 0 || len(note.said) != 1 || len(note.clear) != 1 {
				t.Fatalf("said again after %s: %d %q, cleared %q", to, said, note.said, note.clear)
			}
		})
	}
	// A warning left open by a tracker that kept it for a resolved issue is cleared at the first sweep, and an
	// issue resolved before its fix was ever said stuck is never said stuck.
	h := newHarness(t)
	n := h.open(t, "x")
	fixedAtFixing(t, h, n, "mesh-catalog#9")
	note := &notified{err: errStuck}
	h.Notify = note.notify
	transition(t, h, 9, "failed", "the gate failed", "abc", merged)
	if _, err := h.change(n, Act{Op: "warned", By: Seat}, func(i *Issue) error {
		i.Fixes[0].StuckSaid, i.StuckOpen = true, true
		return nil
	}); err != nil {
		t.Fatal(err)
	}
	if _, err := h.Move(MoveAsk{Number: n, Status: StatusResolved, FixedBy: []string{"mesh-catalog#9"}}); err != nil {
		t.Fatal(err)
	}
	moved(t, h, n) // the channel does not answer: kept open, tried at the sweep
	note.err = nil
	if said, _ := h.Sweep(); said != 0 || len(note.clear) != 1 || get(t, h, n).StuckOpen {
		t.Fatalf("said %d, cleared %q", said, note.clear)
	}
	m := h.open(t, "y")
	fixedAtFixing(t, h, m, "mesh-catalog#10")
	transition(t, h, 10, "failed", "the gate failed", "abc", merged)
	if _, err := h.Move(MoveAsk{Number: m, Status: StatusResolved, FixedBy: []string{"mesh-catalog#10"}}); err != nil {
		t.Fatal(err)
	}
	if said, _ := h.Sweep(); said != 0 || len(note.said) != 0 {
		t.Fatalf("a resolved issue said stuck: %q", note.said)
	}
}