mesh-controller/internal/inventory/busy_test.go

package inventory

import (
	"strings"
	"testing"
	"time"
)

// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the
// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the
// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said
// the container replaced.
func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) {
	raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`,
		`"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z",
		  "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
		    "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1)
	w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
	sumsUp(t, w)
	if w.TotalMS != 282894 {
		t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS)
	}
	busy := phase(w, PhaseBusy, 0)
	if busy.State != PhaseMeasured || busy.TookMS != 60000 {
		t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy)
	}
	if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 {
		t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged)
	}
	var names []string
	for _, ph := range w.Phases {
		names = append(names, ph.Name)
	}
	if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) {
		t.Fatalf("the busy wait is not between the send and the judging: %v", names)
	}
}

// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no
// such phase.
func TestAnOpenBusyWaitIsSaid(t *testing.T) {
	raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
      "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`,
		`"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",
		  "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
		    "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1)
	raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1)
	p := walkOf(t, raw)
	if !p.Modules["mesh-delivery"].Gate.Waiting() {
		t.Fatal("a gate whose busy wait has not ended does not say it waits")
	}
	w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil)
	busy := phase(w, PhaseBusy, 0)
	if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") {
		t.Fatalf("an open busy wait is not said: %+v", busy)
	}
	if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" {
		t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases)
	}
}

// A container a report lists waiting is its module's by its id; its output alone is not a new account.
func TestABusyContainerIsItsModules(t *testing.T) {
	list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"},
		{ID: "web.server", State: "replaced-when-idle"}}
	if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" {
		t.Fatalf("build-agent's container is not found waiting: %+v", b)
	}
	if _, ok := WaitingFor(list, "web"); ok {
		t.Error("a container replaced reads as waiting")
	}
	if _, ok := WaitingFor(list, "build"); ok {
		t.Error("another module's container is found by a prefix of its name")
	}
	again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]}
	if !sameBusy(list, again) {
		t.Error("a wait whose output changed is a new account")
	}
	if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() {
		t.Error("a wait that ended is not a new account")
	}
}

// A machine's report of a busy container is kept with its last account and read back beside it, and its
// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store.
func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) {
	inv, node := aNodeWithModules(t)
	ctx := t.Context()
	record, err := inv.NodeByName(ctx, node)
	if err != nil {
		t.Fatal(err)
	}
	since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting,
		Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}}
	keep := func(busy []BusyWait) bool {
		t.Helper()
		news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1,
			Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false })
		if err != nil {
			t.Fatal(err)
		}
		return news
	}
	read := func() []BusyWait {
		t.Helper()
		reports, err := inv.LastReports(ctx)
		if err != nil {
			t.Fatal(err)
		}
		for _, r := range reports {
			if r.Node == node {
				return r.Busy
			}
		}
		t.Fatal("the machine's report is not read back")
		return nil
	}
	keep(nil)
	if got := read(); len(got) != 0 {
		t.Fatalf("a report with nothing busy reads back %+v", got)
	}
	if !keep(waiting) {
		t.Error("a container starting to wait is not news")
	}
	got := read()
	if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" {
		t.Fatalf("the wait is not read back: %+v", got)
	}
	again := []BusyWait{waiting[0]}
	again[0].Said = "busy: a check, a minute on"
	if keep(again) {
		t.Error("a wait whose output changed is news")
	}
	if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle",
		Since: since, Until: since.Add(15 * time.Minute)}}) {
		t.Error("a container replaced after it waited is not news")
	}
	if _, ok := WaitingFor(read(), "build-agent"); ok {
		t.Error("a container replaced still reads as waiting")
	}
}

// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept
// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335).
func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) {
	c := walk582(t)
	shanks := c.walk.Modules["build-agent"].Rest["shanks"]
	ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000")
	answered := shanks.At.Add(15 * time.Second)
	// shanks answered, and said a container waits: its apply row is the answer, marked busy.
	c.rows["shanks"][0].Detail = ApplyBusy
	at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send
	w := c.walk.WalkPhases(at, c.lookup)
	var got RestReport
	for _, r := range w.Rest {
		if r.Node == "shanks" {
			got = r
		}
	}
	if got.State != RestBusy || slicesContains(w.Silent, "shanks") {
		t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent)
	}
	if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") {
		t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1))
	}

	// 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then.
	c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref,
		Started: answered, Took: 17 * time.Minute})
	w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
	sumsUp(t, w)
	for _, r := range w.Rest {
		if r.Node == "shanks" {
			got = r
		}
	}
	if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) {
		t.Fatalf("a machine replaced after its wait reads %+v", got)
	}
	if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) {
		t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End)
	}
	// And the apply time itself is the answer's, never the wait's.
	if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute {
		t.Fatalf("the busy wait is not kept apart from the apply: %+v", r)
	}
}

func slicesContains(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}