mesh-controller/cmd/mesh-controller/busy_test.go

package main

import (
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
)

// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
	sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
		Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
	pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
	reported := sent.Add(20 * time.Second)
	waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
		State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
		Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
	facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
		// As judgeMoves hands it: the report held against the gate's own send.
		return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
			Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
	}

	now := reported.Add(time.Minute)
	if !busyWait(g, pairs, facts(reported, waiting), now) {
		t.Fatal("a gate whose module's container waits as busy judged it")
	}
	if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
		t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
	}
	for _, said := range []string{gateLine(g), g.Last} {
		if !strings.Contains(said, "waits for a busy container on novox") {
			t.Errorf("the walk does not say it waits for a busy container there: %s", said)
		}
	}
	if !strings.Contains(g.Last, "check of #280") {
		t.Errorf("the wait does not say what the container holds: %s", g.Last)
	}

	// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
	now = reported.Add(10 * time.Minute)
	if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
		t.Fatalf("the gate stopped waiting within the bound: %+v", g)
	}

	// The machine reports it replaced: the judging starts at that report.
	replacedAt := reported.Add(12 * time.Minute)
	now = replacedAt.Add(5 * time.Second)
	replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
		State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
	if busyWait(g, pairs, facts(replacedAt, replaced), now) {
		t.Fatal("the gate still waits after the container was replaced")
	}
	if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
		t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
	}
	if strings.Contains(gateLine(g), "waits for a busy container") {
		t.Errorf("a gate judging still says it waits: %s", gateLine(g))
	}
	// And a later report listing it waiting again does not hold a gate that started.
	if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
		t.Error("a gate that started waits again")
	}
}

// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
	sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
	pairs := []judged{{module: "build-agent", node: "novox"}}
	waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
		State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
	f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
		At: &sent, Current: true, Busy: waiting}}}
	now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
	if busyWait(g, pairs, f, now) {
		t.Fatal("a gate waited past the engine's bound and the slack")
	}
	if !g.Busy.PastBound || !g.Since.Equal(now) {
		t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
	}
	if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
		t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
	}
}

// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
// report not about the gate's send says nothing of it.
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
	sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
		Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
	pairs := []judged{{module: "build-agent", node: "ace"}}
	at := sent.Add(time.Minute)
	older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
		At: &at, Declared: "d2", ReportedSequence: 9}}}
	if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
		t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
	}
	elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
		At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
			State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
	if busyWait(g, pairs, elsewhere, at) {
		t.Fatal("a wait reported about an earlier declaration held the gate")
	}
}

// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
	seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
	pairs := []judged{{module: "build-agent", node: "novox"}}
	ahead := seen.Add(time.Hour)
	f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
		At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
			State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
	if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
		t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
	}
	if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
		t.Fatal("the gate stopped waiting within the bound on its own clock")
	}
	if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
		t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
	}
}

// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
	modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
		{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
		{"id": "agent-env", "type": "file"}}}}
	lines := busyNotSent(modules, false)
	if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
		!strings.Contains(lines[0], "replaced at once") {
		t.Fatalf("the plan does not say the busy not sent: %v", lines)
	}
	if lines := busyNotSent(modules, true); len(lines) != 0 {
		t.Fatalf("a busy sent is said as not sent: %v", lines)
	}
}