Keep a walk's sends to what was kept, and say a silent rest (hq ADR 0353 review)

A release's own gates held its other sends back for ever; an adopted send
lost what the machine ran before; a kept send could grow a build no gate
passed; a take-over matched by commit alone; a rest that went silent ended
done. The node-engine and the resolver still go one machine at a time, since
a fault that cuts a machine off shows only as silence.

cmd/mesh-controller/rest_at_once_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -165,9 +166,18 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
 	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
-	// The old controller sent anchor app c2 and was replaced before it kept the send.
-	if _, err := sendRollout(ctx, g.open, []string{"anchor"}); err != nil {
-		t.Fatal(err)
+	// The old controller sends anchor app c2 and is replaced before it keeps the send.
+	made := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if _, err := made(ctx, open, names); err != nil {
+			return nil, err
+		}
+		return nil, context.Canceled
+	}
+	advancePlans(ctx, g.open)
+	sendRollout = made
+	if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
+		t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
 	}
 	last, _, err := g.open.inventory.SentTo(ctx, "anchor")
 	if err != nil {
@@ -181,6 +191,10 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
 	if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
 		t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
 	}
+	// What anchor ran before was kept before the send, so a failure still puts it back (§5).
+	if s.Previous != "c1" || s.Gate.From != "c1" {
+		t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
+	}
 }
 
 // A release's send kept before it was made, and made, is judged by the controller that takes over without being
@@ -224,11 +238,131 @@ func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
 		"novox":  {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
 	}
 	passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
-	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed)
+	never := func(string) bool { return false }
+	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
 	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
 		t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
 	}
-	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }); !slices.Equal(step, []string{"laptop"}) {
+	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
 		t.Fatalf("a machine whose every build passed waits: %v", step)
 	}
 }
+
+// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
+// carries either to one machine only, while what else passed goes everywhere at once.
+func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: hostModule, To: "h2"}},
+		"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "dns", To: "d2"}},
+		"ace":    {{Module: "dns", To: "d2"}},
+	}
+	resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
+	alone := func(m string) bool {
+		if m == "dns" {
+			return oneAtATime(m, resolver)
+		}
+		return oneAtATime(m, catalogue.Manifest{Module: m})
+	}
+	step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
+		func(string, string) bool { return true }, alone)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
+	}
+	if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
+		t.Fatal("a leaf module is held to one machine at a time")
+	}
+}
+
+// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
+func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	gateEvery = 0
+	b.broken["anchor"] = true
+	for i := 0; i < 6 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	for _, s := range b.sent {
+		if !slices.Equal(s, []string{"anchor"}) {
+			t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
+		}
+	}
+}
+
+// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
+// ordered after every other.
+func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
+	if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
+		t.Fatal(err)
+	}
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
+		t.Fatalf("the order is %v", p.Release.Order)
+	}
+	laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
+	if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
+		t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
+	}
+}
+
+// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
+// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
+func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	reporting := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
+			b.sent = append(b.sent, names)
+			current, _ := open.inventory.CurrentBuilds(ctx)
+			return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
+				map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
+		}
+		return reporting(ctx, open, names)
+	}
+	t.Cleanup(func() { sendRollout = reporting })
+	gateEvery = 0
+	for i := 0; i < 8; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	g := p.Release.Gates["laptop"]
+	if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
+		t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
+	}
+	since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
+	g.Since = &since
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	p = b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
+		t.Fatal("a silent machine's build was marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
+		t.Fatalf("laptop was put back: %v", sent)
+	}
+	obs := silentObservation(ctx, inv)
+	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
+		t.Fatalf("the silence is not said: %+v", obs)
+	}
+}