diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 54664d70..13063e9d 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1443,6 +1443,8 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error } // A release held after a failed one, while builds still wait for a gate (ADR 0236). out = append(out, backlogObservation()...) + // And a rest that went silent (novox/hq ADR 0353 §4). + out = append(out, silentObservation(ctx, d.open.inventory)...) return sortedFound(dedupeObservations(out)), nil } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index a5f705b1..7d60b16a 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{ "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), + "release-silent": worded(func(o conditions.Observation) words { + return releaseSilentWords(o.Also) + }), "facts-stale": worded(func(o conditions.Observation) words { return words{Headline: "Merge checks use outdated facts", Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + @@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words { Resolved: "Resolved: the held updates are released"} } +// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4). +func releaseSilentWords(machines []string) words { + where := "Some machines" + if len(machines) > 0 { + where = conditions.Capital(namesWords(machines, 4)) + } + return words{Headline: "Machines did not answer an update", + Needs: "check that they are awake and on the mesh; the next walk sends them what waits.", + Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " + + "they were left as they were. What they run is not known to be the update.", + Resolved: "Resolved: the next walk reached them"} +} + // reloginNeeds is what an account waiting for its groups needs (ADR 0252). func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index e53ca77d..daaea3ff 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -81,6 +81,15 @@ type moveFacts struct { srcs map[string]map[string]string passed map[string]map[string]bool plans []inventory.Plan + // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). + self string +} + +type walkingKey struct{} + +// withWalking says a release plan is the one sending: its own gates never hold its own sends back. +func withWalking(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, walkingKey{}, id) } func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { @@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er return f, err } f.plans, err = inv.OpenPlans(ctx) + f.self, _ = ctx.Value(walkingKey{}).(string) return f, err } @@ -193,7 +203,7 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string, // feet. A release keeps no module records: its open gate's carried moves are its walk. func (f moveFacts) walkedBy(module, node, to string) string { for _, p := range f.plans { - if !p.Open() { + if !p.Open() || (p.Release != nil && p.ID == f.self) { continue } if p.Release != nil { @@ -203,7 +213,8 @@ func (f moveFacts) walkedBy(module, node, to string) string { gates = append(gates, g) } for _, g := range gates { - if g == nil || g.Verdict != "" || g.Silent { + // A send kept and not yet made walks nothing yet (ADR 0353 §8). + if g == nil || g.Verdict != "" || g.Silent || g.Pending { continue } for _, c := range g.Carried { @@ -734,6 +745,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, if r.Settings != nil { return advanceSettingsWalk(ctx, open, p) } + ctx = withWalking(ctx, p.ID) now := time.Now().UTC() if r.Rest { return advanceRest(ctx, open, p, now) @@ -915,6 +927,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } } if len(pending) > 0 { + // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on + // a machine since its step was chosen could reach a second machine in it; the step is chosen again. + if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { + return false, err + } else if grew != "" { + for _, n := range pending { + delete(r.Gates, n) + } + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return true, err + } + p.Note = "chosen again: " + grew + if !opened && waiting != "" { + p.Note += "; " + waiting + } + return true, nil + } var sent, already, waiting []string for _, node := range pending { out, err := sendKept(ctx, open, p, node, r.Gates[node], now) @@ -1056,12 +1086,44 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. if len(r.Failed) > 0 { p.State = inventory.PlanFailed end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end + } else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) { + // **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it. + p.State = inventory.PlanFailed + end = "no machine of the rest reported on its send; " + end } p.Note = strings.Join(append(notes, end), "; ") fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } +// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; +// empty when none would. +func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return "", err + } + for _, node := range pending { + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + continue // sendKept says it waits + } + if err != nil { + return "", err + } + kept := r.Gates[node].Carried + for _, mv := range moves { + if f.gated(mv.Module, mv.To) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { + return c.Module == mv.Module && sameCommit(c.To, mv.To) + }) { + continue + } + return fmt.Sprintf("%s's build %s, which passed no gate, came to wait on %s", mv.Module, short(mv.To), node), nil + } + } + return "", nil +} + // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { @@ -1136,7 +1198,11 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s waiting[node] = moves nodes = append(nodes, node) } - step := restStep(nodes, waiting, f.gated) + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return false, "", err + } + step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) if len(step) == 0 { if len(walked) > 0 { return false, "waiting before " + strings.Join(walked, "; "), nil @@ -1156,16 +1222,30 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s return true, "", nil } +// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the +// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. +func oneAtATime(module string, m catalogue.Manifest) bool { + if module == hostModule { + return true + } + return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) +} + // restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless // a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's // first machine. Pure. -func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool) []string { +// +// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a +// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus +// shows only as its silence and no rollback reaches it. +func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, + alone func(module string) bool) []string { first := map[string]string{} var out []string for _, n := range nodes { var unpassed []string for _, mv := range waiting[n] { - if !passed(mv.Module, mv.To) { + if !passed(mv.Module, mv.To) || alone(mv.Module) { unpassed = append(unpassed, mv.Module+"@"+mv.To) } } @@ -1180,6 +1260,33 @@ func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed return out } +// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 +// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. +func silentObservation(ctx context.Context, inv *inventory.Inventory) []conditions.Observation { + recent, err := inv.RecentPlans(ctx, 20) + if err != nil { + return nil + } + for _, p := range recent { + if p.Release == nil || p.Release.Settings != nil || p.Open() { + continue + } + if len(p.Release.Silent) == 0 { + return nil + } + machines := append([]string(nil), p.Release.Silent...) + sort.Strings(machines) + o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, + Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+ + "left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)} + w := releaseSilentWords(machines) + o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs + return []conditions.Observation{o} + } + return nil +} + // backlogObservation is what the gate's probe says of a release held after a failure. func backlogObservation() []conditions.Observation { if backlogNow.held == "" || len(backlogNow.waiting) == 0 { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 4d537ba2..68fdca5d 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1059,16 +1059,33 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from // the readings of a gate already judging it there. - if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 { + // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before + // (issue 280: what it ran before is then that build, and a failure leaves it as it was). + if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && + appliedLastSend(ctx, inv, node) { if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { - adoptSend(ctx, open, p, node, modules, runningOf) + adoptSend(ctx, open, p, node, modules, before, runningOf) return nil } } + // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced + // between the send and keeping it reads it here, and a failure still puts the machine back. + kept := false + for _, m := range modules { + s := p.Modules[m] + if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { + s.Previous, kept = before[m], true + } + } + if kept { + if err := inv.SavePlan(ctx, p); err != nil { + return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) + } + } owns := make([]inventory.CarriedMove, 0, len(modules)) for _, m := range modules { s := p.Modules[m] - owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build}) + owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) } // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. @@ -1083,10 +1100,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string sentWhat := sentNow(ctx, inv, sent) for _, m := range modules { s := p.Modules[m] - // What the first machine ran before: what a failed gate puts back (ADR 0236). - if beforeKnown { - s.Previous = before[m] - } + // What the first machine ran before (ADR 0236), kept above before the send. s.First, s.FirstAt = sent, &now s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} @@ -1119,7 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string // from the machine's last send, and takes over the readings of an open gate of another walk judging the same build // there — whose carried move also says what the machine ran before, for a rollback. func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, - runningOf map[string][]string) { + before map[string]string, runningOf map[string][]string) { inv := open.inventory since := time.Now().UTC() last, found, err := inv.SentTo(ctx, node) @@ -1127,11 +1141,17 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string since = last.At } lead := modules[0] - other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit) + other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) for _, m := range modules { s := p.Modules[m] - if m == lead && from != "" { + // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send + // moved it from; else the build itself, so a failure leaves it as it was (issue 280). + switch { + case s.Previous != "": + case m == lead && from != "": s.Previous = from + default: + s.Previous = before[m] } at := since s.First, s.FirstAt = []string{node}, &at @@ -1156,13 +1176,34 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string fmt.Printf("%s: %s\n", p.ID, p.Note) } +// appliedLastSend says a machine reported it applied the declaration it was sent last. +func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { + reports, err := inv.LastReports(ctx) + if err != nil { + return false + } + for _, r := range reports { + if r.Node == node { + return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied + } + } + return false +} + // judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the // walk's id, and what the machine ran before; nil when none does. -func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit string) (*inventory.PlanGate, string, string) { +func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { plans, err := inv.OpenPlans(ctx) if err != nil { return nil, "", "" } + // The same build, by its id where both name one, else by its commit. + same := func(b, c string) bool { + if b != "" && build != "" { + return b == build + } + return sameCommit(c, commit) + } usable := func(g *inventory.PlanGate) bool { return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && slices.Contains(g.Machines, node) @@ -1181,14 +1222,14 @@ func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, n continue } for _, c := range g.Carried { - if c.Module == module && c.Node == node && sameCommit(c.To, commit) { + if c.Module == module && c.Node == node && same(c.Build, c.To) { return g, q.ID, c.From } } } continue } - if s := q.Modules[module]; s != nil && usable(s.Gate) && sameCommit(s.Commit, commit) { + if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { return s.Gate, q.ID, s.Previous } } diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index dc926d98..e8ff4a2d 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" ) @@ -165,9 +166,18 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() - // The old controller sent anchor app c2 and was replaced before it kept the send. - if _, err := sendRollout(ctx, g.open, []string{"anchor"}); err != nil { - t.Fatal(err) + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) } last, _, err := g.open.inventory.SentTo(ctx, "anchor") if err != nil { @@ -181,6 +191,10 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + } } // A release's send kept before it was made, and made, is judged by the controller that takes over without being @@ -224,11 +238,131 @@ func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, } passed := func(module, commit string) bool { return module == "app" && commit == "c2" } - step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed) + never := func(string) bool { return false } + step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) } - if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }); !slices.Equal(step, []string{"laptop"}) { + if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { t.Fatalf("a machine whose every build passed waits: %v", step) } } + +// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step +// carries either to one machine only, while what else passed goes everywhere at once. +func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: hostModule, To: "h2"}}, + "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "dns", To: "d2"}}, + "ace": {{Module: "dns", To: "d2"}}, + } + resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} + alone := func(m string) bool { + if m == "dns" { + return oneAtATime(m, resolver) + } + return oneAtATime(m, catalogue.Manifest{Module: m}) + } + step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, + func(string, string) bool { return true }, alone) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) + } + if oneAtATime("app", catalogue.Manifest{Module: "app"}) { + t.Fatal("a leaf module is held to one machine at a time") + } +} + +// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + +// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is +// ordered after every other. +func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) + if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { + t.Fatal(err) + } + said := events(t) + gateEvery = 0 + for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { + t.Fatalf("the order is %v", p.Release.Order) + } + laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") + if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { + t.Fatalf("the control node is not sent last in the rest's step: %v", *said) + } +} + +// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put +// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). +func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs := silentObservation(ctx, inv) + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/testdata/store-tests b/testdata/store-tests index 4d3eaa55..48692606 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -66,3 +66,6 @@ ./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere ./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain ./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain +./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk +./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep +./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid